<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 Goal
Step1:  Large dense
Step2:  Large sparse
Step3:  Large dynamic sparse
Step4:  Small dense
Step5:  Comparing all
Step6:  Test with kwinners
Step7:  test_noise = True
use_kwinners = True
epochs = 3
on_perc = 0.1
large dense
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLP(config=config)
model = BaseModel(network=network)
model.setup()
print("\nLarge Dense")
large_dense = model.train(dataset, epochs, test_noise=test_noise);
large sparse
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLP(config=config)
config = dict(debug_sparse=True, on_perc=on_perc)
model = SparseModel(network=network, config=config)
model.setup()
print("\nLarge Sparse")
large_sparse = model.train(dataset, epochs, test_noise=test_noise);
dynamic sparse
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLPHeb(config=config)
config = dict(
    weight_prune_perc=0.15,
    hebbian_prune_perc=0.60,
    pruning_es =False,
    pruning_active=True,
    pruning_interval=1,
    debug_sparse=True,
    on_perc=on_perc,
    hebbian_grow=True  
)
model = DSNNMixedHeb(network=network, config=config)
model.setup()
print("\nDynamic Sparse")
dynamic_sparse = model.train(dataset, epochs, test_noise=test_noise);
small dense
config = dict(hidden_sizes=[12,16,23], use_kwinners=use_kwinners)
network = MLP(config=config)
load a regular base model for training
model = BaseModel(network=network)
model.setup()
print("\nSmall Dense")
small_dense = model.train(dataset, epochs, test_noise=test_noise);
<ASSISTANT_TASK:>
Python Code:
%load_ext autoreload
%autoreload 2
import sys
sys.path.append(os.path.expanduser("~/nta/nupic.research/projects/"))
# general imports
import os
import numpy as np
# torch imports
import torch
import torch.optim as optim
import torch.optim.lr_scheduler as schedulers
import torch.nn as nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from torchsummary import summary
# nupic research imports
from nupic.research.frameworks.pytorch.image_transforms import RandomNoise
from nupic.torch.modules import KWinners
# local library
from dynamic_sparse.networks import *
from dynamic_sparse.models import *
from dynamic_sparse.common.utils import *
# local files
import math
# plotting
import matplotlib.pyplot as plt
from matplotlib import rcParams
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
rcParams['figure.figsize'] = (12,6)
PATH_TO_WHERE_DATASET_WILL_BE_SAVED = PATH = "~/nta/datasets"
# load dataset
config = (dict(
    dataset_name="MNIST",
    data_dir="~/nta/datasets",
    test_noise=True
))
dataset = Dataset(config)
Explanation: Goal: Investigate how DSNN fares in a toy problem. 
Compare following models:
- Large dense (same architecture as large sparse, but dense)
- Small dense (same number of params as large sparse, but dense)
- Large sparse
- Large sparse + dynamic sparse
End of explanation
# load a regular network
network = MLP()
# load a regular base model for training
model = BaseModel(network=network)
model.setup()
%%time
large_dense = model.train(dataset, 3, test_noise=False);
Explanation: Large dense
End of explanation
# load a regular network
network = MLP()
# load a regular base model for training
config = dict(
    debug_sparse=True,
    on_perc=0.1,
)
model = SparseModel(network=network, config=config)
model.setup()
%%time
large_sparse = model.train(dataset, 3, test_noise=False);
results = large_sparse
h, w = math.ceil(len(results)/4), 4 
combinations = []
for i in range(h):
    for j in range(w):
        combinations.append((i,j))
fig, axs = plt.subplots(h, w, gridspec_kw={'hspace': 0.5, 'wspace': 0.5})
fig.set_size_inches(16,16)
for (i, j), k in zip(combinations[:len(results)], sorted(results.keys())):
    axs[i, j].plot(range(len(results[k])), results[k])
    axs[i, j].set_title(k)
Explanation: Large sparse
End of explanation
# load a regular network
network = MLPHeb()
# load a regular base model for training
config = dict(
    weight_prune_perc=0.15,
    hebbian_prune_perc=0.60,
    pruning_es =False,
    pruning_active=True,
    pruning_interval=1,
    debug_sparse=True,
    on_perc=0.1,
)
model = DSNNMixedHeb(network=network, config=config)
model.setup()
# debug sparsity
# [b/a for a,b in zip([784*400, 400*400, 400*400, 400*10], model.num_params)]
%%time
dynamic_sparse = model.train(dataset, 3, test_noise=False);
# results.keys(), len(results)
# results
results = dynamic_sparse
h, w = math.ceil(len(results)/4), 4 
combinations = []
for i in range(h):
    for j in range(w):
        combinations.append((i,j))
fig, axs = plt.subplots(h, w, gridspec_kw={'hspace': 0.5, 'wspace': 0.5})
fig.set_size_inches(16,16)
for (i, j), k in zip(combinations[:len(results)], sorted(results.keys())):
    axs[i, j].plot(range(len(results[k])), results[k])
    axs[i, j].set_title(k)
Explanation: Large dynamic sparse
End of explanation
# load a regular network
config = dict(
    hidden_sizes = [12,16,23]
)
network = MLP(config=config)
# load a regular base model for training
model = BaseModel(network=network)
model.setup()
%%time
small_dense = model.train(dataset, 3, test_noise=False);
Explanation: Small dense
End of explanation
from matplotlib import rcParams
rcParams['image.cmap'] = 'Accent'
plt.figure(figsize=(14,7))
names = ['large_dense', 'large_sparse', 'dynamic_sparse', 'small_dense']
for name, res in zip(names, [large_dense, large_sparse, dynamic_sparse, small_dense]):
    plt.plot(res['train_acc'], label=name)
plt.legend()
plt.ylim((0.85,1.0));
plt.figure(figsize=(14,7))
names = ['large_dense', 'large_sparse', 'dynamic_sparse', 'small_dense']
for name, res in zip(names, [large_dense, large_sparse, dynamic_sparse, small_dense]):
    plt.plot(res['val_acc'], label=name)
plt.legend()
plt.ylim((0.85,1.0));
Explanation: Comparing all
End of explanation
from models import *
from networks import *
# load dataset
config = (dict(
    dataset_name="MNIST",
    data_dir="~/nta/datasets",
    test_noise=True
))
dataset = Dataset(config)
test_noise = True
use_kwinners = True
epochs = 3
on_perc = 0.1
# large dense
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLP(config=config)
model = BaseModel(network=network)
model.setup()
print("\nLarge Dense")
large_dense = model.train(dataset, epochs, test_noise=test_noise);
# large sparse
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLP(config=config)
config = dict(debug_sparse=True, on_perc=on_perc)
model = SparseModel(network=network, config=config)
model.setup()
print("\nLarge Sparse")
large_sparse = model.train(dataset, epochs, test_noise=test_noise);
# dynamic sparse
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLPHeb(config=config)
config = dict(
    weight_prune_perc=0.15,
    hebbian_prune_perc=0.60,
    pruning_es =False,
    pruning_active=True,
    pruning_interval=1,
    debug_sparse=True,
    on_perc=on_perc,
    hebbian_grow=False    
)
model = DSNNMixedHeb(network=network, config=config)
model.setup()
print("\nDynamic Sparse")
dynamic_sparse = model.train(dataset, epochs, test_noise=test_noise);
# small dense
config = dict(hidden_sizes=[12,16,23], use_kwinners=use_kwinners)
network = MLP(config=config)
# load a regular base model for training
model = BaseModel(network=network)
model.setup()
print("\nSmall Dense")
small_dense = model.train(dataset, epochs, test_noise=test_noise);
Explanation: Test with kwinners
End of explanation
plt.figure(figsize=(14,7))
names = ['large_dense', 'large_sparse', 'dynamic_sparse', 'small_dense']
for name, res in zip(names, [large_dense, large_sparse, dynamic_sparse, small_dense]):
    plt.plot(res['train_acc'], label=name)
plt.legend()
plt.ylim((0.85,1.0));
plt.figure(figsize=(14,7))
names = ['large_dense', 'large_sparse', 'dynamic_sparse', 'small_dense']
for name, res in zip(names, [large_dense, large_sparse, dynamic_sparse, small_dense]):
    plt.plot(res['val_acc'], label=name)
plt.legend()
plt.ylim((0.85,1.0));
plt.figure(figsize=(14,7))
names = ['large_dense', 'large_sparse', 'dynamic_sparse', 'small_dense']
for name, res in zip(names, [large_dense, large_sparse, dynamic_sparse, small_dense]):
    plt.plot(res['noise_acc'], label=name)
plt.legend()
plt.ylim((0.85,1.0));
Explanation: test_noise = True
use_kwinners = True
epochs = 3
on_perc = 0.1
large dense
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLP(config=config)
model = BaseModel(network=network)
model.setup()
print("\nLarge Dense")
large_dense = model.train(dataset, epochs, test_noise=test_noise);
large sparse
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLP(config=config)
config = dict(debug_sparse=True, on_perc=on_perc)
model = SparseModel(network=network, config=config)
model.setup()
print("\nLarge Sparse")
large_sparse = model.train(dataset, epochs, test_noise=test_noise);
dynamic sparse
config = dict(hidden_sizes=[100,100,100], use_kwinners=use_kwinners)
network = MLPHeb(config=config)
config = dict(
    weight_prune_perc=0.15,
    hebbian_prune_perc=0.60,
    pruning_es =False,
    pruning_active=True,
    pruning_interval=1,
    debug_sparse=True,
    on_perc=on_perc,
    hebbian_grow=True  
)
model = DSNNMixedHeb(network=network, config=config)
model.setup()
print("\nDynamic Sparse")
dynamic_sparse = model.train(dataset, epochs, test_noise=test_noise);
small dense
config = dict(hidden_sizes=[12,16,23], use_kwinners=use_kwinners)
network = MLP(config=config)
load a regular base model for training
model = BaseModel(network=network)
model.setup()
print("\nSmall Dense")
small_dense = model.train(dataset, epochs, test_noise=test_noise);
End of explanation
<END_TASK>