mapping_mnist/mapping_mnist.ipynb

Idea here is to make a 2 param NN (x,y,loss), then do a grid search where we init model, do one epoch, record results. Then contour it.

So first I need something to convert it to features.... then I have a linear(3) convert 3 features to one thing...

In [67]:

from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

from tqdm import tqdm_notebook as tqdm
%pylab inline
import pickle
import itertools
import datetime

Populating the interactive namespace from numpy and matplotlib

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['seed']
`%matplotlib` prevents importing * from pylab and numpy
  "\n`%matplotlib` prevents importing * from pylab and numpy"

In [68]:

import matplotlib.pyplot as plt

from tqdm import tqdm_notebook as tqdm

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import LogNorm
from matplotlib import animation
from IPython.display import HTML

from collections import defaultdict
from itertools import zip_longest
from functools import partial

In [69]:

cuda=False
seed=0
batch_size=512


ts = datetime.datetime.utcnow().strftime('%Y%m%d_%H-%M-%S')

model_path = 'data/model_%s.pickle' % ts
points_file = 'data/points_%s.pickle' % ts


torch.manual_seed(seed)
if cuda:
    torch.cuda.manual_seed(seed)

In [19]:

train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=batch_size, shuffle=True)

# test_loader = torch.utils.data.DataLoader(
#     datasets.MNIST('../data', train=False, transform=transforms.Compose([
#                        transforms.ToTensor(),
#                        transforms.Normalize((0.1307,), (0.3081,))
#                    ])),
#     batch_size=args.test_batch_size, shuffle=True, **kwargs)

Model

In [426]:

# https://github.com/pytorch/examples/blob/master/mnist/main.py
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 2, kernel_size=28, stride=(28,28))
#         self.conv2 = nn.Conv2d(10, 2, kernel_size=5)
#         self.fc1 = nn.Linear(320, 2)
        self.fc2 = nn.Linear(2, 1)

    def forward(self, x):
        x = F.tanh(self.conv1(x))
#         x = F.relu(F.max_pool2d(self.conv2(x), 2))
        x = x.view(-1, 2)
#         x = F.tanh(self.fc1(x))
#         x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.sigmoid(x)

model = Net()
optimizer = optim.SGD(model.parameters(), lr=1e-3)

In [204]:

# nn.Conv2d?

Pretrain

In [205]:

# I got from 0.7 to 0.4 in 100 epochs
epochs=200
for epoch in range(1, epochs + 1):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        if cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        
        # reduce this to a binary problem
        target = (target>5).type(torch.FloatTensor)
        
        optimizer.zero_grad()
        output = model(data)
        loss = F.binary_cross_entropy(output, target)
        loss.backward()
        optimizer.step()
    print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
        epoch, batch_idx * len(data), len(train_loader.dataset),
        100. * batch_idx / len(train_loader), loss.data[0]))

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torch/nn/functional.py:767: UserWarning: Using a target size (torch.Size([512])) that is different to the input size (torch.Size([512, 1])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))
/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torch/nn/functional.py:767: UserWarning: Using a target size (torch.Size([96])) that is different to the input size (torch.Size([96, 1])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))

Train Epoch: 1 [11232/60000 (99%)]	Loss: 0.636986
Train Epoch: 2 [11232/60000 (99%)]	Loss: 0.622902
Train Epoch: 3 [11232/60000 (99%)]	Loss: 0.602751
Train Epoch: 4 [11232/60000 (99%)]	Loss: 0.579772
Train Epoch: 5 [11232/60000 (99%)]	Loss: 0.581861

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-205-68b33f9dff4c> in <module>()
      3 for epoch in range(1, epochs + 1):
      4     model.train()
----> 5     for batch_idx, (data, target) in enumerate(train_loader):
      6         if cuda:
      7             data, target = data.cuda(), target.cuda()

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torch/utils/data/dataloader.py in __next__(self)
    177         if self.num_workers == 0:  # same-process loading
    178             indices = next(self.sample_iter)  # may raise StopIteration
--> 179             batch = self.collate_fn([self.dataset[i] for i in indices])
    180             if self.pin_memory:
    181                 batch = pin_memory_batch(batch)

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torch/utils/data/dataloader.py in <listcomp>(.0)
    177         if self.num_workers == 0:  # same-process loading
    178             indices = next(self.sample_iter)  # may raise StopIteration
--> 179             batch = self.collate_fn([self.dataset[i] for i in indices])
    180             if self.pin_memory:
    181                 batch = pin_memory_batch(batch)

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torchvision/datasets/mnist.py in __getitem__(self, index)
     50         # doing this so that it is consistent with all other datasets
     51         # to return a PIL Image
---> 52         img = Image.fromarray(img.numpy(), mode='L')
     53 
     54         if self.transform is not None:

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/PIL/Image.py in fromarray(obj, mode)
   2436             obj = obj.tostring()
   2437 
-> 2438     return frombuffer(mode, size, obj, "raw", rawmode, 0, 1)
   2439 
   2440 

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/PIL/Image.py in frombuffer(mode, size, data, decoder_name, *args)
   2386                 core.map_buffer(data, size, decoder_name, None, 0, args)
   2387                 )
-> 2388             im.readonly = 1
   2389             return im
   2390 

KeyboardInterrupt: 

In [425]:

minima = model.fc2.weight.data.numpy()
minima_z = loss.data.numpy()[0]
minima, minima_z

Out [425]:

(array([[ 0.22772104,  0.45285982]], dtype=float32), 5.7115297)

In [427]:

torch.save(model, model_path)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torch/serialization.py:147: UserWarning: Couldn't retrieve source code for container of type Net. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "

Explore

In [428]:

model2 = torch.load(model_path)
# freeze all except last layer
mlayers = [
    model2.conv1,
#     model2.conv2,
#     model2.fc1
]
for layer in mlayers:
    for param in layer.parameters():
          param.requires_grad = False

In [429]:

model2.fc2.weight.data.size()

Out [429]:

torch.Size([1, 2])

In [ ]:

In [430]:

xm, ym = torch.Tensor(minima.T)
xm, ym = xm.numpy()[0], ym.numpy()[0]

In [431]:

xs=torch.arange(xm-40,xm+0,0.3)
# xs = torch.Tensor(np.sort(np.random.normal(xm,100,100)))
ys=torch.arange(ym-40,xm+40,0.3)
# ys = torch.Tensor(np.sort(np.random.normal(ym,100,100)))
xys = list(itertools.product(xs,ys))

In [434]:

xs=torch.arange(xm-20,xm+20,1)
# xs = torch.Tensor(np.sort(np.random.normal(xm,100,100)))
ys=torch.arange(ym-20,xm+20,1)
# ys = torch.Tensor(np.sort(np.random.normal(ym,100,100)))
xys = list(itertools.product(xs,ys))

In [ ]:

# grid search, run this overnight
points = []

for x,y in tqdm(xys):
    
    torch.manual_seed(seed)
    if cuda:
        torch.cuda.manual_seed(seed)

    # set last layer weights
    model2.fc2.weight.data =  torch.Tensor([[y,x]])

    zs=[]
    dzs=[]
    batches=0

    for batch_idx, (data, target) in enumerate(train_loader):
        if cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)

        # reduce this to a binary problem
        target = (target>5).type(torch.FloatTensor)

        optimizer.zero_grad()
        output = model2(data)
        loss = F.binary_cross_entropy(output, target)
        loss.backward()
        optimizer.step()
        dzs.append(model2.fc2.weight.grad.data.numpy())
        zs.append(loss.data.numpy()[0])
        batches += 1 
        
        if batches>6:
            break
            
    output = model2(data)
    loss = F.binary_cross_entropy(output, target)

    z = np.mean(zs)
    z_var = np.std(zs)
    dz = np.mean(dzs,0)
    dz_var = np.std(dzs,0)
    points.append([x,y,z, z_var, dz, dz_var, batches])

[Data output - unsupported data type map[string]interface {} for mime type application/vnd.jupyter.widget-view+json]

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/torch/nn/functional.py:767: UserWarning: Using a target size (torch.Size([512])) that is different to the input size (torch.Size([512, 1])) is deprecated. Please ensure they have the same size.
  "Please ensure they have the same size.".format(target.size(), input.size()))

In [415]:

x = np.array([p[0] for p in points])
y = np.array([p[1] for p in points])
z = np.array([p[2] for p in points])
zv = np.array([p[3] for p in points])
dz = np.array([p[4] for p in points])[:,0,:]
dzv = np.array([p[5] for p in points])[:,0,:]

In [416]:

plt.figure(figsize=(10,8))
# x,y,z,zv,dz,dzv, n = np.array(points).T
plt.quiver(x, y, dz[:,0], dz[:,1], angles='xy', scale_units='xy', scale=1)
plt.show()

In [417]:

plt.figure(figsize=(10,8))
# x,y,z,zv,dz,dzv, n = np.array(points).T
plt.quiver(x, y, dzv[:,0], dzv[:,1], angles='xy', scale_units='xy', scale=1)
plt.show()

In [418]:

plt.figure(figsize=(10,8))
# x,y,z,zv,dz,dzv, n = np.array(points).T
plt.scatter(x,y,c=dz[:,0])
plt.show()

In [419]:

plt.figure(figsize=(10,8))
plt.scatter(x,y,c=z)
plt.title('loss')
plt.colorbar()

Out [419]:

<matplotlib.colorbar.Colorbar at 0x7f0ad88e1898>

In [420]:

plt.figure(figsize=(10,8))
plt.scatter(x,y,c=z)
plt.title('log loss')
plt.colorbar()

Out [420]:

<matplotlib.colorbar.Colorbar at 0x7f0adad59518>

In [ ]:

In [421]:

plt.figure(figsize=(10,8))
plt.title('log std')
plt.scatter(x,y,c=np.log(v))
plt.colorbar()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in to_rgba(c, alpha)
    140     try:
--> 141         rgba = _colors_full_map.cache[c, alpha]
    142     except (KeyError, TypeError):  # Not in cache, or unhashable.

KeyError: (-0.93098905452915248, None)

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-421-03316127cc9a> in <module>()
      1 plt.figure(figsize=(10,8))
      2 plt.title('log std')
----> 3 plt.scatter(x,y,c=np.log(v))
      4 plt.colorbar()

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/pyplot.py in scatter(x, y, s, c, marker, cmap, norm, vmin, vmax, alpha, linewidths, verts, edgecolors, hold, data, **kwargs)
   3432                          vmin=vmin, vmax=vmax, alpha=alpha,
   3433                          linewidths=linewidths, verts=verts,
-> 3434                          edgecolors=edgecolors, data=data, **kwargs)
   3435     finally:
   3436         ax._hold = washold

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/__init__.py in inner(ax, *args, **kwargs)
   1896                     warnings.warn(msg % (label_namer, func.__name__),
   1897                                   RuntimeWarning, stacklevel=2)
-> 1898             return func(ax, *args, **kwargs)
   1899         pre_doc = inner.__doc__
   1900         if pre_doc is None:

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/axes/_axes.py in scatter(self, x, y, s, c, marker, cmap, norm, vmin, vmax, alpha, linewidths, verts, edgecolors, **kwargs)
   4032                 offsets=offsets,
   4033                 transOffset=kwargs.pop('transform', self.transData),
-> 4034                 alpha=alpha
   4035                 )
   4036         collection.set_transform(mtransforms.IdentityTransform())

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in __init__(self, paths, sizes, **kwargs)
    900         """
    901 
--> 902         Collection.__init__(self, **kwargs)
    903         self.set_paths(paths)
    904         self.set_sizes(sizes)

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in __init__(self, edgecolors, facecolors, linewidths, linestyles, antialiaseds, offsets, transOffset, norm, cmap, pickradius, hatch, urls, offset_position, zorder, **kwargs)
    138 
    139         self._hatch_color = mcolors.to_rgba(mpl.rcParams['hatch.color'])
--> 140         self.set_facecolor(facecolors)
    141         self.set_edgecolor(edgecolors)
    142         self.set_linewidth(linewidths)

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in set_facecolor(self, c)
    679         """
    680         self._original_facecolor = c
--> 681         self._set_facecolor(c)
    682 
    683     def set_facecolors(self, c):

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in _set_facecolor(self, c)
    664         except AttributeError:
    665             pass
--> 666         self._facecolors = mcolors.to_rgba_array(c, self._alpha)
    667         self.stale = True
    668 

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in to_rgba_array(c, alpha)
    237     result = np.empty((len(c), 4), float)
    238     for i, cc in enumerate(c):
--> 239         result[i] = to_rgba(cc, alpha)
    240     return result
    241 

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in to_rgba(c, alpha)
    141         rgba = _colors_full_map.cache[c, alpha]
    142     except (KeyError, TypeError):  # Not in cache, or unhashable.
--> 143         rgba = _to_rgba_no_colorcycle(c, alpha)
    144         try:
    145             _colors_full_map.cache[c, alpha] = rgba

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in _to_rgba_no_colorcycle(c, alpha)
    192         # float)` and `np.array(...).astype(float)` all convert "0.5" to 0.5.
    193         # Test dimensionality to reject single floats.
--> 194         raise ValueError("Invalid RGBA argument: {!r}".format(orig_c))
    195     # Return a tuple to prevent the cached value from being modified.
    196     c = tuple(c.astype(float))

ValueError: Invalid RGBA argument: -0.93098905452915248

In [422]:

plt.figure(figsize=(10,8))
plt.title('std')
plt.scatter(x,y,c=v)
plt.colorbar()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in to_rgba(c, alpha)
    140     try:
--> 141         rgba = _colors_full_map.cache[c, alpha]
    142     except (KeyError, TypeError):  # Not in cache, or unhashable.

KeyError: (0.39416366815567017, None)

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-422-0554dbdab3a9> in <module>()
      1 plt.figure(figsize=(10,8))
      2 plt.title('std')
----> 3 plt.scatter(x,y,c=v)
      4 plt.colorbar()

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/pyplot.py in scatter(x, y, s, c, marker, cmap, norm, vmin, vmax, alpha, linewidths, verts, edgecolors, hold, data, **kwargs)
   3432                          vmin=vmin, vmax=vmax, alpha=alpha,
   3433                          linewidths=linewidths, verts=verts,
-> 3434                          edgecolors=edgecolors, data=data, **kwargs)
   3435     finally:
   3436         ax._hold = washold

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/__init__.py in inner(ax, *args, **kwargs)
   1896                     warnings.warn(msg % (label_namer, func.__name__),
   1897                                   RuntimeWarning, stacklevel=2)
-> 1898             return func(ax, *args, **kwargs)
   1899         pre_doc = inner.__doc__
   1900         if pre_doc is None:

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/axes/_axes.py in scatter(self, x, y, s, c, marker, cmap, norm, vmin, vmax, alpha, linewidths, verts, edgecolors, **kwargs)
   4032                 offsets=offsets,
   4033                 transOffset=kwargs.pop('transform', self.transData),
-> 4034                 alpha=alpha
   4035                 )
   4036         collection.set_transform(mtransforms.IdentityTransform())

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in __init__(self, paths, sizes, **kwargs)
    900         """
    901 
--> 902         Collection.__init__(self, **kwargs)
    903         self.set_paths(paths)
    904         self.set_sizes(sizes)

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in __init__(self, edgecolors, facecolors, linewidths, linestyles, antialiaseds, offsets, transOffset, norm, cmap, pickradius, hatch, urls, offset_position, zorder, **kwargs)
    138 
    139         self._hatch_color = mcolors.to_rgba(mpl.rcParams['hatch.color'])
--> 140         self.set_facecolor(facecolors)
    141         self.set_edgecolor(edgecolors)
    142         self.set_linewidth(linewidths)

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in set_facecolor(self, c)
    679         """
    680         self._original_facecolor = c
--> 681         self._set_facecolor(c)
    682 
    683     def set_facecolors(self, c):

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/collections.py in _set_facecolor(self, c)
    664         except AttributeError:
    665             pass
--> 666         self._facecolors = mcolors.to_rgba_array(c, self._alpha)
    667         self.stale = True
    668 

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in to_rgba_array(c, alpha)
    237     result = np.empty((len(c), 4), float)
    238     for i, cc in enumerate(c):
--> 239         result[i] = to_rgba(cc, alpha)
    240     return result
    241 

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in to_rgba(c, alpha)
    141         rgba = _colors_full_map.cache[c, alpha]
    142     except (KeyError, TypeError):  # Not in cache, or unhashable.
--> 143         rgba = _to_rgba_no_colorcycle(c, alpha)
    144         try:
    145             _colors_full_map.cache[c, alpha] = rgba

~/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/colors.py in _to_rgba_no_colorcycle(c, alpha)
    192         # float)` and `np.array(...).astype(float)` all convert "0.5" to 0.5.
    193         # Test dimensionality to reject single floats.
--> 194         raise ValueError("Invalid RGBA argument: {!r}".format(orig_c))
    195     # Return a tuple to prevent the cached value from being modified.
    196     c = tuple(c.astype(float))

ValueError: Invalid RGBA argument: 0.39416366815567017

In [348]:

# save
pickle.dump('points267x266', open(points_file,'wb'))

In [328]:

# save

pickle.dump(points, open(points_file,'wb'))

In [78]:

# save
points = pickle.load(open(points_file,'rb'))

In [346]:

x,y,z,v,n = np.array(points).T

# scale lossses to they look OK
z=(z-z.min())*10000000
logzmax = np.log(z.max())
logzmax

Out [346]:

19.090419312520552

In [349]:

# now reshape into square arrays
x = x.reshape((len(xs),len(ys)))
y = y.reshape((len(xs),len(ys)))
z = z.reshape((len(xs),len(ys)))
# dz = dz.reshape((len(xs),len(ys)))
v = v.reshape((len(xs),len(ys)))
# dzv = dzv.reshape((len(xs),len(ys)))
z.shape

Out [349]:

(267, 266)

In [345]:

ax = plt.gca()
cm=ax.contour(x, y, z, levels=np.logspace(0, logzmax//2, 55), norm=LogNorm(), cmap=plt.cm.jet, alpha=0.15)
plt.colorbar(cm)
# ax.plot(*minima_, 'r*', markersize=10)
# ax.plot(*problem.x0, 'r+', markersize=10)
plt.title('minst: a slice of the problem surface for 2 neurons: loss')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')
plt.show()

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/contour.py:1518: UserWarning: Log scale: values of z <= 0 have been masked
  warnings.warn('Log scale: values of z <= 0 have been masked')

In [351]:

# scale lossses to they look OK
v=(v-v.min())*10000000
logvmax = np.log(v.max())
logvmax

Out [351]:

15.494302237578435

In [352]:

ax = plt.gca()
cm=ax.contour(x, y, v, levels=np.logspace(0, logvmax//2, 55), norm=LogNorm(), cmap=plt.cm.jet, alpha=0.15)
plt.colorbar(cm)
# ax.plot(*minima_, 'r*', markersize=10)
# ax.plot(*problem.x0, 'r+', markersize=10)
plt.title('minst: a slice of the problem surface for 2 neurons - variance')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')
plt.show()

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/matplotlib/contour.py:1518: UserWarning: Log scale: values of z <= 0 have been masked
  warnings.warn('Log scale: values of z <= 0 have been masked')

Generate path

In [ ]:

model3 = torch.load(model_path)
# freeze all except last layer
mlayers = [
    model3.conv1,
    model3.conv2,
    model3.fc1
]
for layer in mlayers:
    for param in layer.parameters():
          param.requires_grad = False
            
optimizer = optim.SGD(model3.parameters(), lr=1e-3)

In [ ]:

epochs=100
points2=[]
for epoch in range(1, epochs + 1):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        if cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        
        # reduce this to a binary problem
        target = (target>5).type(torch.FloatTensor)
        
        optimizer.zero_grad()
        output = model3(data)
        loss = F.binary_cross_entropy(output, target)
        loss.backward()
        optimizer.step()
        
        x,y=model3.conv2.weights.data
        z=loss.data
        points2.append([x,y,z])
    
    
    print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
        epoch, batch_idx * len(data), len(train_loader.dataset),
        100. * batch_idx / len(train_loader), loss.data[0]))