#!/usr/bin/env python
# coding: utf-8

# ## CIFAR 10

# In[1]:


get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('reload_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')


# In[2]:


from fastai.conv_learner import *
PATH = Path("data/cifar10/")
# PATH = Path("/home/ubuntu/data/cifar10/")
os.makedirs(PATH,exist_ok=True)


# In[3]:


from torchvision import transforms, datasets


# In[4]:


torch.cuda.set_device(0)


# In[5]:


classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
stats = (np.array([ 0.4914 ,  0.48216,  0.44653]), np.array([ 0.24703,  0.24349,  0.26159]))


# In[6]:


bs=256


# In[7]:


num_workers = num_cpus()


# In[8]:


num_workers = 16


# In[9]:


traindir = str(PATH/'train')
valdir = str(PATH/'test')
tfms = [transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))]

train_dataset = datasets.ImageFolder(
    traindir,
    transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
    ] + tfms))

train_loader = torch.utils.data.DataLoader(
    train_dataset, batch_size=bs, shuffle=True, num_workers=num_workers, pin_memory=True)

val_dataset = datasets.ImageFolder(valdir, transforms.Compose(tfms))

val_loader = torch.utils.data.DataLoader(
    val_dataset, batch_size=bs*2, shuffle=False, num_workers=num_workers, pin_memory=True)


# In[10]:


data = ModelData(PATH, train_loader, val_loader)
data.sz=32


# In[17]:


from models.wideresnet import WideResNet


# In[18]:


m = WideResNet(3, 3, num_classes=10, k=6, drop_p=0.)


# In[19]:


m = nn.DataParallel(m, [0,1,2,3])


# In[20]:


lr = 1.3


# In[21]:


learn = ConvLearner.from_model_data(m, data)
learn.crit = nn.CrossEntropyLoss()
learn.metrics = [accuracy]
wd=1e-4


# In[ ]:





# In[22]:


# DP: m = WideResNet(depth=22, num_classes=10, widen_factor=6, dropRate=0.)
learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))


# In[ ]:


get_ipython().run_line_magic('time', 'learn.fit(lr, 1, wds=wd, cycle_len=30, use_clr_beta=(20, 20, 0.95, 0.85))')


# In[ ]:





# In[15]:


# DP: m = WideResNet(depth=22, num_classes=10, widen_factor=6, dropRate=0.)
learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))


# In[16]:


get_ipython().run_line_magic('time', 'learn.fit(lr, 1, wds=wd, cycle_len=30, use_clr_beta=(20, 20, 0.95, 0.85))')


# In[ ]:





# In[18]:


learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))


# In[19]:


get_ipython().run_line_magic('time', 'learn.fit(lr, 1, wds=wd, cycle_len=40, use_clr_beta=(10, 15, 0.95, 0.85))')


# In[ ]:





# In[16]:


learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))


# In[17]:


get_ipython().run_line_magic('time', 'learn.fit(1., 1, wds=wd, cycle_len=30, use_clr_beta=(10, 25, 0.95, 0.85))')


# In[ ]:





# In[22]:


learn.fit(lr/10, 1, wds=wd, cycle_len=1, use_clr_beta=(100, 1, 0.9, 0.8))


# In[23]:


get_ipython().run_line_magic('time', 'learn.fit(1., 1, wds=wd, cycle_len=30, use_clr_beta=(10, 25, 0.95, 0.85))')


# In[ ]:





# In[15]:


get_ipython().run_line_magic('time', 'learn.fit(lr, 1, wds=wd, cycle_len=40, use_clr_beta=(100, 15, 0.95, 0.85))')


# In[ ]:





# In[51]:


# darknet 2222 lr 1.3 65 cl
get_ipython().run_line_magic('time', 'learn.fit(lr, 1, wds=wd, cycle_len=65, use_clr_beta=(30, 20, 0.95, 0.85))')


# In[ ]: