Notebook

fastai OOM memory recover¶

In [ ]:

%reload_ext autoreload
%autoreload 2
%matplotlib inline

In [ ]:

from fastai import *
from fastai.vision import *
from ipyexperiments import IPyExperimentsPytorch

In [ ]:

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
assert str(device) == 'cuda:0', f"we want GPU, got {device}"

In [ ]:

torch.ones(1).cuda() # preload

Out[ ]:

tensor([1.], device='cuda:0')

In [ ]:

import pynvml, torch, gc

pynvml.nvmlInit()
id = torch.cuda.current_device()
def mem_free():
    gc.collect()
    torch.cuda.empty_cache()
    handle = pynvml.nvmlDeviceGetHandleByIndex(id)
    info = pynvml.nvmlDeviceGetMemoryInfo(handle)
    return int( info.free / 2**20 )

def mem_report(): print(f"free mem={mem_free()}")

def mem_allocate_mbs(n, fatal=False): 
    " allocate n MBs, return the var holding it on success, None on failure "
    if n < 6: return None # don't try to allocate less than 6MB
    try:
        d = int(2**9*n**0.5)
        return torch.ones((d, d)).cuda().contiguous()
    except Exception as e:
        if not fatal: return None
        raise e
        
def leave_free_mbs(n):
    " consume whatever memory is needed so that n MBs are left free "
    avail = mem_free()
    assert avail > n, f"already have less available mem than desired {n}MBs"
    consume = avail - n
    print(f"consuming {consume}MB to bring free mem to {n}MBs")
    return mem_allocate_mbs(consume, fatal=True)

buf = leave_free_mbs(1000)

consuming 6614MB to bring free mem to 1000MBs

In [ ]:

exp = IPyExperimentsPytorch()

*** Experiment started with the Pytorch backend
Device: ID 0, GeForce GTX 1070 Ti (8119 RAM)


*** Current state:
RAM:  Used  Free  Total      Util
CPU:  2284 19978  31588 MB   7.23% 
GPU:  7119  1000   8119 MB  87.68% 


･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.000
･ CPU:         0       0     2284 MB |
･ GPU:         0       0     7119 MB |

In [ ]:

path = untar_data(URLs.PETS)
path_anno = path/'annotations'
path_img = path/'images'
fnames = get_image_files(path_img)
np.random.seed(2)
pat = re.compile(r'/([^/]+)_\d+.jpg$')

･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.044
･ CPU:         7       0     2291 MB |
･ GPU:         0       0     7119 MB |

In [ ]:

#bs=128
bs = 32

data = ImageDataBunch.from_name_re(path_img, fnames, pat, ds_tfms=None, size=224, bs=bs).normalize(imagenet_stats)

･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.239
･ CPU:         1       3     2343 MB |
･ GPU:         0       0     7119 MB |

In [ ]:

learn = create_cnn(data, models.resnet34, metrics=accuracy)

･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.446
･ CPU:         0       0     2480 MB |
･ GPU:       110       0     7229 MB |

In [ ]:

learn.fit_one_cycle(1,1e-2)

0.00% [0/1 00:00<00:00]

epoch	train_loss	valid_loss	accuracy

Interrupted

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-10-e85309452eba> in <module>
----> 1 learn.fit_one_cycle(1,1e-2)

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, wd, callbacks, **kwargs)
     20     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor,
     21                                         pct_start=pct_start, **kwargs))
---> 22     learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
     23 
     24 def lr_find(learn:Learner, start_lr:Floats=1e-7, end_lr:Floats=10, num_it:int=100, stop_div:bool=True, **kwargs:Any):

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
    174         callbacks = [cb(self) for cb in self.callback_fns] + listify(callbacks)
    175         fit(epochs, self.model, self.loss_func, opt=self.opt, data=self.data, metrics=self.metrics,
--> 176             callbacks=self.callbacks+callbacks)
    177 
    178     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/utils/mem.py in wrapper(*args, **kwargs)
     68         except:
     69             type, val, tb = get_ref_free_exc_info() # must!
---> 70             raise type(val).with_traceback(tb) from None
     71     return wrapper

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/utils/mem.py in wrapper(*args, **kwargs)
     65     def wrapper(*args, **kwargs):
     66         try:
---> 67             return func(*args, **kwargs)
     68         except:
     69             type, val, tb = get_ref_free_exc_info() # must!

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/basic_train.py in fit(***failed resolving arguments***)
     96     except Exception as e:
     97         exception = e
---> 98         raise e
     99     finally: cb_handler.on_train_end(exception)
    100 

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/basic_train.py in fit(***failed resolving arguments***)
     86             for xb,yb in progress_bar(data.train_dl, parent=pbar):
     87                 xb, yb = cb_handler.on_batch_begin(xb, yb)
---> 88                 loss = loss_batch(model, xb, yb, loss_func, opt, cb_handler)
     89                 if cb_handler.on_batch_end(loss): break
     90 

/mnt/nvme1/fast.ai-1/br/fastai/master/fastai/basic_train.py in loss_batch(***failed resolving arguments***)
     18     if not is_listy(xb): xb = [xb]
     19     if not is_listy(yb): yb = [yb]
---> 20     out = model(*xb)
     21     out = cb_handler.on_loss_begin(out)
     22 

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(***failed resolving arguments***)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(***failed resolving arguments***)
     90     def forward(self, input):
     91         for module in self._modules.values():
---> 92             input = module(input)
     93         return input
     94 

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(***failed resolving arguments***)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(***failed resolving arguments***)
     90     def forward(self, input):
     91         for module in self._modules.values():
---> 92             input = module(input)
     93         return input
     94 

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(***failed resolving arguments***)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(***failed resolving arguments***)
     90     def forward(self, input):
     91         for module in self._modules.values():
---> 92             input = module(input)
     93         return input
     94 

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(***failed resolving arguments***)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torchvision/models/resnet.py in forward(***failed resolving arguments***)
     47 
     48         if self.downsample is not None:
---> 49             residual = self.downsample(x)
     50 
     51         out += residual

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(***failed resolving arguments***)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(***failed resolving arguments***)
     90     def forward(self, input):
     91         for module in self._modules.values():
---> 92             input = module(input)
     93         return input
     94 

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(***failed resolving arguments***)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/modules/batchnorm.py in forward(***failed resolving arguments***)
     74             input, self.running_mean, self.running_var, self.weight, self.bias,
     75             self.training or not self.track_running_stats,
---> 76             exponential_average_factor, self.eps)
     77 
     78     def extra_repr(self):

~/anaconda3/envs/fastai/lib/python3.7/site-packages/torch/nn/functional.py in batch_norm(***failed resolving arguments***)
   1621     return torch.batch_norm(
   1622         input, weight, bias, running_mean, running_var,
-> 1623         training, momentum, eps, torch.backends.cudnn.enabled
   1624     )
   1625 

RuntimeError: CUDA out of memory. Tried to allocate 6.12 MiB (GPU 0; 7.93 GiB total capacity; 7.34 GiB already allocated; 2.56 MiB free; 3.15 MiB cached)

･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:01.641
･ CPU:         1       0     2487 MB |
･ GPU:         6     882     7235 MB |

In [ ]:

# Now run this cell, after the OOM cell above

mem_report()
del buf # that was just the buffer to make the test reliable

# Now you can get all of the GPU memory back
mem_report()

free mem=884
free mem=7498
･ RAM: △Consumed △Peaked  Used Total | Exec time 0:00:00.066
･ CPU:         0       0     2487 MB |
･ GPU:     -6614    6614      621 MB |