Memory leak on exception¶

This notebook demonstrates a temporary memory leak that happens when an exception occurs

In [ ]:

import numpy as np
import gc, os, sys, time, psutil

In [ ]:

def consume_cpu_ram(n): return np.ones((n, n))
def consume_cpu_ram_128mb():  return consume_cpu_ram(2**12)

process = psutil.Process(os.getpid())
def cpu_ram_used():  return process.memory_info().rss

In [ ]:

def fail():
    x = consume_cpu_ram_128mb()
    raise ValueError("Ouch")

In [ ]:

before = cpu_ram_used()
fail()

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-4-3b8b03759f74> in <module>
      1 before = cpu_ram_used()
----> 2 fail()

<ipython-input-3-4f2e9e8bf6e8> in fail()
      1 def fail():
      2     x = consume_cpu_ram_128mb()
----> 3     raise ValueError("Ouch")

ValueError: Ouch

In [ ]:

_ = gc.collect()
after = cpu_ram_used()
diff = int((after-before)/2**20)
# Without the leak the difference should be ~0
# With the leak, the locals() are tied up in the tb and aren't released, so expecting ~128MB
assert diff < 2, f"got leak of {diff} MB"

In [ ]:

# force ipython to reset its %tb
raise ValueError("Reset")

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-6-31374e7d7384> in <module>
      1 # force ipython to reset its %tb
----> 2 raise ValueError("Reset")

ValueError: Reset

In [ ]:

_ = gc.collect()
after = cpu_ram_used()
diff = int((after-before)/2**20)
# 2nd exception resets locals tied up in tb, and they now can be released
# so expecting difference close to 0MB
assert diff < 2, f"got Difference {diff} MB"