Data block foundations¶

In [1]:

%install '.package(path: "$cwd/FastaiNotebook_07_batchnorm")' FastaiNotebook_07_batchnorm

Installing packages:
	.package(path: "/home/ubuntu/fastai_docs/dev_swift/FastaiNotebook_07_batchnorm")
		FastaiNotebook_07_batchnorm
With SwiftPM flags: []
Working in: /tmp/tmp_xqc507i/swift-install
Fetching https://github.com/mxcl/Path.swift
Fetching https://github.com/JustHTTP/Just
Completed resolution in 3.70s
Cloning https://github.com/mxcl/Path.swift
Resolving https://github.com/mxcl/Path.swift at 0.16.2
Cloning https://github.com/JustHTTP/Just
Resolving https://github.com/JustHTTP/Just at 0.7.1
Compile Swift Module 'Just' (1 sources)
Compile Swift Module 'Path' (9 sources)
Compile Swift Module 'FastaiNotebook_07_batchnorm' (12 sources)
Compile Swift Module 'jupyterInstalledPackages' (1 sources)
Linking ./.build/x86_64-unknown-linux/debug/libjupyterInstalledPackages.so
Initializing Swift...
Installation complete!

In [2]:

import FastaiNotebook_07_batchnorm
%include "EnableIPythonDisplay.swift"
IPythonDisplay.shell.enable_matplotlib("inline")

Out[2]:

('inline', 'module://ipykernel.pylab.backend_inline')

In [3]:

// export
import Path
import TensorFlow
import Python

In [4]:

let plt = Python.import("matplotlib.pyplot")

Image ItemList¶

Download Imagenette¶

In [5]:

//export
public let dataPath = Path.home/".fastai"/"data"

In [6]:

//export
public func downloadImagette(path: Path = dataPath) -> Path {
    let url = "https://s3.amazonaws.com/fast-ai-imageclas/imagenette-160.tgz"
    let fname = "imagenette-160"
    let file = path/fname
    try! path.mkdir(.p)
    if !file.exists {
        downloadFile(url, dest:(path/"\(fname).tgz").string)
        _ = shellCommand("/bin/tar", ["-xzf", (path/"\(fname).tgz").string, "-C", path.string])
    }
    return file
}

In [7]:

let path = downloadImagette()

In [8]:

print(path.ls())

[Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/train)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/models))]

In [9]:

print((path/"val").ls())

[Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03888257)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03445777)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03425413)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n01440764)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03028079)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n02979186)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03394916)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n02102040)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03417042)), Path.Entry(kind: Path.Entry.Kind.directory, path: Path(/home/ubuntu/.fastai/data/imagenette-160/val/n03000684))]

Let's have a look inside a class folder (the first class is tench):

In [10]:

let pathTench = path/"val"/"n01440764"

In [11]:

let imgFn = Path.home/".fastai/data/imagenette-160/val/n01440764/ILSVRC2012_val_00006697.JPEG"
imgFn.string

Out[11]:

"/home/ubuntu/.fastai/data/imagenette-160/val/n01440764/ILSVRC2012_val_00006697.JPEG"

In [12]:

let imgBytes = Raw.readFile(filename: StringTensor(imgFn.string))
let decodedImg = Raw.decodeJpeg(contents: imgBytes, channels: 3, dctMethod: "")

In [13]:

decodedImg.shape

Out[13]:

▿ TensorShape
  ▿ dimensions : 3 elements
    - 0 : 160
    - 1 : 213
    - 2 : 3

In [14]:

let numpyImg = decodedImg.makeNumpyArray()
plt.imshow(numpyImg) 
plt.axis("off")
plt.show()

Out[14]:

None

In [15]:

//export
public func fetchFiles(path: Path, recurse: Bool = false, extensions: [String]? = nil) -> [Path]{
    var res: [Path] = []
    for p in try! path.ls(){
        if p.kind == .directory && recurse { 
            res += fetchFiles(path: p.path, recurse: recurse, extensions: extensions)
        } else if extensions == nil || extensions!.contains(p.path.extension.lowercased) {
            res.append(p.path)
        }
    }
    return res
}

In [16]:

time() { let fNames = fetchFiles(path: path, recurse: true, extensions: ["jpeg", "jpg"]) }

1121.42815 ms

In [17]:

let fNames = fetchFiles(path: path, recurse: true, extensions: ["jpeg", "jpg"])

In [18]:

fNames.count == 13394

Out[18]:

true

Prepare the data¶

Dataset can handle all the transforms that go on a Tensor, including opening an image and resizing it since it takes StringTensor. That makes the tfms attribute of ItemList irrelevant, so ItemList is just an array of Item with a path (if get method seems useful later, we can add it).

In [19]:

// export
public struct ItemList<Item>{
    public var items: [Item]
    public let path: Path
    
    public init(items: [Item], path: Path){
        (self.items,self.path) = (items,path)
    }
}

In [20]:

// export
public extension ItemList where Item == Path{
    init(fromFolder path: Path, extensions: [String], recurse: Bool = true) {
        self.init(items: fetchFiles(path: path, recurse: recurse, extensions: extensions),
                  path:  path)
    }
}

In [21]:

let il = ItemList(fromFolder: path, extensions: ["jpeg", "jpg"])

Split¶

In [22]:

// export
public struct SplitData<Item>{
    public let train: ItemList<Item>
    public let valid: ItemList<Item>
    public var path: Path { return train.path }
    
    public init(train: ItemList<Item>, valid: ItemList<Item>){
        (self.train, self.valid) = (train, valid)
    }
    
    public init(_ il: ItemList<Item>, fromFunc: (Item) -> Bool){
        var (trn, val): ([Item], [Item]) = ([], [])
        for x in il.items {
            if fromFunc(x) { val.append(x) }
            else           { trn.append(x) }
        }
        self.init(train: ItemList(items: trn, path: il.path),
                  valid: ItemList(items: val, path: il.path))
    }
}

In [23]:

// export
public func grandParentSplitter(fName: Path, valid: String = "valid") -> Bool{
    return fName.parent.parent.basename() == valid
}

In [24]:

let sd = SplitData(il, fromFunc: {grandParentSplitter(fName: $0, valid: "val")})

Processor¶

In [25]:

// export
public protocol Processor {
    associatedtype Input
    associatedtype Output
    
    mutating func initState(items: [Input])
    func process1(item: Input) -> Output
    func deprocess1(item: Output) -> Input
}

In [26]:

// export
public extension Processor {
    
    func process(items: [Input]) -> [Output]{
        return items.map(){process1(item: $0)}
    }
    
    func deprocess(items: [Output]) -> [Input]{
        return items.map(){deprocess1(item: $0)}
    }
}

In [27]:

// export
public struct NoopProcessor<Item>: Processor {
    public init() {}
    public typealias Input = Item
    public typealias Output = Item
    
    public mutating func initState(items: [Input]){}
    
    public func process1  (item: Input)  -> Output { return item }
    public func deprocess1(item: Output) -> Input  { return item }
}

In [28]:

// export
public struct CategoryProcessor: Processor {
    public init() {}
    public typealias Input = String
    public typealias Output = Int32
    public var vocab: [Input]? = nil
    public var reverseMap: [Input: Output]? = nil
    
    public mutating func initState(items: [Input]){
        vocab = Array(Set(items)).sorted()
        reverseMap = [:]
        for (i,x) in vocab!.enumerated(){ reverseMap![x] = Int32(i) }
    }
    
    public func process1  (item: Input)  -> Output { return reverseMap![item]! }
    public func deprocess1(item: Output) -> Input  { return vocab![Int(item)] }
}

Label¶

When we build the datasets, we don't need to return a tupe (item, label) but to have the tensor(s) with the items and the tensor(s) with the labels separately.

In [29]:

//export
public struct LabeledItemList<PI,PL> where PI: Processor, PL: Processor{
    public var items: [PI.Output]
    public var labels: [PL.Output]
    public let path: Path
    public var procItem: PI
    public var procLabel: PL
    
    public init(rawItems: [PI.Input], rawLabels: [PL.Input], path: Path, procItem: PI, procLabel: PL){
        (self.procItem,self.procLabel,self.path) = (procItem,procLabel,path)
        self.items = procItem.process(items: rawItems)
        self.labels = procLabel.process(items: rawLabels)
    }
    
    public init(_ il: ItemList<PI.Input>, fromFunc: (PI.Input) -> PL.Input, procItem: PI, procLabel: PL){
        self.init(rawItems:  il.items,
                  rawLabels: il.items.map{ fromFunc($0)},
                  path:      il.path,
                  procItem:  procItem,
                  procLabel: procLabel)
    }
    
    public func rawItem (_ idx: Int) -> PI.Input { return procItem.deprocess1 (item: items[idx])  }
    public func rawLabel(_ idx: Int) -> PL.Input { return procLabel.deprocess1(item: labels[idx]) }
}

In [30]:

//export
public struct SplitLabeledData<PI,PL> where PI: Processor, PL: Processor{
    public let train: LabeledItemList<PI,PL>
    public let valid: LabeledItemList<PI,PL>
    public var path: Path { return train.path }
    
    public init(train: LabeledItemList<PI,PL>, valid: LabeledItemList<PI,PL>){
        (self.train, self.valid) = (train, valid)
    }
    
    public init(_ sd: SplitData<PI.Input>, fromFunc: (PI.Input) -> PL.Input, procItem: inout PI, procLabel: inout PL){
        procItem.initState (items: sd.train.items)
        let trainLabels = sd.train.items.map{ fromFunc($0) }
        procLabel.initState(items: trainLabels)
        self.init(train: LabeledItemList(rawItems: sd.train.items, rawLabels: trainLabels, path: sd.path, 
                                         procItem: procItem, procLabel: procLabel),
                  valid: LabeledItemList(sd.valid, fromFunc: fromFunc, procItem: procItem, procLabel: procLabel))
    }
}

In [31]:

//export
public func parentLabeler(_ fName: Path) -> String { return fName.parent.basename() }

In [32]:

var (procItem,procLabel) = (NoopProcessor<Path>(),CategoryProcessor())
let sld = SplitLabeledData(sd, fromFunc: parentLabeler, procItem: &procItem, procLabel: &procLabel)

In [33]:

print(sld.train.labels[0])
print(sld.train.rawLabel(0))
print(sld.train.procLabel.vocab!)

9
n03888257
["n01440764", "n02102040", "n02979186", "n03000684", "n03028079", "n03394916", "n03417042", "n03425413", "n03445777", "n03888257"]

Datasets¶

To go in a Dataset, our array of items and array of labels need to be converted to tensors.

In [34]:

// export
public struct LabeledElement<I: TensorGroup, L: TensorGroup>: TensorGroup {
    public var xb: I
    public var yb: L    
    
    public init(xb: I, yb: L){
        (self.xb, self.yb) = (xb, yb)
    }
}

In [35]:

// export
public extension SplitLabeledData {
    func toDataBunch<XB, YB> (
        itemToTensor: ([PI.Output]) -> XB, labelToTensor: ([PL.Output]) -> YB, bs: Int = 64
    ) -> DataBunch<LabeledElement<XB, YB>> where XB: TensorGroup, YB: TensorGroup {
        let trainDs = Dataset<LabeledElement<XB, YB>>(
            elements: LabeledElement(xb: itemToTensor(train.items), yb: labelToTensor(train.labels)))
        let validDs = Dataset<LabeledElement<XB, YB>>(
            elements: LabeledElement(xb: itemToTensor(valid.items), yb: labelToTensor(valid.labels)))
        return DataBunch(train: trainDs, 
                         valid: validDs, 
                         trainLen: train.items.count, 
                         validLen: valid.items.count,
                         bs: bs)
    }
}

In [36]:

// export
public func pathsToTensor(_ paths: [Path]) -> StringTensor { return StringTensor(paths.map{ $0.string })}
public func intsToTensor(_ items: [Int32]) -> Tensor<Int32> { return Tensor<Int32>(items)}

In [37]:

let dataset = sld.toDataBunch(itemToTensor: pathsToTensor, labelToTensor: intsToTensor)

Transforms¶

We directly plug in to the dataset the transforms we want to apply.

In [38]:

// export
public func transformData<I,TI,L>(
    _ data: DataBunch<LabeledElement<I,L>>, 
    tfmItem: (I) -> TI
) -> DataBunch<DataBatch<TI,L>> 
where I: TensorGroup, TI: TensorGroup & Differentiable, L: TensorGroup{
    return DataBunch(train: data.train.innerDs.map{ DataBatch(xb: tfmItem($0.xb), yb: $0.yb) },
                     valid: data.valid.innerDs.map{ DataBatch(xb: tfmItem($0.xb), yb: $0.yb) },
                     trainLen: data.train.dsCount, 
                     validLen: data.valid.dsCount,
                     bs: data.train.bs)
}

In [39]:

// export
public func openAndResize(fname: StringTensor, size: Int) -> TF{
    let imgBytes = Raw.readFile(filename: fname)
    let decodedImg = Raw.decodeJpeg(contents: imgBytes, channels: 3, dctMethod: "")
    let resizedImg = Tensor<Float>(Raw.resizeNearestNeighbor(
        images: Tensor<UInt8>([decodedImg]), 
        size: Tensor<Int32>([Int32(size), Int32(size)]))) / 255.0
    return resizedImg.reshaped(to: TensorShape(size, size, 3))
}

In [40]:

let tfmData = transformData(dataset, tfmItem: { openAndResize(fname: $0, size: 128) })

In [41]:

var firstBatch: DataBatch<TF, TI>? = nil
for batch in tfmData.train.ds {
    firstBatch = batch
    break
}

In [56]:

firstBatch!.xb.shape

Out[56]:

▿ TensorShape
  ▿ dimensions : 4 elements
    - 0 : 64
    - 1 : 128
    - 2 : 128
    - 3 : 3

In [57]:

let (rows,cols) = (3,3)
plt.figure(figsize: [9, 9])
for i in 0..<(rows * cols) {
    let img = plt.subplot(rows, cols, i + 1)
    img.axis("off")
    let x = firstBatch!.xb[i].makeNumpyArray()
    img.imshow(x)
    let title = sld.train.procLabel.vocab![Int(firstBatch!.yb[i].scalarized())]
    img.set_title(title)
    if (i + 1) >= (rows * cols) { break }
}
plt.show()

Out[57]:

None

To summarize:¶

In [40]:

let il = ItemList(fromFolder: path, extensions: ["jpeg", "jpg"])
let sd = SplitData(il, fromFunc: {grandParentSplitter(fName: $0, valid: "val")})
var (procItem,procLabel) = (NoopProcessor<Path>(),CategoryProcessor())
let sld = SplitLabeledData(sd, fromFunc: parentLabeler, procItem: &procItem, procLabel: &procLabel)
var rawData = sld.toDataBunch(itemToTensor: pathsToTensor, labelToTensor: intsToTensor, bs:8)
let data = transformData(rawData, tfmItem: { openAndResize(fname: $0, size: 128) })

Let's try to train it:

In [41]:

//export 
public let imagenetStats = (mean: TF([0.485, 0.456, 0.406]), std: TF([0.229, 0.224, 0.225]))

In [42]:

let opt = SimpleSGD<CnnModelBN, Float>(learningRate: 0.1)
func modelInit() -> CnnModelBN { return CnnModelBN(channelIn: 3, nOut: 10, filters: [8, 16, 32, 64, 128]) }
let learner = Learner(data: data, lossFunction: softmaxCrossEntropy, optimizer: opt, initializingWith: modelInit)
let recorder = learner.makeDefaultDelegates(metrics: [accuracy])
learner.addDelegate(learner.makeNormalize(mean: imagenetStats.mean, std: imagenetStats.std))

In [45]:

learner.fit(1)

Epoch 0: [1.3219278, 0.546]

In [43]:

//export
public func prevPow2(_ x: Int) -> Int { 
    var res = 1
    while res <= x { res *= 2 }
    return res / 2
}

In [44]:

//export
public struct CNNModel: Layer {
    public var convs: [ConvBN<Float>]
    public var pool = FAAdaptiveAvgPool2D<Float>()
    public var flatten = Flatten<Float>()
    public var linear: FADense<Float>
    
    public init(channelIn: Int, nOut: Int, filters: [Int]){
        convs = []
        let (l1,l2) = (channelIn, prevPow2(channelIn * 9))
        convs = [ConvBN(l1,   l2,   stride: 1),
                 ConvBN(l2,   l2*2, stride: 2),
                 ConvBN(l2*2, l2*4, stride: 2)]
        let allFilters = [l2*4] + filters
        for i in 0..<filters.count { convs.append(ConvBN(allFilters[i], allFilters[i+1])) }
        linear = FADense<Float>(inputSize: filters.last!, outputSize: nOut)
    }
    
    @differentiable
    public func applied(to input: TF) -> TF {
        return input.sequenced(through: convs, pool, flatten, linear)
    }
}

In [45]:

let opt = SimpleSGD<CNNModel, Float>(learningRate: 0.1)
func modelInit() -> CNNModel { return CNNModel(channelIn: 3, nOut: 10, filters: [64, 64, 128, 256]) }
let learner = Learner(data: data, lossFunction: softmaxCrossEntropy, optimizer: opt, initializingWith: modelInit)
let recorder = learner.makeDefaultDelegates(metrics: [accuracy])
learner.addDelegate(learner.makeNormalize(mean: imagenetStats.mean, std: imagenetStats.std))

In [ ]:

learner.fit(1)

Fatal error: OOM when allocating tensor with shape[8,16,129,129] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc: file /swift-base/swift/stdlib/public/TensorFlow/CompilerRuntime.swift, line 2108
Current stack trace:
0    libswiftCore.so                    0x00007fb0afce4f70 _swift_stdlib_reportFatalErrorInFile + 115
1    libswiftCore.so                    0x00007fb0afc2d85c <unavailable> + 3012700
2    libswiftCore.so                    0x00007fb0afc2d94e <unavailable> + 3012942
3    libswiftCore.so                    0x00007fb0afa74772 <unavailable> + 1206130
4    libswiftCore.so                    0x00007fb0afbfa272 <unavailable> + 2802290
5    libswiftCore.so                    0x00007fb0afa73bb9 <unavailable> + 1203129
6    libswiftTensorFlow.so              0x00007fb0ace8b1e2 <unavailable> + 557538
7    libswiftTensorFlow.so              0x00007fb0ace89930 checkOk(_:file:line:) + 508
8    libswiftTensorFlow.so              0x00007fb0aceae030 _TFCCheckOk(_:) + 81
9    libswiftTensorFlow.so              0x00007fb0aceae020 _swift_tfc_CheckOk + 9

Export¶

In [46]:

notebookToScript(fname: (Path.cwd / "08_data_block.ipynb").string)

In [ ]: