In [ ]:

%install '.package(path: "$cwd/FastaiNotebook_00_load_data")' FastaiNotebook_00_load_data

Installing packages:
	.package(path: "/home/ubuntu/fastai_docs/dev_swift/FastaiNotebook_00_load_data")
		FastaiNotebook_00_load_data
With SwiftPM flags: []
Working in: /tmp/tmpxb804zjr
Fetching https://github.com/mxcl/Path.swift
Fetching https://github.com/JustHTTP/Just
Completed resolution in 1.20s
Cloning https://github.com/mxcl/Path.swift
Resolving https://github.com/mxcl/Path.swift at 0.16.2
Cloning https://github.com/JustHTTP/Just
Resolving https://github.com/JustHTTP/Just at 0.7.1
Compile Swift Module 'Path' (9 sources)
Compile Swift Module 'Just' (1 sources)
Compile Swift Module 'FastaiNotebook_00_load_data' (1 sources)
Compile Swift Module 'jupyterInstalledPackages' (1 sources)
Linking ./.build/x86_64-unknown-linux/debug/libjupyterInstalledPackages.so
Initializing Swift...
Loading library...
Installation complete!

In [ ]:

import FastaiNotebook_00_load_data

Load the data¶

In [ ]:

import Path
import TensorFlow

In [ ]:

let (xTrain, yTrain, xValid, yValid) = loadMNIST(path: mnistPath, flat: true)

In [ ]:

import Python

In [ ]:

let np = Python.import("numpy")
let plt = Python.import("matplotlib.pyplot")

In [ ]:

%include "EnableIPythonDisplay.swift"
IPythonDisplay.shell.enable_matplotlib("inline")

Out[ ]:

('inline', 'module://ipykernel.pylab.backend_inline')

In [ ]:

let img = np.array(xTrain[0].array.scalars).reshape(28,28)

In [ ]:

plt.figure(figsize: [5,5])
let fig = plt.imshow(X:img, cmap:"gray")
plt.show(fig)

Out[ ]:

None

Matmul¶

In [ ]:

var weights = Tensor<Float>(randomNormal: [784, 10]) / sqrt(784)
var bias = Tensor(repeating: 0.0, shape: [10])

From scratch¶

Flattened version in pure swift

In [ ]:

func swiftMatmul(a:[Float], b:[Float], aShape:[Int], bShape:[Int]) -> [Float]{
    var res:[Float] = Array(repeating: 0.0, count: aShape[0] * bShape[1])
    for i in 0..<aShape[0]{
        for j in 0..<bShape[1]{
            for k in 0..<aShape[1]{
                res[i + aShape[0] * j] += a[i + aShape[0]*k] * b[k + bShape[0]*j]
            }
        }
    }
    return res
}

In [ ]:

let flatA = xTrain[0..<5].array.scalars
let flatB = weights.array.scalars

In [ ]:

let res = swiftMatmul(a: flatA, b:flatB, aShape:[5,784], bShape:[784,10])

In [ ]:

time(repeating: 100) { let _ = swiftMatmul(a: flatA, b:flatB, aShape:[5,784], bShape:[784,10])}

0.23496165999999996 ms

With pointers

In [ ]:

func swiftMatmultPointers(a:[Float], b:[Float], aShape:[Int], bShape:[Int]) -> [Float]{
    var res:[Float] = Array(repeating: 0.0, count: aShape[0] * bShape[1])
    res.withUnsafeMutableBufferPointer { res_ in
        a.withUnsafeBufferPointer { a_ in
            b.withUnsafeBufferPointer { b_ in                              
                for i in 0..<aShape[0]{
                    for j in 0..<bShape[1]{
                        for k in 0..<aShape[1]{
                            res_[i + aShape[0] * j] += a_[i + aShape[0]*k] * b_[k + bShape[0]*j]
                        }
                    }
                }
            }
        }
    }
    return res
}

In [ ]:

time(repeating: 100) { let _ = swiftMatmultPointers(a: flatA, b:flatB, aShape:[5,784], bShape:[784,10])}

0.14819732 ms

With ShapedArray

In [ ]:

func shapedArrayMatmul(_ a:ShapedArray<Float>, _ b:ShapedArray<Float>) -> ShapedArray<Float>{
    var res:ShapedArray<Float> = ShapedArray(repeating: 0.0, shape: [a.shape[0], b.shape[1]])
    for i in 0..<a.shape[0]{
        for j in 0..<b.shape[1]{
            for k in 0..<a.shape[1]{
                res[i][j].scalar! += a[i][k].scalar! * b[k][j].scalar!
            }
        }
    }
    return res
}

In [ ]:

let m1 = xTrain[0..<5].reshaped(toShape: [5, 784])
let m2 = weights

In [ ]:

time() { let res = shapedArrayMatmul(m1.array, m2.array)}

3680.167613 ms

With Tensor

In [ ]:

func TensorMatmul(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    var res:Tensor<Float> = Tensor(repeating: 0.0, shape: [a.shape[0], b.shape[1]])
    for i in 0..<a.shape[0]{
        for j in 0..<b.shape[1]{
            for k in 0..<a.shape[1]{
                res[i][j] += a[i][k] * b[k][j]
            }
        }
    }
    return res
}

In [ ]:

time() { let res = TensorMatmul(m1, m2)}

error: Execution was interrupted, reason: signal SIGSTOP.
The process has been left at the point where it was interrupted, use "thread return -x" to return to the state before expression evaluation.

Too long to execture -> Looping over ShapedArray or Tensor indices is a bad idea!

Elementwise ops¶

Operators (+,-,*,/) are usually element-wise.

Examples of element-wise operations:

In [ ]:

var a = Tensor([10.0, 6, -4])
var b = Tensor([2.0, 8, 7])
(a,b)

Out[ ]:

▿ 2 elements
  - .0 : [10.0, 6.0, -4.0]
  - .1 : [2.0, 8.0, 7.0]

In [ ]:

a + b

Out[ ]:

[12.0, 14.0, 3.0]

Comparison operators (>,<,==,!=,...) are true if all the elements of the tensors satisfy the comparison. Elementwise versions have the . prefix: .>, .<, .== ...

In [ ]:

a < b

Out[ ]:

false

In [ ]:

a .< b

Out[ ]:

[false, true, true]

In [ ]:

var m = Tensor([1.0, 2, 3, 4, 5, 6, 7, 8, 9]).reshaped(to: [3,3])
m

Out[ ]:

[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]

In [ ]:

2 * m

Out[ ]:

[[2.0, 4.0, 6.0], [8.0, 10.0, 12.0], [14.0, 16.0, 18.0]]

In [ ]:

sqrt((m * m).sum().scalar!)

Out[ ]:

16.881943016134134

Elementwise matmul¶

In [ ]:

func elementWiseMatmul(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    var res = Tensor<Float>(zeros: [a.shape[1], b.shape[0]])
    for i in 0..<a.shape[0]{
        for j in 0..<b.shape[1]{
            res[i][j] = (a[i] * b.slice(lowerBounds: [0,j], upperBounds: [a.shape[1],j+1]).squeezingShape(at: 1)).sum()
        }
    }
    return res
}

In [ ]:

let res = elementWiseMatmul(m1, m2)

In [ ]:

time() { let _ = elementWiseMatmul(m1, m2)}

386.279412 ms

Broadcasting¶

Broadcasting with a scalar¶

In [ ]:

var a = Tensor([10.0, 6.0, -4.0])

In [ ]:

print(a .> 0)

[true, true, false]

In Tensorflow the operator > between tensors will return true if all the elements of the first tensor are greater than the ones of the second tensor. .> makes the memberwise comparison.

In [ ]:

print(a+1)

[11.0, 7.0, -3.0]

In [ ]:

2 * m

Out[ ]:

[[2.0, 4.0, 6.0], [8.0, 10.0, 12.0], [14.0, 16.0, 18.0]]

Broadcasting a vector with a matrix¶

In [ ]:

let c = Tensor([10.0,20.0,30.0])

By default, broadcasting is done by adding 1 dimensions to the beginning until dimensions of both objects match.

In [ ]:

m + c

Out[ ]:

[[11.0, 22.0, 33.0], [14.0, 25.0, 36.0], [17.0, 28.0, 39.0]]

In [ ]:

c + m

Out[ ]:

[[11.0, 22.0, 33.0], [14.0, 25.0, 36.0], [17.0, 28.0, 39.0]]

To broadcast on the other dimensions, one has to use expandingShape to add the dimension.

In [ ]:

m + c.expandingShape(at: 1)

Out[ ]:

[[11.0, 12.0, 13.0], [24.0, 25.0, 26.0], [37.0, 38.0, 39.0]]

In [ ]:

c.expandingShape(at: 1)

Out[ ]:

[[10.0], [20.0], [30.0]]

Matmult with broadcasting¶

In [ ]:

func broadcastMatmult(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    var res = Tensor<Float>(zeros: [a.shape[0], b.shape[1]])
    for i in 0..<a.shape[0]{
        res[i] = (a[i].expandingShape(at: 1) * b).sum(squeezingAxes: 0)
    }
    return res
}

In [ ]:

let res = broadcastMatmult(m1, m2)

In [ ]:

time(repeating: 100) { let _ = broadcastMatmult(m1, m2)}

1.9344620300000006 ms

A faster way is to concatenate the columns instead of filling the matrix result.

In [ ]:

func fastBcMatmul(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    var res: [Tensor<Float>] = []
    for i in 0..<a.shape[0]{
        res.append((a[i].expandingShape(at: 1) * b).sum(alongAxes: 0))
    }
    return Tensor<Float>(concatenating: res)
}

In [ ]:

time(repeating: 100) { let _ = fastBcMatmul(m1, m2)}

0.7782323200000002 ms

Broadcasting rules¶

In [ ]:

c.expandingShape(at: 0).shape

Out[ ]:

▿ TensorShape
  ▿ dimensions : 2 elements
    - 0 : 1
    - 1 : 3

In [ ]:

c.expandingShape(at: 1).shape

Out[ ]:

▿ TensorShape
  ▿ dimensions : 2 elements
    - 0 : 3
    - 1 : 1

In [ ]:

c.expandingShape(at: 0) * c.expandingShape(at: 1)

Out[ ]:

[[100.0, 200.0, 300.0], [200.0, 400.0, 600.0], [300.0, 600.0, 900.0]]

In [ ]:

c.expandingShape(at: 0) .> c.expandingShape(at: 1)

Out[ ]:

[[false, true, true], [false, false, true], [false, false, false]]

Tensorflow op¶

In [ ]:

time(repeating: 100) { let _ = matmul(m1, m2)}

0.024246269999999986 ms

Export¶

In [ ]:

notebookToScript(fname: (Path.cwd / "01_matmul.ipynb").string)

In [ ]: