In [1]:

%install '.package(path: "$cwd/FastaiNotebook_00_load_data")' FastaiNotebook_00_load_data

Installing packages:
	.package(path: "/home/ubuntu/fastai_docs/dev_swift/FastaiNotebook_00_load_data")
		FastaiNotebook_00_load_data
With SwiftPM flags: []
Working in: /tmp/tmptme5nbcv/swift-install
Fetching https://github.com/mxcl/Path.swift
Fetching https://github.com/JustHTTP/Just
Completed resolution in 1.24s
Cloning https://github.com/JustHTTP/Just
Resolving https://github.com/JustHTTP/Just at 0.7.1
Cloning https://github.com/mxcl/Path.swift
Resolving https://github.com/mxcl/Path.swift at 0.16.2
Compile Swift Module 'Just' (1 sources)
Compile Swift Module 'Path' (9 sources)
Compile Swift Module 'FastaiNotebook_00_load_data' (1 sources)
Compile Swift Module 'jupyterInstalledPackages' (1 sources)
Linking ./.build/x86_64-unknown-linux/debug/libjupyterInstalledPackages.so
Initializing Swift...
Installation complete!

In [2]:

import FastaiNotebook_00_load_data

Load the data¶

In [ ]:

import Path
import TensorFlow

In [ ]:

let (xTrain, yTrain, xValid, yValid) = loadMNIST(path: mnistPath, flat: true)

In [ ]:

//export
import Python

In [ ]:

//export
public let np = Python.import("numpy")
public let plt = Python.import("matplotlib.pyplot")

In [ ]:

%include "EnableIPythonDisplay.swift"
IPythonDisplay.shell.enable_matplotlib("inline")

Out[ ]:

('inline', 'module://ipykernel.pylab.backend_inline')

In [ ]:

let img = np.array(xTrain[0].array.scalars).reshape(28,28)

In [ ]:

plt.figure(figsize: [5,5])
let fig = plt.imshow(X:img, cmap:"gray")
plt.show(fig)

Out[ ]:

None

Matmul¶

In [ ]:

var weights = Tensor<Float>(randomNormal: [784, 10]) / sqrt(784)
var bias = Tensor(repeating: 0.0, shape: [10])

From scratch¶

Flattened version in pure swift

In [ ]:

let (arc,brc) = ((5, 784), (784, 10))

In [ ]:

func swiftMatmul(a:[Float], b:[Float], arc:(Int,Int), brc:(Int,Int)) -> [Float] {
    var res:[Float] = Array(repeating: 0.0, count: arc.0*brc.1)
    for i in 0..<arc.0 {
        for j in 0..<brc.1 {
            for k in 0..<arc.1 {
                res[i+arc.0*j] += a[i+arc.0*k] * b[k+brc.0*j]
            }
        }
    }
    return res
}

In [ ]:

let flatA = xTrain[0..<5].array.scalars
let flatB = weights.array.scalars

In [ ]:

let res = swiftMatmul(a: flatA, b:flatB, arc:arc, brc:brc)

In [ ]:

time(repeating: 100) {
    let _ = swiftMatmul(a: flatA, b:flatB, arc:arc, brc:brc) }

0.13175132 ms

With Tensor

In [ ]:

func TensorMatmul(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    var res:Tensor<Float> = Tensor(repeating: 0.0, shape: [a.shape[0], b.shape[1]])
    for i in 0..<a.shape[0]{
        for j in 0..<b.shape[1]{
            for k in 0..<a.shape[1]{
                res[i][j] += a[i][k] * b[k][j]
            }
        }
    }
    return res
}

In [ ]:

let m1 = Tensor<Float>(randomNormal: [5, 784])
let m2 = Tensor<Float>(randomNormal: [784, 10])

In [ ]:

time() { let res = TensorMatmul(m1, m2)}

34786.166203 ms

Looping over Tensor indices is a bad idea! (For now - in the future this will be even faster than the Swift version above, and will be easier to write too.)

Elementwise ops¶

Operators (+,-,*,/) are usually element-wise.

Examples of element-wise operations:

In [ ]:

var a = Tensor([10.0, 6, -4])
var b = Tensor([2.0, 8, 7])
(a,b)

Out[ ]:

▿ 2 elements
  - .0 : [10.0,  6.0, -4.0]
  - .1 : [2.0, 8.0, 7.0]

In [ ]:

a + b

Out[ ]:

[12.0, 14.0,  3.0]

Comparison operators (>,<,==,!=,...) are true if all the elements of the tensors satisfy the comparison. Elementwise versions have the . prefix: .>, .<, .== ...

In [ ]:

a < b

Out[ ]:

false

In [ ]:

a .< b

Out[ ]:

[false,  true,  true]

In [ ]:

var m = Tensor([1.0, 2, 3, 4, 5, 6, 7, 8, 9]).reshaped(to: [3,3])
m

Out[ ]:

[[1.0, 2.0, 3.0],
 [4.0, 5.0, 6.0],
 [7.0, 8.0, 9.0]]

In [ ]:

2 * m

Out[ ]:

[[ 2.0,  4.0,  6.0],
 [ 8.0, 10.0, 12.0],
 [14.0, 16.0, 18.0]]

In [ ]:

sqrt((m * m).sum().scalar!)

Out[ ]:

16.881943016134134

Elementwise matmul¶

In [ ]:

func elementWiseMatmul(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    let (ar,ac) = (a.shape[0],a.shape[1])
    let (br,bc) = (b.shape[0],b.shape[1])
    var res = Tensor<Float>(zeros: [ac, br])
    for i in 0..<ar {
        for j in 0..<bc {
            res[i][j] = (a[i] * b.slice(lowerBounds: [0,j], upperBounds: [ac,j+1]).squeezingShape(at: 1)).sum()
        }
    }
    return res
}

In [ ]:

let res = elementWiseMatmul(m1, m2)

In [ ]:

time() { let _ = elementWiseMatmul(m1, m2)}

387.499684 ms

Broadcasting¶

Broadcasting with a scalar¶

In [ ]:

var a = Tensor([10.0, 6.0, -4.0])

In [ ]:

print(a .> 0)

[ true,  true, false]

In [ ]:

print((a .> 0).all())

false

In [ ]:

print((a .> 0).any())

true

In Tensorflow the operator > between tensors will return true if all the elements of the first tensor are greater than the ones of the second tensor. .> makes the memberwise comparison.

In [ ]:

print(a+1)

[11.0,  7.0, -3.0]

In [ ]:

2 * m

Out[ ]:

[[ 2.0,  4.0,  6.0],
 [ 8.0, 10.0, 12.0],
 [14.0, 16.0, 18.0]]

Broadcasting a vector with a matrix¶

In [ ]:

let c = Tensor([10.0,20.0,30.0])

By default, broadcasting is done by adding 1 dimensions to the beginning until dimensions of both objects match.

In [ ]:

m + c

Out[ ]:

[[11.0, 22.0, 33.0],
 [14.0, 25.0, 36.0],
 [17.0, 28.0, 39.0]]

In [ ]:

c + m

Out[ ]:

[[11.0, 22.0, 33.0],
 [14.0, 25.0, 36.0],
 [17.0, 28.0, 39.0]]

To broadcast on the other dimensions, one has to use expandingShape to add the dimension.

In [ ]:

m + c.expandingShape(at: 1)

Out[ ]:

[[11.0, 12.0, 13.0],
 [24.0, 25.0, 26.0],
 [37.0, 38.0, 39.0]]

In [ ]:

c.expandingShape(at: 1)

Out[ ]:

[[10.0],
 [20.0],
 [30.0]]

Matmult with broadcasting¶

In [ ]:

func broadcastMatmult(_ a:Tensor<Float>, _ b:Tensor<Float>) -> Tensor<Float>{
    var res = Tensor<Float>(zeros: [a.shape[0], b.shape[1]])
    for i in 0..<a.shape[0]{
        res[i] = (a[i].expandingShape(at: 1) * b).sum(squeezingAxes: 0)
    }
    return res
}

In [ ]:

let res = broadcastMatmult(m1, m2)

In [ ]:

time(repeating: 100) { let _ = broadcastMatmult(m1, m2)}

1.9344620300000006 ms

Broadcasting rules¶

In [ ]:

c.expandingShape(at: 0).shape

Out[ ]:

▿ TensorShape
  ▿ dimensions : 2 elements
    - 0 : 1
    - 1 : 3

In [ ]:

c.expandingShape(at: 1).shape

Out[ ]:

▿ TensorShape
  ▿ dimensions : 2 elements
    - 0 : 3
    - 1 : 1

In [ ]:

c.expandingShape(at: 0) * c.expandingShape(at: 1)

Out[ ]:

[[100.0, 200.0, 300.0],
 [200.0, 400.0, 600.0],
 [300.0, 600.0, 900.0]]

In [ ]:

c.expandingShape(at: 0) .> c.expandingShape(at: 1)

Out[ ]:

[[false,  true,  true],
 [false, false,  true],
 [false, false, false]]

Tensorflow op¶

In [ ]:

time(repeating: 100) { let _ = matmul(m1, m2)}

0.025312370000000004 ms

Export¶

In [ ]:

notebookToScript(fname: (Path.cwd / "01_matmul.ipynb").string)

In [ ]: