#!/usr/bin/env python # coding: utf-8 # ![book](https://raw.githubusercontent.com/ageron/tensorflow-safari-course/master/images/intro_to_tf_course.png) # **Try not to peek at the solutions when you go through the exercises. ;-)** # First let's make sure this notebook works well in both Python 2 and Python 3: # In[ ]: from __future__ import absolute_import, division, print_function, unicode_literals # In[ ]: import tensorflow as tf tf.__version__ # *__From notebook 4 linear regression__* # In[ ]: import numpy as np data = np.loadtxt("data/life_satisfaction.csv", dtype=np.float32, delimiter=",", skiprows=1, usecols=[1, 2]) X_train = data[:, 0:1] / 10000 # feature scaling y_train = data[:, 1:2] learning_rate = 0.01 # In[ ]: get_ipython().run_line_magic('matplotlib', 'inline') import matplotlib.pyplot as plt plt.rcParams['axes.labelsize'] = 14 plt.rcParams['xtick.labelsize'] = 12 plt.rcParams['ytick.labelsize'] = 12 # In[ ]: def plot_life_satisfaction(X_train, y_train): plt.plot(X_train * 10000, y_train, "bo") plt.axis([0, 60000, 0, 10]) plt.xlabel("GDP per capita ($)") plt.ylabel("Life Satisfaction") plt.grid() def plot_life_satisfaction_with_linear_model(X_train, y_train, w, b): plot_life_satisfaction(X_train, y_train) plt.plot([0, 60000], [b, w[0][0] * (60000 / 10000) + b]) # # Using autodiff Instead # In[ ]: graph = tf.Graph() with graph.as_default(): X = tf.constant(X_train, dtype=tf.float32, name="X") y = tf.constant(y_train, dtype=tf.float32, name="y") b = tf.Variable(0.0, name="b") w = tf.Variable(tf.zeros([1, 1]), name="w") y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b mse = tf.reduce_mean(tf.square(y_pred - y), name="mse") gradients_w, gradients_b = tf.gradients(mse, [w, b]) # <= IT'S AUTODIFF MAGIC! tweak_w_op = tf.assign(w, w - learning_rate * gradients_w) tweak_b_op = tf.assign(b, b - learning_rate * gradients_b) training_op = tf.group(tweak_w_op, tweak_b_op) init = tf.global_variables_initializer() # In[ ]: n_iterations = 2000 with tf.Session(graph=graph) as sess: init.run() for iteration in range(n_iterations): if iteration % 100 == 0: print("Iteration {:5}, MSE: {:.4f}".format(iteration, mse.eval())) training_op.run() w_val, b_val = sess.run([w, b]) # In[ ]: plt.figure(figsize=(10, 5)) plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val) plt.show() # ## Using Optimizers # In[ ]: graph = tf.Graph() with graph.as_default(): X = tf.constant(X_train, dtype=tf.float32, name="X") y = tf.constant(y_train, dtype=tf.float32, name="y") b = tf.Variable(0.0, name="b") w = tf.Variable(tf.zeros([1, 1]), name="w") y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b mse = tf.reduce_mean(tf.square(y_pred - y), name="mse") optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) training_op = optimizer.minimize(mse) # <= MOAR AUTODIFF MAGIC! init = tf.global_variables_initializer() # In[ ]: n_iterations = 2000 with tf.Session(graph=graph) as sess: init.run() for iteration in range(n_iterations): if iteration % 100 == 0: print("Iteration {:5}, MSE: {:.4f}".format(iteration, mse.eval())) training_op.run() w_val, b_val = sess.run([w, b]) # In[ ]: plt.figure(figsize=(10, 5)) plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val) plt.show() # ## Faster Optimizers # In[ ]: learning_rate = 0.01 momentum = 0.8 graph = tf.Graph() with graph.as_default(): X = tf.constant(X_train, dtype=tf.float32, name="X") y = tf.constant(y_train, dtype=tf.float32, name="y") b = tf.Variable(0.0, name="b") w = tf.Variable(tf.zeros([1, 1]), name="w") y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b mse = tf.reduce_mean(tf.square(y_pred - y), name="mse") optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) training_op = optimizer.minimize(mse) init = tf.global_variables_initializer() # In[ ]: n_iterations = 500 with tf.Session(graph=graph) as sess: init.run() for iteration in range(n_iterations): if iteration % 100 == 0: print("Iteration {:5}, MSE: {:.4f}".format(iteration, mse.eval())) training_op.run() w_val, b_val = sess.run([w, b]) # In[ ]: plt.figure(figsize=(10, 5)) plot_life_satisfaction_with_linear_model(X_train, y_train, w_val, b_val) plt.show() # How does the optimizer know which variables to tweak? Answer: the `TRAINABLE_VARIABLES` collection. # In[ ]: coll = graph.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) [var.op.name for var in coll] # ## Making Predictions Outside of TensorFlow # In[ ]: cyprus_gdp_per_capita = 22000 cyprus_life_satisfaction = w_val[0][0] * cyprus_gdp_per_capita / 10000 + b_val cyprus_life_satisfaction # ## Using placeholders # In[ ]: graph = tf.Graph() with graph.as_default(): X = tf.placeholder(tf.float32, shape=[None, 1], name="X") # <= None allows for any y = tf.placeholder(tf.float32, shape=[None, 1], name="y") # training batch size b = tf.Variable(0.0, name="b") w = tf.Variable(tf.zeros([1, 1]), name="w") y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b mse = tf.reduce_mean(tf.square(y_pred - y), name="mse") optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) training_op = optimizer.minimize(mse) init = tf.global_variables_initializer() # In[ ]: n_iterations = 500 X_test = np.array([[22000]], dtype=np.float32) / 10000 with tf.Session(graph=graph) as sess: init.run() for iteration in range(n_iterations): feed_dict = {X: X_train, y: y_train} if iteration % 100 == 0: print("Iteration {:5}, MSE: {:.4f}".format( iteration, mse.eval(feed_dict))) # <= FEED TRAINING DATA training_op.run(feed_dict) # <= FEED TRAINING DATA # make the prediction: y_pred_val = y_pred.eval(feed_dict={X: X_test}) # <= FEED TEST DATA # In[ ]: y_pred_val # ## Exercise 5 # ![Exercise](https://c1.staticflickr.com/9/8101/8553474140_c50cf08708_b.jpg) # 5.1) Create a simple graph that computes the function $f(x) = x^2 - 3x + 1$. Define $x$ as a placeholder for a simple scalar value of type float32 value (i.e., `shape=[], dtype=tf.float32`). Create a session and evaluate $f(5)$. You should find 11.0. # In[ ]: # In[ ]: # In[ ]: # 5.2) Add an operation that computes the derivative of $f(x)$ with regards to $x$, noted $f'(x)$. Create a session and evaluate $f'(5)$. You should find 7.0. # # Hint: use `tf.gradients()`. # In[ ]: # In[ ]: # In[ ]: # 5.3) Using a `MomentumOptimizer`, find the value of $x$ that minimizes $f(x)$. You should find $\hat{x}=1.5$. # # Hint: you need to change `x` into a `Variable`. Moreover, the `MomentumOptimizer` has its own variables that need to be initialized, so don't forget to create an `init` operation using a `tf.global_variables_initializer()`, and call it at the start of the session. # In[ ]: # In[ ]: # In[ ]: # Try not to peek at the solution below before you have done the exercise! :) # ![thinking](https://upload.wikimedia.org/wikipedia/commons/0/06/Filos_segundo_logo_%28flipped%29.jpg) # ## Exercise 5 - Solution # 5.1) # In[ ]: graph = tf.Graph() with graph.as_default(): x = tf.placeholder(tf.float32, shape=[], name="x") f = tf.square(x) - 3 * x + 1 # In[ ]: with tf.Session(graph=graph): print(f.eval(feed_dict={x: 5.0})) # 5.2) # In[ ]: with graph.as_default(): [fp] = tf.gradients(f, [x]) # In[ ]: with tf.Session(graph=graph): print(fp.eval(feed_dict={x: 5.0})) # 5.3) # In[ ]: learning_rate = 0.01 momentum = 0.8 graph = tf.Graph() with graph.as_default(): x = tf.Variable(0.0, name="x") f = tf.square(x) - 3 * x + 1 optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) training_op = optimizer.minimize(f) init = tf.global_variables_initializer() # In[ ]: n_iterations = 70 with tf.Session(graph=graph): init.run() for iteration in range(n_iterations): training_op.run() if iteration % 10 == 0: print("x={:.2f}, f(x)={:.2f}".format(x.eval(), f.eval())) # Note that it's possible to replace the output value of any operation, not just placeholders. So, for example, even though `x` is now a `Variable`, you can use a `feed_dict` to use any value you want, for example to compute `f(5.0)`. **Important**: this does _not_ affect the variable! # In[ ]: with tf.Session(graph=graph): init.run() print(x.eval()) # x == 0.0 print(f.eval()) # f(0) == 1.0 print(f.eval(feed_dict={x: 5.0})) # use 5.0 instead of the value of x, to compute f(5) print(x.eval()) # x is still 0.0 print(f.eval()) # f(0) is still 1.0 # ## Saving and Restoring a Model # In[ ]: graph = tf.Graph() with graph.as_default(): X = tf.placeholder(tf.float32, shape=[None, 1], name="X") y = tf.placeholder(tf.float32, shape=[None, 1], name="y") b = tf.Variable(0.0, name="b") w = tf.Variable(tf.zeros([1, 1]), name="w") y_pred = tf.add(tf.matmul(X, w), b, name="y_pred") # X @ w + b mse = tf.reduce_mean(tf.square(y_pred - y), name="mse") optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) training_op = optimizer.minimize(mse) init = tf.global_variables_initializer() saver = tf.train.Saver() # <= At the very end of the construction phase # In[ ]: n_iterations = 500 with tf.Session(graph=graph) as sess: init.run() for iteration in range(n_iterations): if iteration % 100 == 0: print("Iteration {:5}, MSE: {:.4f}".format( iteration, mse.eval(feed_dict={X: X_train, y: y_train}))) training_op.run(feed_dict={X: X_train, y: y_train}) # <= FEED THE DICT saver.save(sess, "./my_life_satisfaction_model") # In[ ]: with tf.Session(graph=graph) as sess: saver.restore(sess, "./my_life_satisfaction_model") # make the prediction: y_pred_val = y_pred.eval(feed_dict={X: X_test}) # In[ ]: y_pred_val # ## Restoring a Graph # In[ ]: model_path = "./my_life_satisfaction_model" graph = tf.Graph() with tf.Session(graph=graph) as sess: # restore the graph saver = tf.train.import_meta_graph(model_path + ".meta") saver.restore(sess, model_path) # get references to the tensors we need X = graph.get_tensor_by_name("X:0") y_pred = graph.get_tensor_by_name("y_pred:0") # make the prediction: y_pred_val = y_pred.eval(feed_dict={X: X_test}) # In[ ]: y_pred_val # In[ ]: