jupyter_notebook.code

# gen_data.py
import numpy as np

def gen_data(batch_size,num_inputs,w=[],b=[]):
  y_output = np.zeros((batch_size,num_inputs))
  x_input = np.zeros((batch_size,num_inputs))
  if len(w) == 0:
    w = np.random.randint(10, size=(num_inputs,num_inputs))
    b = np.random.randint(5, size=(1,num_inputs))
#  print(w)
#  print(b)
  for i in range(batch_size):
    x = np.random.randint(10, size=(1,num_inputs))
#    print('----',x)
    y_output[i] = np.add(np.matmul(x,w),b)
    x_input[i] = x
#  print(y_output)
#  print(x_input)
  return w,b,x_input,y_output


# model.py
import tensorflow as tf
import numpy as np

class FF():
  
  def init_weights(self, shape):
    weights = tf.random_normal(shape,stddev=1)
    return tf.Variable(weights)
  
  def init_b(self, num_cols):
    return tf.Variable(tf.zeros([num_cols]))
  
  def __init__(self, args, graph):
    self.graph = graph
    self.num_inputs = int(args['num_inputs'])
    self.num_neurons = int(args['num_neurons'])
    self.num_layers = int(args['num_layers'])
    #self.num_outputs = int(args.num_outputs)
    self.num_outputs = self.num_inputs
    self.learning_rate = float(args['learning_rate'])
    
    # setup inputs and outputs
    self.x = tf.placeholder(name='x', dtype=tf.float32, shape=[None,self.num_inputs]) # shape = [batch_size, num_inputs]
    self.y = tf.placeholder(name='y', dtype=tf.float32, shape=[None,self.num_outputs]) # shape = [batch_size, num_outputs]

    # setup weights & biases
    self.weights = []
    self.biases = []
    for i in range(self.num_layers):
      if i == 0:
        self.weights.append(self.init_weights((self.num_inputs,self.num_neurons)))
        self.biases.append(self.init_b(self.num_outputs))
      else: 
        self.weights.append(self.init_weights((self.num_neurons,self.num_neurons)))
        self.biases.append(self.init_b(self.num_neurons))

    # define the graph
    layer_outputs = []
    for i in range(self.num_layers):
      if i == 0:
        layer_outputs.append(tf.matmul(self.x,self.weights[i]) + self.biases[i])
      else:
        layer_outputs.append(tf.matmul(layer_outputs[i-1],self.weights[i]) + self.biases[i])
    self.y_ = layer_outputs[self.num_layers - 1]
    self.y_ = tf.identity(self.y_,name='y_')
    self.loss = tf.losses.mean_squared_error(self.y,self.y_)
    self.loss = tf.identity(self.loss,name='loss')
    self.optimizer = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(self.loss)

  def run(self, test=False):
    if test:
      return self.y_, self.loss
    return self.y_, self.loss, self.optimizer


# restore_graph
def restore_graph(sess,args):

  trained_model = tf.train.import_meta_graph(args['restore_path'] + '.meta')
  #trained_model.restore(sess, tf.train.latest_checkpoint(cwd + '/checkpoints/.'))
  trained_model.restore(sess, args['restore_path'])

  w = []
  b = []
  model_vars_file = args['vars_file']  # need to get back w's and b's for gen_data to reproduce same slopes and y-intercepts for lines
  with open(model_vars_file, 'rb') as f:
    w,b = pickle.load(f)
  print('\n\nModel and variables restored.\n\n')

  if (len(w) != int(args['num_inputs'])):
    print('Error: num_inputs length %d does not equal stored variable length %d\n' % (int(args['num_inputs']),len(w)))
    quit()

  return trained_model, w, b

# train
def train(args):

  graph = tf.Graph()
  var_path = cwd + '/' + args['checkpoint_dir'] + '/variables/'

  with graph.as_default():

    # Graph object and scope created
    # ...now define all parts of the graph here
    feed_fwd_model = FF(args, graph)
    saver = tf.train.Saver()
    init = tf.global_variables_initializer()

    # Now that the graph is defined, create a session to begin running
    with tf.Session() as sess:

      sess.run(init)
      # Prepare to Save model
      i = 0
      model = 'model%s' % i
      ckpt_file_index = Path(cwd + '/' + args['checkpoint_dir'] + '/' + model + '.ckpt.index')
      ckpt_file = Path(cwd + '/' + args['checkpoint_dir'] + '/' + model + '.ckpt')
      while ckpt_file_index.is_file():
        i += 1
        model = 'model%s' % i
        ckpt_file_index = Path(cwd + '/' + args['checkpoint_dir'] + '/' + model + '.ckpt.index')
      ckpt_file = Path(cwd + '/' + args['checkpoint_dir'] + '/' + model + '.ckpt')

      num_epochs = int(args['num_epochs'])
      y_acc = np.zeros((int(args['batch_size']),int(args['num_outputs'])))
      loss = None
      y_ = None
      
      w = []
      b = []
      if (args['restore_path'] != None):
        trained_model_saver, w, b = restore_graph(sess,args)
        print('...continuing training')

	# guards against accidental updates to the graph which can cause graph
      # increase and performance decay over time (with more iterations)
      sess.graph.finalize()
    
      for e in range(num_epochs):
        w, b, train_input, train_output = gen_data(int(args['batch_size']),int(args['num_inputs']), w, b)
        y_, loss, _ = sess.run(feed_fwd_model.run(), feed_dict={feed_fwd_model.x: train_input, feed_fwd_model.y: train_output})
        y_acc = y_
        threshold = 1000
        w_b_saved = False
        if ((e % 50) == 0):
          print('epoch: %d - loss: %2f' % (e,loss))
          if (e > 0 and (e % threshold == 0)):
            print('Writing checkpoint %d' % e)
            print(train_output, w, b)
            print('\n')
            print(y_acc, sess.run(feed_fwd_model.weights)[0], sess.run(feed_fwd_model.biases)[0])
    #        save_path = saver.save(sess, str(ckpt_file), global_step=e)
    #        if not w_b_saved:
    #          try:
    #            with open(var_path + model + '.pkl', 'wb') as f:
    #              pickle.dump([w,b],f)
    #              w_b_saved = True
    #          except FileNotFoundError as fnf:
    #            os.makedirs(var_path)
    #            with open(var_path + model + '.pkl', 'wb') as f:
    #              pickle.dump([w,b],f)
    #              w_b_saved = True
#      save_path = saver.save(sess, str(ckpt_file))
#      if not w_b_saved:
#        try:
#          with open(var_path + model + '.pkl', 'wb') as f:
#            pickle.dump([w,b],f)
#        except FileNotFoundError as fnf:
#          os.makedirs(var_path)
#          with open(var_path + model + '.pkl', 'wb') as f:
#            pickle.dump([w,b],f)
#      print('Model saved to %s' % str(save_path))
      sess.close()

      # test
def test(args):

  inference_graph = tf.Graph()
  with tf.Session(graph=inference_graph) as sess:

    if not args['restore_path'] or not args['vars_file']:
      print('\n\n\tSpecify a restore_path: --restore_path=<path_to_ckpt> and --vars_file=<vars_file_pathname>\n\n')
      quit()

    trained_model_saver, w, b = restore_graph(sess,args)

    _y_ = inference_graph.get_tensor_by_name('y_:0')
    _loss = inference_graph.get_tensor_by_name('loss:0')
    _x = inference_graph.get_tensor_by_name('x:0')
    _y = inference_graph.get_tensor_by_name('y:0')

    while(1):
      w, b, train_input, train_output = gen_data(int(args['batch_size']),int(args['num_inputs']), w, b)
      y_ = sess.run(_y_, feed_dict={_x: train_input, _y: train_output})
      loss = sess.run(_loss, feed_dict={_x: train_input, _y: train_output})
      y_acc = y_
      print('Mean Squared Error Loss: %2f\n' % loss)
      print(train_output)
      print('\n')
      print(y_acc)
      print('\n')
      input('Press Enter to continue...')


#import argparse
#import matplotlib.pyplot as plt
from pathlib import Path
import os
import pickle

NUM_INPUTS = 2
NUM_OUTPUTS = NUM_INPUTS

cwd = os.getcwd()

#def main():
#  parser = argparse.ArgumentParser()
#  parser.add_argument('--mode', default='train')
#  parser.add_argument('--num_inputs', default=NUM_INPUTS)
#  parser.add_argument('--batch_size', default=3)
#  parser.add_argument('--num_neurons', default=2)
#  parser.add_argument('--num_layers', default=1)
#  parser.add_argument('--num_outputs', default=NUM_OUTPUTS)
#  parser.add_argument('--learning_rate', default=0.001)
#  parser.add_argument('--num_epochs', default=10)
#  parser.add_argument('--checkpoint_dir', default='./checkpoints')
#  parser.add_argument('--restore_path', default=None)
#  parser.add_argument('--vars_file', default=None)

#  args = parser.parse_args()
mode = 'train'
num_inputs = 2
batch_size = 3
num_neurons = 2
num_layers = 1
num_outputs = num_inputs
learning_rate = 0.01
num_epochs = 1200
checkpoint_dir = './checkpoints'
restore_path = None
vars_file = None

args = {'mode': mode,\
        'num_inputs': num_inputs,\
        'batch_size': batch_size,\
        'num_neurons': num_neurons,\
        'num_layers': num_layers,\
        'num_outputs': num_outputs,\
        'learning_rate': learning_rate,\
        'num_epochs': num_epochs,\
        'checkpoint_dir': checkpoint_dir,\
        'restore_path': restore_path,\
        'vars_file': vars_file}

if args['mode'] == 'train':
  print('Training...')
  train(args)
elif args['mode'] == 'test':
  test(args)