lab-09-2-XOR-NN-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 09 XOR - Neural Network - Eager Excuetion\n",
    "* XOR 문제를  Neural Network을 활용해 풀어보도록 하겠습니다.\n",
    "\n",
    "### 기본 Library 선언 및 Tensorflow 버전 확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.eager as tfe\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "tf.set_random_seed(777)  # for reproducibility\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 강의에 설명할 Data입니다\n",
    "* x_data가 2차원 배열이기에 2차원 공간에 표현하여 x1과 x2를 기준으로 y_data 0과 1로 구분하는 예제입니다\n",
    "* 붉은색과 푸른색으로 0과 1을 표시해 보도록 하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEFNJREFUeJzt3W2MXGd5h/HrH5sQCgGqeJFQbOM0dVTcgEq6hFSoJTRp5eSDrQqEEolXpViChlYFoaalBWr3C0WlEpJbcAUJhEIwVCIrMHUlCIoEOPVGKRFOGro1Ads4igkhQkph83L3w4wfJht7d7zZM+NdXz/J8syZRzP38dpz+czZnUlVIUkSwFnjHkCSdPowCpKkxihIkhqjIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpWT3uAU7VmjVrasOGDeMeQ5KWlTvvvPPHVTWx0LplF4UNGzYwPT097jEkaVlJ8oNh1vnykSSpMQqSpMYoSJIaoyBJaoyCJKkxCpKkxihIkhqjIElqOotCkk8meTDJd09ye5J8NMlMkruTXNLVLHMdPQoXXggPPDCqR5SkRRjDk1WXRwo3AZvnuf0qYGP/1zbgnzuc5Sl27ID77+/9LkmnrTE8WXUWhaq6HfjJPEu2Ap+unn3AC5O8uKt5jjt6FG68EZ58sve7RwuSTktjerIa5zmF84FDA9cP97c9TZJtSaaTTB87duwZPeiOHb0/Y4AnnvBoQdJpakxPVsviRHNV7aqqyaqanJhY8E3+Tup4eGdne9dnZz1akHQaGuOT1TijcARYN3B9bX9bZwbDe5xHC5JOO2N8shpnFKaAN/e/C+ky4JGqOtrpA079MrzHzc7Crbd2+aiSdIrG+GTV2ecpJPkccDmwJslh4APAswCq6mPAHuBqYAZ4FHhbV7Mcd/hw148gSUtgjE9WnUWhqq5d4PYC/qSrx5cknbplcaJZkjQaRkGS1BgFSVJjFCRJjVGQJDVGQZLUGAVJUmMUJEmNUZAkNUZBktQYBUlSYxQkSY1RkCQ1RkGS1BgFSVJjFCRJjVGQJDVGQZLUGAVJUmMUJEmNUZAkNUZBktQYBUlSYxQkSY1RkCQ1RkGS1BgFSVJjFCRJjVGQJDWdRiHJ5iT3JZlJcsMJbl+f5LYkdyW5O8nVXc4jSZpfZ1FIsgrYCVwFbAKuTbJpzrK/BnZX1SuAa4B/6moeSdLCujxSuBSYqaqDVTUL3AJsnbOmgOf3L78A+FGH80iSFrC6w/s+Hzg0cP0w8Ko5az4I/EeSdwHPBa7scB5J0gLGfaL5WuCmqloLXA3cnORpMyXZlmQ6yfSxY8dGPqQknSm6jMIRYN3A9bX9bYOuA3YDVNW3gXOANXPvqKp2VdVkVU1OTEx0NK4kqcso7Ac2Jrkgydn0TiRPzVnzQ+AKgCQvpRcFDwUkaUw6i0JVPQ5cD+wF7qX3XUYHkmxPsqW/7D3A25N8B/gc8Naqqq5mkiTNr8sTzVTVHmDPnG3vH7h8D/DqLmeQJA1v3CeaJUmnEaMgSWqMgiSpMQqSpMYoSJIaoyBJaoyCJKkxCpKkxihIkhqjIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpMQqSpMYoSJIaoyBJaoyCJKkxCpKkxihIkhqjIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpMQqSpMYoSJKaTqOQZHOS+5LMJLnhJGvekOSeJAeSfLbLeSRJ81vd1R0nWQXsBP4AOAzsTzJVVfcMrNkI/CXw6qp6OMmLuppHkrSwLo8ULgVmqupgVc0CtwBb56x5O7Czqh4GqKoHO5xHkrSALqNwPnBo4Prh/rZBFwEXJflmkn1JNnc4jyRpAZ29fHQKj78RuBxYC9ye5GVV9dPBRUm2AdsA1q9fP+oZJemM0eWRwhFg3cD1tf1tgw4DU1X1WFV9H/gevUg8RVXtqqrJqpqcmJjobGBJOtN1GYX9wMYkFyQ5G7gGmJqz5kv0jhJIsobey0kHO5xJkjSPzqJQVY8D1wN7gXuB3VV1IMn2JFv6y/YCDyW5B7gNeG9VPdTVTJKk+aWqxj3DKZmcnKzp6elxjyFJy0qSO6tqcqF1/kSzJKkxCpKkxihIkhqjIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpMQqSpMYoSJKaeaOQ5PlJLjzB9pd3N5IkaVxOGoUkbwD+G/i3JAeSvHLg5pu6HkySNHrzHSn8FfDbVfVbwNuAm5P8Uf+2dD6ZJGnk5vs4zlVVdRSgqv4zyWuBLydZByyv99uWJA1lviOFnw2eT+gH4nJgK/CbHc8lSRqD+aLwDuCsJJuOb6iqnwGbgT/uejBJ0uidNApV9Z2q+h9gd5K/SM9zgI8A7xzZhJKkkRnm5xReBawDvgXsB34EvLrLoSRJ4zFMFB4D/g94DnAO8P2qerLTqSRJYzFMFPbTi8Irgd8Frk3yhU6nkiSNxXzfknrcdVU13b98FNia5E0dziRJGpMFjxQGgjC47eZuxpEkjZNviCdJaoyCJKkxCpKkxihIkhqjIElqjIIkqTEKkqSm0ygk2ZzkviQzSW6YZ93rklSSyS7nkSTNr7MoJFkF7ASuAjbRe3uMTSdYdy7wZ8AdXc0iSRpOl0cKlwIzVXWwqmaBW+h9QM9cO4APAT/vcBZJ0hC6jML5wKGB64f725oklwDrquor891Rkm1JppNMHzt2bOknlSQBYzzRnOQseh/Y856F1lbVrqqarKrJiYmJ7oeTpDNUl1E4Qu/DeY5b29923LnAxcA3ktwPXAZMebJZksanyyjsBzYmuSDJ2cA1wNTxG6vqkapaU1UbqmoDsA/YcqJ3ZZUkjUZnUaiqx4Hrgb3AvcDuqjqQZHuSLV09riRp8Yb5kJ1Fq6o9wJ45295/krWXdzmLJGlh/kSzJKkxCpKkxihIkhqjIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpMQqSpMYoSJIaoyBJaoyCJKkxCpKkxihIkhqjIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpMQqSpMYoSJIaoyBJaoyCJKkxCpKkxihIkhqjIElqjIIkqek0Ckk2J7kvyUySG05w+7uT3JPk7iRfS/KSLueRJM2vsygkWQXsBK4CNgHXJtk0Z9ldwGRVvRz4IvD3Xc0jSVpYl0cKlwIzVXWwqmaBW4Ctgwuq6raqerR/dR+wtsN5JEkL6DIK5wOHBq4f7m87meuAr3Y4jyRpAavHPQBAkjcCk8BrTnL7NmAbwPr160c4mSSdWbo8UjgCrBu4vra/7SmSXAm8D9hSVb840R1V1a6qmqyqyYmJiU6GlSR1G4X9wMYkFyQ5G7gGmBpckOQVwMfpBeHBDmeRJA2hsyhU1ePA9cBe4F5gd1UdSLI9yZb+sg8DzwO+kOS/kkyd5O4kSSPQ6TmFqtoD7Jmz7f0Dl6/s8vElSafGn2iWJDVGQZLUGAVJUmMUJEmNUZAkNUZBktQYBUlSYxQkSY1RkCQ1RkGS1BgFSVJjFCRJjVGQJDVGQZLUGAVJUmMUJEmNUZAkNUZBktQYBUlSYxQkSY1RkCQ1RkGS1BgFSVJjFCRJjVGQJDVGQZLUGAVJUmMUJEmNUZAkNZ1GIcnmJPclmUlywwluf3aSz/dvvyPJhi7nkSTNr7MoJFkF7ASuAjYB1ybZNGfZdcDDVfXrwD8CH+pqnqc4ehQuvBAeeGAkDydJizGOp6oujxQuBWaq6mBVzQK3AFvnrNkKfKp/+YvAFUnS4Uw9O3bA/ff3fpek09Q4nqq6jML5wKGB64f72064pqoeBx4Bzutwpl56b7wRnnyy97tHC5JOQ+N6qloWJ5qTbEsynWT62LFjz+zOduzo/SkDPPGERwuSTkvjeqrqMgpHgHUD19f2t51wTZLVwAuAh+beUVXtqqrJqpqcmJhY/ETH0zs727s+O+vRgqTTzjifqrqMwn5gY5ILkpwNXANMzVkzBbylf/n1wNerqjqbaDC9x3m0IOk0M86nqs6i0D9HcD2wF7gX2F1VB5JsT7Klv+wTwHlJZoB3A0/7ttUlNTX1y/QeNzsLt97a6cNK0qkY51NVuvyPeRcmJydrenp63GNI0rKS5M6qmlxo3bI40SxJGg2jIElqjIIkqTEKkqTGKEiSGqMgSWqMgiSpMQqSpGbZ/fBakmPAD5bgrtYAP16C+1ku3N+V60zaV3B/F+slVbXgm8ctuygslSTTw/x030rh/q5cZ9K+gvvbNV8+kiQ1RkGS1JzJUdg17gFGzP1duc6kfQX3t1Nn7DkFSdLTnclHCpKkOVZ8FJJsTnJfkpkkT/sQnyTPTvL5/u13JNkw+imXzhD7++4k9yS5O8nXkrxkHHMuhYX2dWDd65JUkmX9HSvD7G+SN/S/vgeSfHbUMy6lIf4ur09yW5K7+n+frx7HnEshySeTPJjkuye5PUk+2v+zuDvJJZ0NU1Ur9hewCvhf4NeAs4HvAJvmrHkn8LH+5WuAz4977o7397XAr/Qvv2O57u8w+9pfdy5wO7APmBz33B1/bTcCdwG/2r/+onHP3fH+7gLe0b+8Cbh/3HM/g/39PeAS4Lsnuf1q4KtAgMuAO7qaZaUfKVwKzFTVwaqaBW4Bts5ZsxX4VP/yF4ErkmSEMy6lBfe3qm6rqkf7V/cBa0c841IZ5msLsAP4EPDzUQ7XgWH29+3Azqp6GKCqHhzxjEtpmP0t4Pn9yy8AfjTC+ZZUVd0O/GSeJVuBT1fPPuCFSV7cxSwrPQrnA4cGrh/ubzvhmup9rvQjwHkjmW7pDbO/g66j97+P5WjBfe0fYq+rqq+McrCODPO1vQi4KMk3k+xLsnlk0y29Yfb3g8AbkxwG9gDvGs1oY3Gq/7YXbXUXd6rTX5I3ApPAa8Y9SxeSnAV8BHjrmEcZpdX0XkK6nN4R4O1JXlZVPx3rVN25Fripqv4hye8ANye5uKqeHPdgy9lKP1I4AqwbuL62v+2Ea5KspncY+tBIplt6w+wvSa4E3gdsqapfjGi2pbbQvp4LXAx8I8n99F6HnVrGJ5uH+doeBqaq6rGq+j7wPXqRWI6G2d/rgN0AVfVt4Bx67xO0Eg31b3sprPQo7Ac2Jrkgydn0TiRPzVkzBbylf/n1wNerf2ZnGVpwf5O8Avg4vSAs59ec593XqnqkqtZU1Yaq2kDv/MmWqpoez7jP2DB/l79E7yiBJGvovZx0cJRDLqFh9veHwBUASV5KLwrHRjrl6EwBb+5/F9JlwCNVdbSLB1rRLx9V1eNJrgf20vtuhk9W1YEk24HpqpoCPkHvsHOG3omea8Y38TMz5P5+GHge8IX++fQfVtWWsQ29SEPu64ox5P7uBf4wyT3AE8B7q2pZHvUOub/vAf4lyZ/TO+n81uX6H7okn6MX9DX9cyQfAJ4FUFUfo3fO5GpgBngUeFtnsyzTP0NJUgdW+stHkqRTYBQkSY1RkCQ1RkGS1BgFSVJjFKQllOTfk/w0yZfHPYu0GEZBWlofBt407iGkxTIK0iIkeWX/fe3PSfLc/ucXXFxVXwN+Nu75pMVa0T/RLHWlqvYnmQL+DngO8JmqOuEHpEjLiVGQFm87vffo+Tnwp2OeRVoSvnwkLd559N5H6lx6b8YmLXtGQVq8jwN/A/wrvU93k5Y9Xz6SFiHJm4HHquqzSVYB30ry+8DfAr8BPK//bpfXVdXecc4qnQrfJVWS1PjykSSpMQqSpMYoSJIaoyBJaoyCJKkxCpKkxihIkhqjIElq/h+CL7kPwpocagAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_data = [[0, 0],\n",
    "          [0, 1],\n",
    "          [1, 0],\n",
    "          [1, 1]]\n",
    "y_data = [[0],\n",
    "          [1],\n",
    "          [1],\n",
    "          [0]]\n",
    "\n",
    "plt.scatter(x_data[0][0],x_data[0][1], c='red' , marker='^')\n",
    "plt.scatter(x_data[3][0],x_data[3][1], c='red' , marker='^')\n",
    "plt.scatter(x_data[1][0],x_data[1][1], c='blue' , marker='^')\n",
    "plt.scatter(x_data[2][0],x_data[2][1], c='blue' , marker='^')\n",
    "\n",
    "plt.xlabel(\"x1\")\n",
    "plt.ylabel(\"x2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Tensorflow Session\n",
    "### 위 Data를 기준으로 XOR처리를 위한 모델을 만들도록 하겠습니다\n",
    "* Tensorflow data API를 통해 학습시킬 값들을 담는다 (Batch Size는 한번에 학습시킬 Size로 정한다)\n",
    "* features,labels는 실재 학습에 쓰일 Data (연산을 위해 Type를 맞춰준다)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data)).batch(len(x_data))\n",
    "\n",
    "def preprocess_data(features, labels):\n",
    "    features = tf.cast(features, tf.float32)\n",
    "    labels = tf.cast(labels, tf.float32)\n",
    "    return features, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Neural Network를 통해 XOR해결 \n",
    "* 위의 Data를 3Layer의 Neural Network를 통해 학습시킨 후 모델을 생성합니다.(강의에서 설명한 구조)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = tf.Variable(tf.random_normal([2, 1]), name='weight1')\n",
    "b1 = tf.Variable(tf.random_normal([1]), name='bias1')\n",
    "\n",
    "W2 = tf.Variable(tf.random_normal([2, 1]), name='weight2')\n",
    "b2 = tf.Variable(tf.random_normal([1]), name='bias2')\n",
    "\n",
    "W3 = tf.Variable(tf.random_normal([2, 1]), name='weight3')\n",
    "b3 = tf.Variable(tf.random_normal([1]), name='bias3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def neural_net(features):\n",
    "    layer1 = tf.sigmoid(tf.matmul(features, W1) + b1)\n",
    "    layer2 = tf.sigmoid(tf.matmul(features, W2) + b2)\n",
    "    layer3 = tf.concat([layer1, layer2],-1)\n",
    "    layer3 = tf.reshape(layer3, shape = [-1,2])\n",
    "    hypothesis = tf.sigmoid(tf.matmul(layer3, W3) + b3)\n",
    "    return hypothesis\n",
    "\n",
    "def loss_fn(hypothesis, labels):\n",
    "    cost = -tf.reduce_mean(labels * tf.log(hypothesis) + (1 - labels) * tf.log(1 - hypothesis))\n",
    "    return cost\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n",
    "\n",
    "def accuracy_fn(hypothesis, labels):\n",
    "    predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)\n",
    "    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(hypothesis, features, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss_value = loss_fn(neural_net(features),labels)\n",
    "    return tape.gradient(loss_value, [W1, W2, W3, b1, b2, b3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter: 0, Loss: 0.8487\n",
      "Iter: 5000, Loss: 0.6847\n",
      "Iter: 10000, Loss: 0.6610\n",
      "Iter: 15000, Loss: 0.6154\n",
      "Iter: 20000, Loss: 0.5722\n",
      "Iter: 25000, Loss: 0.5433\n",
      "Iter: 30000, Loss: 0.5211\n",
      "Iter: 35000, Loss: 0.4911\n",
      "Iter: 40000, Loss: 0.4416\n",
      "Iter: 45000, Loss: 0.3313\n",
      "Testset Accuracy: 1.0000\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 50000\n",
    "\n",
    "for step in range(EPOCHS):\n",
    "    for features, labels  in tfe.Iterator(dataset):\n",
    "        features, labels = preprocess_data(features, labels)\n",
    "        grads = grad(neural_net(features), features, labels)\n",
    "        optimizer.apply_gradients(grads_and_vars=zip(grads,[W1, W2, W3, b1, b2, b3]))\n",
    "        if step % 5000 == 0:\n",
    "            print(\"Iter: {}, Loss: {:.4f}\".format(step, loss_fn(neural_net(features),labels)))\n",
    "x_data, y_data = preprocess_data(x_data, y_data)\n",
    "test_acc = accuracy_fn(neural_net(x_data),y_data)\n",
    "print(\"Testset Accuracy: {:.4f}\".format(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}