Merge branch 'cherry_pick_0.4' into release-0.4

# Conflicts:
#	flink/pom.xml
#	hive/pom.xml
#	pom.xml
#	spark/pom.xml

tensorflow/notebooks/PyTorch/mnist.ipynb

@@ -0,0 +1,207 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Pytorch example"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Starting Spark application\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>7717</td><td>application_1513605045578_5456</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://hadoop30:8088/proxy/application_1513605045578_5456/\">Link</a></td><td><a target=\"_blank\" href=\"http://hadoop17:8042/node/containerlogs/container_e28_1513605045578_5456_01_000001/copystufftest__robin_er\">Link</a></td><td>✔</td></tr></table>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "SparkSession available as 'spark'.\n"
+     ]
+    }
+   ],
+   "source": [
+    "def wrapper():\n",
+    "    import argparse\n",
+    "    import torch\n",
+    "    import torch.nn as nn\n",
+    "    import torch.nn.functional as F\n",
+    "    import torch.optim as optim\n",
+    "    from torchvision import datasets, transforms\n",
+    "    from torch.autograd import Variable\n",
+    "\n",
+    "    # Training settings\n",
+    "    parser = argparse.ArgumentParser(description='PyTorch MNIST Example')\n",
+    "    parser.add_argument('--batch-size', type=int, default=64, metavar='N',\n",
+    "                        help='input batch size for training (default: 64)')\n",
+    "    parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',\n",
+    "                        help='input batch size for testing (default: 1000)')\n",
+    "    parser.add_argument('--epochs', type=int, default=10, metavar='N',\n",
+    "                        help='number of epochs to train (default: 10)')\n",
+    "    parser.add_argument('--lr', type=float, default=0.01, metavar='LR',\n",
+    "                        help='learning rate (default: 0.01)')\n",
+    "    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',\n",
+    "                        help='SGD momentum (default: 0.5)')\n",
+    "    parser.add_argument('--no-cuda', action='store_true', default=False,\n",
+    "                        help='disables CUDA training')\n",
+    "    parser.add_argument('--seed', type=int, default=1, metavar='S',\n",
+    "                        help='random seed (default: 1)')\n",
+    "    parser.add_argument('--log-interval', type=int, default=10, metavar='N',\n",
+    "                        help='how many batches to wait before logging training status')\n",
+    "    args = parser.parse_args()\n",
+    "    args.cuda = not args.no_cuda and torch.cuda.is_available()\n",
+    "\n",
+    "    torch.manual_seed(args.seed)\n",
+    "    if args.cuda:\n",
+    "        torch.cuda.manual_seed(args.seed)\n",
+    "\n",
+    "\n",
+    "    kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}\n",
+    "    train_loader = torch.utils.data.DataLoader(\n",
+    "        datasets.MNIST('../data', train=True, download=True,\n",
+    "                       transform=transforms.Compose([\n",
+    "                           transforms.ToTensor(),\n",
+    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
+    "                       ])),\n",
+    "        batch_size=args.batch_size, shuffle=True, **kwargs)\n",
+    "    test_loader = torch.utils.data.DataLoader(\n",
+    "        datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
+    "                           transforms.ToTensor(),\n",
+    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
+    "                       ])),\n",
+    "        batch_size=args.test_batch_size, shuffle=True, **kwargs)\n",
+    "\n",
+    "\n",
+    "    class Net(nn.Module):\n",
+    "        def __init__(self):\n",
+    "            super(Net, self).__init__()\n",
+    "            self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
+    "            self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
+    "            self.conv2_drop = nn.Dropout2d()\n",
+    "            self.fc1 = nn.Linear(320, 50)\n",
+    "            self.fc2 = nn.Linear(50, 10)\n",
+    "\n",
+    "        def forward(self, x):\n",
+    "            x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
+    "            x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
+    "            x = x.view(-1, 320)\n",
+    "            x = F.relu(self.fc1(x))\n",
+    "            x = F.dropout(x, training=self.training)\n",
+    "            x = self.fc2(x)\n",
+    "            return F.log_softmax(x)\n",
+    "\n",
+    "    model = Net()\n",
+    "    if args.cuda:\n",
+    "        model.cuda()\n",
+    "\n",
+    "    optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)\n",
+    "\n",
+    "    def train(epoch):\n",
+    "        model.train()\n",
+    "        for batch_idx, (data, target) in enumerate(train_loader):\n",
+    "            if args.cuda:\n",
+    "                data, target = data.cuda(), target.cuda()\n",
+    "            data, target = Variable(data), Variable(target)\n",
+    "            optimizer.zero_grad()\n",
+    "            output = model(data)\n",
+    "            loss = F.nll_loss(output, target)\n",
+    "            loss.backward()\n",
+    "            optimizer.step()\n",
+    "            if batch_idx % args.log_interval == 0:\n",
+    "                print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
+    "                    epoch, batch_idx * len(data), len(train_loader.dataset),\n",
+    "                    100. * batch_idx / len(train_loader), loss.data[0]))\n",
+    "\n",
+    "    def test():\n",
+    "        model.eval()\n",
+    "        test_loss = 0\n",
+    "        correct = 0\n",
+    "        for data, target in test_loader:\n",
+    "            if args.cuda:\n",
+    "                data, target = data.cuda(), target.cuda()\n",
+    "            data, target = Variable(data, volatile=True), Variable(target)\n",
+    "            output = model(data)\n",
+    "            test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss\n",
+    "            pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
+    "            correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
+    "\n",
+    "        test_loss /= len(test_loader.dataset)\n",
+    "        print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
+    "            test_loss, correct, len(test_loader.dataset),\n",
+    "            100. * correct / len(test_loader.dataset)))\n",
+    "\n",
+    "\n",
+    "    for epoch in range(1, args.epochs + 1):\n",
+    "        train(epoch)\n",
+    "        test()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": [
+    "from hops import experiment\n",
+    "experiment.launch(spark, wrapper)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "deletable": true,
+    "editable": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "PySpark",
+   "language": "",
+   "name": "pysparkkernel"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "python",
+    "version": 2
+   },
+   "mimetype": "text/x-python",
+   "name": "pyspark",
+   "pygments_lexer": "python2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tensorflow/notebooks/Serving/serving_client.ipynb

@@ -0,0 +1,158 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "from __future__ import print_function\n",
+    "\n",
+    "import sys\n",
+    "import threading\n",
+    "import os\n",
+    "\n",
+    "from grpc.beta import implementations\n",
+    "import numpy\n",
+    "import tensorflow as tf\n",
+    "\n",
+    "from tensorflow_serving.apis import predict_pb2\n",
+    "from tensorflow_serving.apis import prediction_service_pb2\n",
+    "from tensorflow.examples.tutorials.mnist import input_data as mnist_input_data\n",
+    "\n",
+    "from hops import serving\n",
+    "from hops import hdfs\n",
+    "\n",
+    "concurrency=1\n",
+    "num_tests=100\n",
+    "work_dir=os.getcwd()\n",
+    "server=\"host:ip\"\n",
+    "\n",
+    "\n",
+    "class _ResultCounter(object):\n",
+    "  \"\"\"Counter for the prediction results.\"\"\"\n",
+    "\n",
+    "  def __init__(self, num_tests, concurrency):\n",
+    "    self._num_tests = num_tests\n",
+    "    self._concurrency = concurrency\n",
+    "    self._error = 0\n",
+    "    self._done = 0\n",
+    "    self._active = 0\n",
+    "    self._condition = threading.Condition()\n",
+    "\n",
+    "  def inc_error(self):\n",
+    "    with self._condition:\n",
+    "      self._error += 1\n",
+    "\n",
+    "  def inc_done(self):\n",
+    "    with self._condition:\n",
+    "      self._done += 1\n",
+    "      self._condition.notify()\n",
+    "\n",
+    "  def dec_active(self):\n",
+    "    with self._condition:\n",
+    "      self._active -= 1\n",
+    "      self._condition.notify()\n",
+    "\n",
+    "  def get_error_rate(self):\n",
+    "    with self._condition:\n",
+    "      while self._done != self._num_tests:\n",
+    "        self._condition.wait()\n",
+    "      return self._error / float(self._num_tests)\n",
+    "\n",
+    "  def throttle(self):\n",
+    "    with self._condition:\n",
+    "      while self._active == self._concurrency:\n",
+    "        self._condition.wait()\n",
+    "      self._active += 1\n",
+    "\n",
+    "\n",
+    "def _create_rpc_callback(label, result_counter):\n",
+    "  \"\"\"Creates RPC callback function.\n",
+    "  Args:\n",
+    "    label: The correct label for the predicted example.\n",
+    "    result_counter: Counter for the prediction result.\n",
+    "  Returns:\n",
+    "    The callback function.\n",
+    "  \"\"\"\n",
+    "  def _callback(result_future):\n",
+    "    \"\"\"Callback function.\n",
+    "    Calculates the statistics for the prediction result.\n",
+    "    Args:\n",
+    "      result_future: Result future of the RPC.\n",
+    "    \"\"\"\n",
+    "    exception = result_future.exception()\n",
+    "    if exception:\n",
+    "      result_counter.inc_error()\n",
+    "      print(exception)\n",
+    "    else:\n",
+    "      sys.stdout.write('.')\n",
+    "      sys.stdout.flush()\n",
+    "      response = numpy.array(\n",
+    "          result_future.result().outputs['scores'].float_val)\n",
+    "      prediction = numpy.argmax(response)\n",
+    "      if label != prediction:\n",
+    "        result_counter.inc_error()\n",
+    "    result_counter.inc_done()\n",
+    "    result_counter.dec_active()\n",
+    "  return _callback\n",
+    "\n",
+    "\n",
+    "def do_inference(hostport, work_dir, concurrency, num_tests):\n",
+    "  \"\"\"Tests PredictionService with concurrent requests.\n",
+    "  Args:\n",
+    "    hostport: Host:port address of the PredictionService.\n",
+    "    work_dir: The full path of working directory for test data set.\n",
+    "    concurrency: Maximum number of concurrent requests.\n",
+    "    num_tests: Number of test images to use.\n",
+    "  Returns:\n",
+    "    The classification error rate.\n",
+    "  Raises:\n",
+    "    IOError: An error occurred processing test data set.\n",
+    "  \"\"\"\n",
+    "  test_data_set = mnist_input_data.read_data_sets(work_dir).test\n",
+    "  host, port = hostport.split(':')\n",
+    "  channel = implementations.insecure_channel(host, int(port))\n",
+    "  stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)\n",
+    "  result_counter = _ResultCounter(num_tests, concurrency)\n",
+    "  for _ in range(num_tests):\n",
+    "    request = predict_pb2.PredictRequest()\n",
+    "    request.model_spec.name = 'mnist'\n",
+    "    request.model_spec.signature_name = 'predict_images'\n",
+    "    image, label = test_data_set.next_batch(1)\n",
+    "    request.inputs['images'].CopyFrom(\n",
+    "        tf.contrib.util.make_tensor_proto(image[0], shape=[1, image[0].size]))\n",
+    "    result_counter.throttle()\n",
+    "    result_future = stub.Predict.future(request, 5.0)  # 5 seconds\n",
+    "    result_future.add_done_callback(\n",
+    "        _create_rpc_callback(label[0], result_counter))\n",
+    "  return result_counter.get_error_rate()\n",
+    "\n",
+    "\n",
+    "\n",
+    "error_rate = do_inference(server, work_dir, concurrency, num_tests)\n",
+    "print('\\nAccuracy : %s%%' % (100 - (error_rate * 100)))\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "PySpark",
+   "language": "",
+   "name": "pysparkkernel"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "python",
+    "version": 2
+   },
+   "mimetype": "text/x-python",
+   "name": "pyspark",
+   "pygments_lexer": "python2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}