Pytorch Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Pytorch 自动求导数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "************************************************** \n",
      " x is : \n",
      " tensor([[1., 1.],\n",
      "        [1., 1.]], requires_grad=True)\n",
      "************************************************** \n",
      " y = x + 2 is :\n",
      " tensor([[3., 3.],\n",
      "        [3., 3.]], grad_fn=<AddBackward0>)\n",
      "************************************************** \n",
      " z = y * y *3 is :\n",
      " tensor([[27., 27.],\n",
      "        [27., 27.]], grad_fn=<MulBackward0>)\n",
      "************************************************** \n",
      " out = y.mean() is :\n",
      " tensor(27., grad_fn=<MeanBackward0>)\n",
      "\n",
      " ************************************************** \n",
      " x grad is:\n",
      " tensor([[4.5000, 4.5000],\n",
      "        [4.5000, 4.5000]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "## 1. Inference\n",
    "x= torch.ones(2,2,requires_grad = True)\n",
    "print(\"*\"*50,'\\n x is : \\n',x)\n",
    "\n",
    "y = x + 2\n",
    "print(\"*\"*50,'\\n y = x + 2 is :\\n',y)\n",
    "\n",
    "z = y * y  * 3\n",
    "print(\"*\"*50,'\\n z = y * y *3 is :\\n',z)\n",
    "\n",
    "out = z.mean()\n",
    "print(\"*\"*50, '\\n out = y.mean() is :\\n', out)\n",
    "\n",
    "## 2 . Backward - out is scalar\n",
    "out.backward()\n",
    "print(\"\\n\",\"*\"*50, \"\\n x grad is:\\n\", x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " tensor([ 0.5926,  0.5438, -1.0933], requires_grad=True) \n",
      " tensor([  606.7927,   556.8962, -1119.4907], grad_fn=<MulBackward0>)\n",
      "tensor([1024., 1024.,  512.])\n"
     ]
    }
   ],
   "source": [
    "## when output is vector \n",
    "x = torch.randn(3, requires_grad = True)\n",
    "y = x * 2\n",
    "while y.data.norm() < 1000:\n",
    "    y  = y * 2\n",
    "print('\\n',x, '\\n',y)\n",
    "v = torch.tensor([1,1,.5])\n",
    "y.backward(v)\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Pytorch: 定义新的autograd函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1., 1., 1., 1., 0., 1., 0., 1.])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "class MyRelu(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, input):\n",
    "        ctx.save_for_backward(input)\n",
    "        return input.clamp(min = 0)\n",
    "    \n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        input, = ctx.saved_tensors\n",
    "        grad_input = grad_output.clone()\n",
    "        grad_input[input < 0] = 0\n",
    "        return grad_input\n",
    "    \n",
    "x = torch.tensor([1,1,2,3,-1,1,-2,0], dtype = torch.float32, requires_grad = True)\n",
    "relu = MyRelu.apply\n",
    "y = relu(x)\n",
    "y.backward(torch.ones(x.shape))\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: DeprecationWarning: 'saved_variables' is deprecated; use 'saved_tensors'\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "class LinearFunction(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, input, weight, bias = None):\n",
    "        #print(type(input))\n",
    "        ctx.save_for_backward(input, weight, bias)\n",
    "        output = input.mm(weight.t())\n",
    "        if bias is not None:\n",
    "            output += bias.unsqueeze(0).expand_as(output)\n",
    "        return output\n",
    "    \n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        input, weight, bias = ctx.saved_variables\n",
    "        grad_input = grad_weight = grad_bias = None\n",
    "\n",
    "        if ctx.needs_input_grad[0]:\n",
    "            grad_input = grad_output.mm(weight)\n",
    "        if ctx.needs_input_grad[1]:\n",
    "            grad_weight = grad_output.t().mm(input)\n",
    "        if bias is not None and ctx.needs_input_grad[2]:\n",
    "            grad_bias = grad_output.sum(0).squeeze(0)\n",
    "\n",
    "        return grad_input, grad_weight, grad_bias\n",
    "\n",
    "def linear(input, weight, bias = None):\n",
    "    return LinearFunction()(intput, weight, bias)\n",
    "\n",
    "linear = LinearFunction.apply\n",
    "\n",
    "#Check backwordb !!\n",
    "input = (Variable(torch.randn(3, 5).double(), requires_grad = True),\n",
    "        Variable(torch.randn(3, 5).double(), requires_grad = True),None)\n",
    "test = gradcheck(linear, input, eps=1e-6, atol=1e-4)\n",
    "print(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.1637,  0.1026,  0.0172,  0.1918],\n",
      "        [ 0.1469,  0.0212,  0.1668, -0.0239],\n",
      "        [ 0.2117, -0.0148,  0.1033,  0.0719]],\n",
      "       grad_fn=<LinearFunctionBackward>)\n"
     ]
    }
   ],
   "source": [
    "#expand to Module \n",
    "import torch.nn as nn\n",
    "class MyLinear(nn.Module):\n",
    "    def __init__(self, input_features, output_features, bias = True):\n",
    "        super(MyLinear, self).__init__()\n",
    "        self.input_features = input_features\n",
    "        self.output_features = output_features\n",
    "        \n",
    "        self.weight = nn.Parameter(torch.Tensor(output_features, input_features))\n",
    "        if bias:\n",
    "            self.bias = nn.Parameter(torch.Tensor(output_features))\n",
    "        else:\n",
    "            self.register_parameter('bias', None)\n",
    "        self.weight.data.uniform_(-0.1, 0.1)\n",
    "        if bias is not None:\n",
    "            self.bias.data.uniform_(-0.1, 0.1)\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return LinearFunction.apply(input, self.weight, self.bias)\n",
    "\n",
    "x = torch.randn(3,5)\n",
    "fc1 = MyLinear(5, 4, bias= None)\n",
    "output =fc1(x)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.6238e-03,  1.4402e-02, -5.3506e-01, -2.5703e-01,  9.4487e-02],\n",
      "        [ 2.3796e+00, -1.4976e+00,  1.4442e-01, -4.4399e-01,  1.0962e+00],\n",
      "        [ 1.3845e+00, -9.0367e-01, -9.9576e-02, -1.2792e-01,  1.0103e+00],\n",
      "        [-1.2547e+00,  2.4198e-01,  4.0399e-01, -2.2229e+00,  5.6809e-01],\n",
      "        [ 3.9360e-01,  2.0778e+00, -3.9482e-01, -9.9242e-01,  1.7436e+00],\n",
      "        [-5.0620e-01,  7.0693e-01, -4.6644e-01, -6.2862e-01,  1.6091e-02],\n",
      "        [-1.8387e+00, -8.8481e-01,  1.0101e+00, -2.5769e-01,  1.9752e+00],\n",
      "        [-4.1672e-01,  5.5078e-01, -5.5916e-01, -1.1018e+00, -1.5692e+00],\n",
      "        [ 1.8142e+00,  7.6748e-01, -5.8636e-01,  7.6672e-02,  7.0167e-01],\n",
      "        [-1.0566e-01, -3.5950e-01, -4.6776e-01, -2.2812e-01,  1.9953e+00]])\n",
      "tensor([[-1.4433, -0.1183, -0.1677,  0.9031, -1.1447],\n",
      "        [-0.9931, -0.2226, -2.3782, -0.6939,  0.4632],\n",
      "        [ 0.5125,  0.8370, -0.4732,  0.0306,  0.8650],\n",
      "        [-0.7645,  0.8148, -1.5711,  0.7891, -0.3908],\n",
      "        [ 0.0653, -0.2223, -2.1257,  0.5043,  0.6903],\n",
      "        [ 0.2363,  0.0552,  1.0784, -0.5121,  0.4725],\n",
      "        [-2.0604, -0.2659, -2.2937, -0.4184, -1.0059],\n",
      "        [ 1.6356,  0.6396, -1.0074,  0.2790,  0.8447],\n",
      "        [ 0.0712,  1.4029,  0.4416, -0.6537,  0.1877],\n",
      "        [ 0.8661,  0.2511, -1.2870, -1.0338, -1.2414]])\n",
      "tensor([[1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1.]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AddBackward0 at 0x7f8de23cc940>"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torch.autograd import Variable\n",
    "a = torch.randn(10, 5)\n",
    "print(a)\n",
    "b = torch.randn(10, 5)\n",
    "print(b)\n",
    "x = Variable(a, requires_grad=True)\n",
    "y = Variable(b, requires_grad=True)\n",
    "z = x + y\n",
    "z.backward(torch.ones(a.shape))\n",
    "print(x.grad)            # x的梯度 10x1 的全1 tensor\n",
    "z.grad_fn         # <SumBackward0 object at 0x7f809e33fcf8> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}