Predict Future Sales _ Kaggle course final project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing the necessary tools required also setting the plot features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sys.version_info(major=3, minor=7, micro=3, releaselevel='final', serial=0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import product\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from xgboost import XGBRegressor\n",
    "from xgboost import plot_importance\n",
    "\n",
    "def plot_features(booster , features):\n",
    "    fig , ax = plt,subplots(1,1,figsize = figsize)\n",
    "    return plot_importance(booster = booster , ax = ax)\n",
    "\n",
    "import time\n",
    "import sys\n",
    "import gc\n",
    "import pickle\n",
    "sys.version_info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = pd.read_csv('E:\\Kaggle-coursera\\items.csv')\n",
    "shops = pd.read_csv('E:\\Kaggle-coursera\\shops.csv')\n",
    "cats = pd.read_csv('E:\\Kaggle-coursera\\item_categories.csv')\n",
    "train = pd.read_csv('E:\\Kaggle-coursera\\sales_train.csv')\n",
    "test = pd.read_csv('E:\\Kaggle-coursera\\TEST.csv').set_index('ID')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking the differrnt labels and actual content of the files provided"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Items are : \n",
      "\n",
      "                                           item_name  item_id  \\\n",
      "0          ! ВО ВЛАСТИ НАВАЖДЕНИЯ (ПЛАСТ.)         D        0   \n",
      "1  !ABBYY FineReader 12 Professional Edition Full...        1   \n",
      "2      ***В ЛУЧАХ СЛАВЫ   (UNV)                    D        2   \n",
      "3    ***ГОЛУБАЯ ВОЛНА  (Univ)                      D        3   \n",
      "4        ***КОРОБКА (СТЕКЛО)                       D        4   \n",
      "5  ***НОВЫЕ АМЕРИКАНСКИЕ ГРАФФИТИ  (UNI)         ...        5   \n",
      "6           ***УДАР ПО ВОРОТАМ (UNI)               D        6   \n",
      "7         ***УДАР ПО ВОРОТАМ-2 (UNI)               D        7   \n",
      "8           ***ЧАЙ С МУССОЛИНИ                     D        8   \n",
      "9      ***ШУГАРЛЭНДСКИЙ ЭКСПРЕСС (UNI)             D        9   \n",
      "\n",
      "   item_category_id  \n",
      "0                40  \n",
      "1                76  \n",
      "2                40  \n",
      "3                40  \n",
      "4                40  \n",
      "5                40  \n",
      "6                40  \n",
      "7                40  \n",
      "8                40  \n",
      "9                40  \n",
      "\n",
      "The shape is : (22170, 3)\n",
      "\n",
      "Shops are : \n",
      "\n",
      "                        shop_name  shop_id\n",
      "0   !Якутск Орджоникидзе, 56 фран        0\n",
      "1   !Якутск ТЦ \"Центральный\" фран        1\n",
      "2                Адыгея ТЦ \"Мега\"        2\n",
      "3  Балашиха ТРК \"Октябрь-Киномир\"        3\n",
      "4        Волжский ТЦ \"Волга Молл\"        4\n",
      "5          Вологда ТРЦ \"Мармелад\"        5\n",
      "6      Воронеж (Плехановская, 13)        6\n",
      "7          Воронеж ТРЦ \"Максимир\"        7\n",
      "8    Воронеж ТРЦ Сити-Парк \"Град\"        8\n",
      "9               Выездная Торговля        9\n",
      "\n",
      "The shape is : (60, 2)\n",
      "\n",
      "Item categories are : \n",
      "\n",
      "        item_category_name  item_category_id\n",
      "0  PC - Гарнитуры/Наушники                 0\n",
      "1         Аксессуары - PS2                 1\n",
      "2         Аксессуары - PS3                 2\n",
      "3         Аксессуары - PS4                 3\n",
      "4         Аксессуары - PSP                 4\n",
      "5      Аксессуары - PSVita                 5\n",
      "6    Аксессуары - XBOX 360                 6\n",
      "7    Аксессуары - XBOX ONE                 7\n",
      "8           Билеты (Цифра)                 8\n",
      "9          Доставка товара                 9\n",
      "\n",
      "The shape is : (84, 2)\n",
      "\n",
      "Sales train are : \n",
      "\n",
      "         date  date_block_num  shop_id  item_id  item_price  item_cnt_day\n",
      "0  02.01.2013               0       59    22154      999.00           1.0\n",
      "1  03.01.2013               0       25     2552      899.00           1.0\n",
      "2  05.01.2013               0       25     2552      899.00          -1.0\n",
      "3  06.01.2013               0       25     2554     1709.05           1.0\n",
      "4  15.01.2013               0       25     2555     1099.00           1.0\n",
      "5  10.01.2013               0       25     2564      349.00           1.0\n",
      "6  02.01.2013               0       25     2565      549.00           1.0\n",
      "7  04.01.2013               0       25     2572      239.00           1.0\n",
      "8  11.01.2013               0       25     2572      299.00           1.0\n",
      "9  03.01.2013               0       25     2573      299.00           3.0\n",
      "\n",
      "The shape is : (2935849, 6)\n",
      "\n",
      "Test set are : \n",
      "\n",
      "    shop_id  item_id\n",
      "ID                  \n",
      "0         5     5037\n",
      "1         5     5320\n",
      "2         5     5233\n",
      "3         5     5232\n",
      "4         5     5268\n",
      "5         5     5039\n",
      "6         5     5041\n",
      "7         5     5046\n",
      "8         5     5319\n",
      "9         5     5003\n",
      "\n",
      "The shape is : (214200, 2)\n"
     ]
    }
   ],
   "source": [
    "print(\"Items are : \")\n",
    "print()\n",
    "print(items.head(10))\n",
    "print()\n",
    "print(\"The shape is : {}\".format(items.shape))\n",
    "print()\n",
    "print(\"Shops are : \")\n",
    "print()\n",
    "print(shops.head(10))\n",
    "print()\n",
    "print(\"The shape is : {}\".format(shops.shape))\n",
    "print()\n",
    "print(\"Item categories are : \")\n",
    "print()\n",
    "print(cats.head(10))\n",
    "print()\n",
    "print(\"The shape is : {}\".format(cats.shape))\n",
    "print()\n",
    "print(\"Sales train are : \")\n",
    "print()\n",
    "print(train.head(10))\n",
    "print()\n",
    "print(\"The shape is : {}\".format(train.shape))\n",
    "print()\n",
    "print(\"Test set are : \")\n",
    "print()\n",
    "print(test.head(10))\n",
    "print()\n",
    "print(\"The shape is : {}\".format(test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setting to dark grid plotting\n",
    "sns.set(style=\"darkgrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature preprocessing and generation with respect to models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1c0bdf475f8>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjwAAAE/CAYAAABCcxmpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAWs0lEQVR4nO3dfZSVZb3w8d8eRkQERTiMCSGuOrpci+OjdrIkBUSPqLxoKpaaIItl+HiwHrJMyNKyoxhZGmKmnc5ZK8VTmYnhKpSWS9NxIiNf+iPEEiQZBGQGGBCHebmeP2z2mYEZGMiZYa75fP5i9r6v+7r2fa29/a69ndmFlFIKAICMlXT1AgAAOprgAQCyJ3gAgOwJHgAge4IHAMie4AEAsle6twOqq7dHY+P7/5vrgwb1i02btr3v56Vz2L/uzf51b/av+7OH77+SkkIcccShbd6/1+BpbEwdEjxN56b7sn/dm/3r3uxf92cPO5ePtACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALJX2hWTlpf/Ng477JA44YRTumJ6AKCH6ZLgeeihH0ehELFggeABADqej7QAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAslfaFZPu3FkbhUKhK6YGAHqgLgmehoaGrpgWAOihfKQFAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGSvkFJKezpg06Zt0di4x0P2aPPm6liw4M4oFCL+7d/Oi/vuu3u/z9VRSkpK/v4YUxQKhbj66s/FU089GZdfPjX++7/vj/Xr34qZM78QP/nJA1FZuTauvvraePLJX0ehEPHJT14S3//+XTF79k2RUopvfeubMXPmrPjFLx6OiIipU6fHQw/9OK655vNx+OEDIuK9a/KDH9wd11zz+diyZXN861vfbDF+9uybon//w4rHHH74gBZjms6zN2vWrI7bb78lBg36p+jTp09ce+117R7bpLV5ly17Pu67b0GUlR0ZffocEqWlpfG5z13XYp2XXz51t8e9v5qvIaXU5nXYn2vUE23eXB0/+tH346qrZrpO3dTgwf1j48aarl4G/4ADcQ+7+2toSUkhBg3q1+b9vb7+9a9/fU8n2LFjZ+w5ifbs4Yf/J1588Q9RXV0VL730x0ipcf9P1kF2bb4XX1wemzZtjJUrV8Sbb/4t6uvr4+WXX4yqqk1/v/+PUVX1dlRXV8XLL78YO3a8EytXrojly1/4++N8MTZsWB/V1VWxcuWKeOON1VFbWxsnnnhyRLx3Tf74xxeitrY2nnzyV8XjmsavXLkiNm16u3jMiSee3GJM03n25tvfvjWqq6ujpmZrVFdXxc6d7R/bpLV5v/GNGyOlxti+fXts2bK5xbmbjm/tce+v5mtYuXJFm9dhf65RT/Tww/8TL7zwe9epGzv00IPjnXd2dvUy+AcciHvY3V9DC4VC9O3bu837O/Qjrc2bq+PZZ58u/tzQUN+R071vGhrqI6UUlZVri7e98872Fvfventl5dri8c2PraxcGymleO6538aWLZtj8+bqeO65ZyKlFL/97dPFMc3HV1aujWeffbo4bs2aN4pjms6zN2vWrG6x/oiI5557pl1jmzRfa9O8y5Y93+o+PvvsMy3Wuevj3l/N1/Dss8/Es8+2fh1aWyu7c52A1vSE14YODZ7Fix+N+vqGjpyi22hsbIxf/vIXsXjxo8WPCPcUgPX19cVx99+/oDim6Tx7c//997R6zvaMbdJ8rU3z/ud//qDN9TZfZ5P2rrc9a6ivry9es13P29pa2Z3rBLSmJ7w2dGjwVFSUR8Q/8HlYRhoa6qOiojwqKsr36Z2uhob6qKxcWxzTdJ692fXdnYj3Prprz9gmzdfaNG/ba08t1tl8/fsy557WEJGKHz/uet7W1sruXCegNT3htaFDg2fkyNMiotCRU3QbvXqVxsiRp8XIkadFr16l+zRuyJChxTFN59mbIUOG7nZboVBo19gmzdfaNG/bay+0WGfz9e/LnHtaQ0QhCoVCq+dtba3sznUCWtMTXhs6NHgmTbowSkt7deQU3UZJSUmcf/5FMWnShVFS8r//0W5LaWlpcdyMGdcWxzSdZ29mzJjZ6jnbM7ZJ87U2zXvVVf+3zfU2X2eT9q63PWsoLS0tXrNdz9vaWtmd6wS0pie8NnRo8AwYcESMGnVG8ed9eWejK/XqVRqFQqHFuyR9+x7a4v5dbx8yZGjx+ObHDhkyNAqFQpx++ug4/PABMWDAEXH66WOiUCjE6NFnFMc0Hz9kyNAYNeqM4rijjx5eHNN0nr05+uhjdnuX5/TTx+zTrxo2X2vTvB//+Cda3cdRo8a0WOeuj3t/NV/DqFFjYtSo1q9Da2tld64T0Jqe8NrQ4b+WPnz4MfHaaytj4MCBMXnyZbF8+e/3/2QdpKSkpPgYC4VCzJhxbWzduiWmTftsrFr116itfTf+/d9nxZo1q2PbtpqYMWNmVFVVxcCBA2PKlOnx0kvLY9as6+Nf//WUWLbs+Zg58//FW2+9FUccMTCmT786KivXxhVXTIs+ffpExHvXZNWqv8YVV0yLf/mX/xPLlj3fYvysWdfHCSecVDymT58+LcY0nWdv/vmfj4vf/a48jjzyA1FWVhZTpkxv99gmrc175JFHxvLlv4+ysiNj0KDBMWjQP8XUqdNbrHPatM/u9rj3V/M1HHfc8W1eh/25Rj3R8OHHxJtvro7LLrvSdeqmDsRfaWbfHIh72N1fQ/f2a+kd/ocHWzN9+uUREfFf//XQ+3peOs+B+EezaD/7173Zv+7PHr7/9vaHB321BACQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQvdKumLRXr15RKBS6YmoAoAfqkuDp3fvg0DsAQGfxkRYAkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQMAZE/wAADZEzwAQPYEDwCQPcEDAGRP8AAA2RM8AED2BA8AkD3BAwBkT/AAANkTPABA9kq7YtLLL58ahx12SFdMDQD0QF0SPKedNjoGD+4fGzfWdMX0AEAP4yMtACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7AkeACB7ggcAyJ7gAQCyJ3gAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7JXu7YCSkkKHTd6R56bj2b/uzf51b/av+7OH76+9Xc9CSil10loAALqEj7QAgOwJHgAge4IHAMie4AEAsid4AIDsCR4AIHuCBwDInuABALIneACA7HV68CxevDjGjx8f48aNi4ULF3b29LTTlClTYsKECXHBBRfEBRdcEC+//HKbe/f888/HpEmTYty4cXHnnXd24arZtm1bTJw4Md58882IaHtv/vznP8dFF10U55xzTtx4441RX18fERGVlZXxmc98Js4999y45pprYvv27V3yOHqqXfdvzpw5MW7cuOLzcOnSpRGx7/tK51iwYEFMmDAhJkyYEPPmzYsIz8EDSupEb731Vho7dmyqrq5O27dvT5MmTUqvvfZaZy6BdmhsbEynn356qqurK97W1t7t2LEjjRkzJq1ZsybV1dWl6dOnp6effroLV99zvfTSS2nixIlpxIgR6W9/+9se92bChAnpxRdfTCmlNGfOnLRw4cKUUkozZsxIjz/+eEoppQULFqR58+Z1zYPpgXbdv5RSmjhxYlq/fn2L4/ZnX+l45eXl6dOf/nSqra1NO3fuTFOnTk2LFy/2HDyAdOo7PM8//3yceuqpMWDAgOjbt2+cc845sWTJks5cAu3w+uuvR0TE9OnT4/zzz48HH3ywzb175ZVXYvjw4TFs2LAoLS2NSZMm2dMu8rOf/SxuvvnmKCsri4hoc2/Wrl0b7777bpx00kkREXHRRRfFkiVLoq6uLl544YU455xzWtxO59h1/3bs2BGVlZXxla98JSZNmhTz58+PxsbGfd5XOsfgwYNj9uzZ0bt37zjooIPiwx/+cKxevdpz8ACy129Lfz9t2LAhBg8eXPy5rKwsXnnllc5cAu2wdevWGDlyZHzta1+Lurq6mDp1apx33nmt7l1re7p+/fquWHaPd+utt7b4ua292fX2wYMHx/r166O6ujr69esXpaWlLW6nc+y6f2+//XaceuqpcfPNN0f//v3j6quvjp///OfRt2/ffdpXOsexxx5b/Pfq1avj17/+dVxxxRWegweQTn2Hp7GxMQqF//369pRSi585MJx88skxb9686N+/fwwcODAmT54c8+fPb3Xv7OmBq629aev21vbOXnadYcOGxT333BNlZWVxyCGHxJQpU+KZZ57Z532lc7322msxffr0+PKXvxzDhg3zHDyAdGrwfOADH4iNGzcWf964cWPx7VsOHH/4wx+ioqKi+HNKKYYOHdrq3tnTA1dbe7Pr7W+//XaUlZXFwIEDo6amJhoaGlocT9d49dVX44knnij+nFKK0tLSfd5XOs/y5ctj2rRp8cUvfjEuvPBCz8EDTKcGzyc+8YmoqKiIqqqq2LFjRzz55JMxevTozlwC7VBTUxPz5s2L2tra2LZtWzz66KPx7W9/u9W9O/HEE2PVqlXxxhtvRENDQzz++OP29ADR1t4MHTo0Dj744Fi+fHlERDz22GMxevToOOigg+KjH/1o/OpXv4qIiEWLFtnLLpRSittuuy22bNkSdXV18dOf/jTOPvvsfd5XOse6deti5syZcccdd8SECRMiwnPwQFNIKaXOnHDx4sVx3333RV1dXUyePDk++9nPdub0tNNdd90VTzzxRDQ2Nsbll18eV155ZZt7V1FREXPnzo3a2toYM2ZMzJkzx9uwXejMM8+MH//4x/HBD36wzb1ZsWJFfPWrX41t27bFiBEjYu7cudG7d+9Yu3ZtzJ49OzZt2hRHHXVUfPe7343DDz+8qx9Sj9J8/xYuXBgLFy6M+vr6GDduXHzpS1+KiLafc23tKx3vP/7jP+KRRx6Jo48+unjbpZdeGsccc4zn4AGi04MHAKCz+UvLAED2BA8AkD3BAwBkT/AAANkTPABA9gQPAJA9wQM9xJ/+9Kf4/Oc/H6+88krcdNNNXb2cNk2fPj2qqqr2acySJUtiypQpHbQiIAeCB3qIE044IebPnx9/+ctfDugvJCwvL+/qJQAZ6tRvSwe6zrJly+LGG2+M+vr6qKmpiTlz5sTcuXPjqaeeinvvvTfq6uqiT58+ccMNN8TJJ58cd999d6xZsybWr18fGzdujBEjRsTHP/7xWLRoUbz55ptx/fXXx8SJE/c456pVq+Kmm26KqqqqKCkpiWuuuSbGjx8fZ555Zlx44YVRUVER69atiwsuuCBmzZoVc+bMiYiIK6+8Mu6///446qij2jz39773vVi8eHEMGDAghg8f3mLOW265JbZv3x4bN26M448/vviXwx966KH4yU9+EhERlZWV8alPfSqeeuopf40YeoIE9Ai/+93v0oQJE9IjjzySZsyYkVJKadWqVWnixImpqqoqpZTSypUr02mnnZa2b9+e5s+fn8aOHZu2bt2aduzYkU455ZQ0d+7clFJKS5cuTePGjdvrnJ/85CfTgw8+mFJKqbKyMp111lmppqYmjR07Nt1+++0ppZTeeuutdMIJJ6Q1a9aklFI67rjj0qZNm/Z43qVLl6bx48enmpqaVFdXl2bMmJGuuOKKlFJKt99+e1q0aFFKKaWdO3emiRMnpiVLlqTa2to0cuTItHLlypRSSnfddVe644479ukaAt2Xd3igBysvL48NGzbEtGnTircVCoVYs2ZNRLz3hb/9+/ePiIiysrIYNWpUREQcffTRsXnz5j2ee/PmzbFixYq45JJLIiLiqKOOit/85jfF+88666yIiDjyyCNj0KBBsWXLlhg2bFi71l1RURFnn3129OvXLyIiLr744njggQciIuL666+P8vLy+OEPfxirV6+ODRs2xDvvvBO9e/eOSy65JB5++OG44YYb4tFHHy2OAfIneKAHa2xsjJEjR8Zdd91VvG3dunVRVlYWS5cu3e2jntLS9r9kNB3b/ItkX3/99RgyZEhERBx88MHF2wuFQqR9/Fq/5sf36tWr+O/rrrsuGhoa4rzzzoszzjgj1q1bVzz20ksvjcmTJ8fHPvaxOPbYY9sdWED3539ahh6mV69eUV9fHxERI0eOjPLy8vjrX/8aERHPPPNMnH/++fHuu+/+w/P069cvRowYEYsWLYqI90Lqsssui5qamnavry2jR4+OJUuWxNatW6OxsTEee+yx4n3PPfdczJw5M8aPHx8RES+//HI0NDRExHvvMp100klx2223xWWXXfaPPDygm/EOD/QwJ510Utxzzz1x7bXXxoIFC+KWW26J6667LlJKUVpaGvfee28ceuih78tc3/nOd+Ib3/hGPPDAA1EoFOLWW2+NwYMH73HMueeeG1OmTIm77747jjvuuFaPGTNmTLz66qtx8cUXx2GHHRbHH398VFdXR0TEF77whZg5c2b07ds3+vXrF6ecckrxI7qIiIsuuii++c1vxpgxY96Xxwh0D4W0r+8jA3RTjY2Nccstt8SQIUNixowZXb0coBN5hwfYb7/85S/jRz/6Uav3TZo0Ka666qr9PvesWbNi1apVrd535513xoc+9KF9Ot+2bdti7Nix8ZGPfCRmz5693+sCuifv8AAA2fM/LQMA2RM8AED2BA8AkD3BAwBkT/AAANn7/7qxjgjZ5a1dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj8AAAE/CAYAAACpRKKqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAUCklEQVR4nO3de4xV9b338c8wA16KqZdCVYQ2NqZpSVNNtZWKQ7TteBkGi7GpR2OrNKXxaGlaqwVFe4lIBOoF4Xho2rQhcp7WqKB4FLw0ahx4EO3FNrGG2FLEUcQLigRhmL3OHx55QJnC8OwZhN/r9d+svX6zf/ubFfY7a88wDVVVVQEAKES/Pb0BAIC+JH4AgKKIHwCgKOIHACiK+AEAiiJ+AICiNPXk5Ndf35BazW/G//867LCBefXVt/b0NvZ65lgf5lg/Zlkf5lgfJc+xX7+GHHLIh7p9vEfxU6tV4qdOzLE+zLE+zLF+zLI+zLE+zHHHfOwFABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQlB7Fz/TpU/Jf/zW3t/YCANDrmnpy8tq1L2fLlq7e2gsAQK/zsRcAUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUXYrftrbH0t7+2P13gsAQK9r2p1Fjz/+aJLkpJOa67oZAIDe5mMvAKAo4gcAKIr4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+AICiiB8AoCjiBwAoivgBAIoifgCAoogfAKAo4gcAKIr4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+AICiiB8AoCjiBwAoivgBAIoifgCAoogfAKAo4gcAKIr4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+AICiiB8AoCjiBwAoivgBAIoifgCAoogfAKAo4gcAKIr4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+AICiiB8AoCjiBwAoivgBAIoifgCAoogfAKAoTbuz6Nlnn0mSjBt3Xl03s6c1NPRLVdW2O3bOOf+W//7vBZk48ZoMHfqxJMmqVSszZcqP09nZmS996fQ8/PCiXHzxhJxwwon/8vuvW/d6/vM/b8kll1yc2bNvzcUXT8iHP3zwLu/v3fU9XQcAu6KU9xl3frbx3vBJkjvu+D/ZuHFj5syZtfXYL34xO52dnUmShx9e9L/H/mOn33/hwvlZseLZ/PznP8+KFc/mnnvu6tH+3l3f03UAsCtKeZ/pcfy8e9enNB0dL+T55/+ZVatWpqPjhfc93tW1JcuX/99u169b93oef/zRVFWVVatWpaqqPP74Y3njjXW79Pzbru/JOgDYFSW9z7jz0wNz5szKL34xu9vH/9Xdn4UL56dWq7Y7VqvVdrmut13fk3UAsCtKep8RPz3Q0fHCDu/6vKura0u3jy1d2v6+x7u6tmTp0vZdeu5t1/dkHQDsipLeZ8RPDxx55JAceeSQbh9vbOz+58dHjDjpfY83NjZlxIiTdum5t13fk3UAsCtKep8RPz3wne9cmvHjL+n28fHj/73bx9raxqZfv4btjvXr1y9jxpy9S8+97fqerAOAXVHS+0yP4+eTn/xUb+zjA+/II4dk6NCPZdiwj+/w7k9jY9O//FX3gw8+JCNHjkpDQ0OGDRuWhoaGjBzZvMu/Srjt+p6sA4BdUdL7jDs/22hoeP84zjnn33LAAQfkO9+5dOux8eMvSf/+/ZMkX/rS6f97rPu7Pu9qaxubY475ZC677LIcc8wne1zV767fl2scgD2nlPeZhqqqqp2f9o5vfetbOeSQw7Z+/aMfXd0rm9rXDRp0UNauXb+nt7HXM8f6MMf6Mcv6MMf6KHmO/fo15LDDBnb/eB/uBQBgjxM/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARWnanUUjR46q9z4AAPrEbsXPSSc113sfAAB9wsdeAEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRF/AAARRE/AEBRxA8AUBTxAwAURfwAAEVp6snJgwYNzhFHHNVbewEA6HU9ip/LL78qtVrVW3sBAOh1PvYCAIoifgCAoogfAKAo4gcAKIr4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+AICiiB8AoCjiBwAoivgBAIoifgCAoogfAKAo4gcAKIr4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+AICiiB8AoCjiBwAoSlNPTu7Xr6G39lEcs6wPc6wPc6wfs6wPc6yPUue4s9fdUFVV1Ud7AQDY43zsBQAURfwAAEURPwBAUcQPAFAU8QMAFEX8AABFET8AQFHEDwBQFPEDABRll+Jn4cKFOfPMM9PS0pJ58+b19p72GhdccEFaW1tz1lln5ayzzsqf//znbme1ZMmStLW1paWlJTfeeOPW488880zOPvvsnHbaabnqqquyZcuWJElHR0fOP//8nH766bn44ouzYcOGPn99ve2tt97K6NGjs3r16iT1m9Gbb76Z8ePH54wzzsj555+ftWvXJkk2b96cyy+/PGeccUbGjh2b5557ro9fce947xwnTZqUlpaWrdflgw8+mKT357u3mzVrVlpbW9Pa2ppp06YlcU3ujh3N0TXZczfffHPOPPPMtLa25te//nUS12NdVTvx0ksvVaecckr1+uuvVxs2bKja2tqqFStW7GzZPq9Wq1UjR46sOjs7tx7rblYbN26sRo0aVa1atarq7Oysxo0bVz3yyCNVVVVVa2tr9cc//rGqqqqaNGlSNW/evKqqqmr8+PHVvffeW1VVVc2aNauaNm1aH7/C3vWnP/2pGj16dDV8+PDq+eefr+uMfvrTn1Zz5sypqqqq5s+fX33ve9+rqqqqfvnLX1ZXX311VVVV9cQTT1Rf+9rX+u4F95L3zrGqqmr06NHVmjVrtjuvL+a7N2tvb6++/vWvV5s2bao2b95cfeMb36gWLlzomuyhHc3xgQcecE320LJly6pzzz236uzsrDZu3Fidcsop1TPPPON6rKOd3vlZsmRJTjzxxBx88ME58MADc9ppp2XRokV90WUfaH//+9+TJOPGjcuYMWNy2223dTurp59+Oh/72McydOjQNDU1pa2tLYsWLcoLL7yQt99+O8cee2yS5Oyzz86iRYvS2dmZ5cuX57TTTtvu+L7k9ttvz49//OMMHjw4Seo6o0ceeSRtbW1JktGjR+exxx5LZ2dnHnnkkYwZMyZJcsIJJ+S1115LR0dHX7/0unrvHDdu3JiOjo5ceeWVaWtry8yZM1Or1fpkvnuzQYMGZeLEiRkwYED69++fT3ziE1m5cqVrsod2NMeOjg7XZA99/vOfz9y5c9PU1JRXX301XV1defPNN12PdbTT+Hn55ZczaNCgrV8PHjw4a9as6dVN7Q3efPPNjBgxIrNnz85vfvOb/Pa3v01HR8cOZ9XdDN97fNCgQVmzZk1ef/31DBw4ME1NTdsd35dMmTIlxx9//Nav6zmjbdc0NTVl4MCBee2113b4vV566aVefZ297b1zfOWVV3LiiSfmuuuuy+23354nn3wyd9xxR5/Md292zDHHbH3zWLlyZe6///40NDS4JntoR3M8+eSTXZO7oX///pk5c2ZaW1szYsQI/0bW2U7jp1arpaHh//1p+Kqqtvu6VMcdd1ymTZuWgw46KIceemjOOeeczJw5c4ez6m6G3R3f0Yz39Zn35oyqqkq/fv3et+bd4/uSoUOHZvbs2Rk8eHAOOOCAXHDBBXn00Uf7ZL77ghUrVmTcuHG54oorMnToUNfkbtp2jkcffbRrcjdNmDAhS5cuzYsvvpiVK1e6Hutop6/q8MMP3+6Hx9auXbv1FnvJnnzyySxdunTr11VVZciQITucVXczfO/xV155JYMHD86hhx6a9evXp6ura7vz92X1nNHgwYPzyiuvJEm2bNmSDRs25OCDD85HP/rRvPzyy+/7XvuSZ599NosXL976dVVVaWpq6pP57u2eeuqpXHjhhbnssssyduxY1+Rueu8cXZM999xzz+WZZ55JkhxwwAFpaWnJsmXLXI91tNP4+eIXv5ilS5fmtddey8aNG/PAAw+kubm5L/b2gbZ+/fpMmzYtmzZtyltvvZX58+dn+vTpO5zVZz/72fzjH//IP//5z3R1deXee+9Nc3NzhgwZkv322y9PPfVUkuTuu+9Oc3Nz+vfvn+OPPz733XdfkmTBggX7/MzrOaNRo0ZlwYIFSZL77rsvxx9/fPr3759Ro0bl7rvvTvJOvO6333458sgj98Cr7T1VVeW6667LG2+8kc7Ozvzud7/LV77ylT6Z797sxRdfzCWXXJIZM2aktbU1iWtyd+xojq7Jnlu9enUmT56czZs3Z/PmzXn44Ydz7rnnuh7rqKGqqmpnJy1cuDBz5sxJZ2dnzjnnnHz729/ui7194N10001ZvHhxarVazjvvvHzzm9/sdlZLly7N1KlTs2nTpowaNSqTJk1KQ0ND/va3v2Xy5Ml56623Mnz48EydOjUDBgzICy+8kIkTJ+bVV1/NEUcckRtuuCEf/vCH9/Arrr9TTz01c+fOzVFHHVW3Ga1bty4TJ07M888/n4MOOigzZszIUUcdlU2bNuWaa67JX//61wwYMCDXXntthg8fvqdHUBfbznHevHmZN29etmzZkpaWlvzwhz9MUr9rsLv57s2uvfba3HnnnRk2bNjWY+eee24+/vGPuyZ7oLs51mo112QP3XLLLbn//vvT2NiYlpaWfPe73/VvZB3tUvwAAOwr9s2fZAIA6Ib4AQCKIn4AgKKIHwCgKOIHACiK+AEAiiJ+gPf5y1/+kgkTJuTpp5/ONddcs6e3s5139wawu/w/P0C37rrrrixevDhz5szZ01sBqBvxA7zPsmXLctVVV2XLli1Zv359WlpaMnXq1Pz+97/Prbfems7Ozuy///750Y9+lOOOOy633HJLVq1alTVr1mTt2rUZPnx4vvCFL2TBggVZvXp1Lr/88owePfpfPuepp56a1tbWtLe3Z/369bnoooty3nnnZdmyZZkyZUoOPPDAbNiwIVdccUWuv/763HvvvdmwYUOuvfba/OEPf0hjY2O+/OUv5/vf/346OzszY8aMLF++PF1dXfn0pz+dyZMnZ+DAgX00QeCDrGlPbwD4YNp///0zbty4LF68OFOnTs3KlStz4403Zu7cuTnkkEOyYsWKXHTRRXnggQeSvPMHLe++++70798/zc3N+chHPpJ58+bloYceyvTp03caP0nyxhtv5M4778yaNWvy1a9+NZ/73OeSvPNXwh966KEMGTIky5Yt23r+zJkzs2nTptx3333p6urKuHHj8sQTT2T58uVpbGzMXXfdlYaGhtxwww2ZMWNGfvKTn/TKrIC9i/gBdkl7e3tefvnlXHjhhVuPNTQ0ZNWqVUne+SPIBx10UJJ3/mr0ySefnCQZNmxY1q1bt0vPcd5556WhoSGHH354Tj755LS3t2f48OE54ogjMmTIkPedv2TJkkyaNCmNjY1pbGzMbbfdliSZPn161q9fnyVLliRJOjs7c9hhh+32awf2LeIH2CW1Wi0jRozITTfdtPXYiy++mMGDB+fBBx/MgAEDtju/qann/7xsu6ZWq6Vfv3d+J+PAAw/s9vyGhobt9rP//vunVqvlyiuvzKhRo5IkGzZsyKZNm3q8H2Df5Le9gG41NjZmy5YtSZIRI0akvb09zz33XJLk0UcfzZgxY/L222/X7fkWLFiQJOno6Eh7e3uam5v/5fkjRozI/PnzU6vVsnnz5kyYMCHLly/PyJEjM2/evGzevDm1Wi1XX311brjhhrrtE9i7ufMDdOvYY4/N7Nmzc+mll2bWrFn52c9+lh/84AepqipNTU259dZb86EPfahuz7d69eqcffbZefvttzN58uQcffTRWbt2bbfnX3rppZkyZUrOOuusdHV15cwzz0xLS0uam5tz/fXXZ+zYsenq6sqnPvWpTJw4sW77BPZuftsL+EA49dRTc/PNN+czn/nMnt4KsI9z5wfoE/fcc09+9atf7fCxtra2Pt4NUDJ3fgCAoviBZwCgKOIHACiK+AEAiiJ+AICiiB8AoCj/A3Eo3bZKBNfaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#1-d plot of the number of items sold\n",
    "plt.figure(figsize = (10,5))\n",
    "plt.xlim(-100 , train.item_cnt_day.max()*1.1) \n",
    "sns.boxplot(x = train.item_cnt_day)\n",
    "\n",
    "#1-d plot of the prices of items sold\n",
    "plt.figure(figsize = (10,5))\n",
    "plt.xlim(-10 , train.item_price.max()*1.1)\n",
    "sns.boxplot(x = train.item_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#droping the outliers\n",
    "train = train[train.item_cnt_day < 1001]\n",
    "train = train[train.item_price < 100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#number of items whose price is absurd (i.e less than 0)\n",
    "train['item_price'][train.item_price < 0].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #now fill up that with that with the median or mean\n",
    "median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()\n",
    "train.loc[train.item_price < 0 , 'item_price'] = median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#several shops are duplicates of each other according to its name ~ fixing them\n",
    "\n",
    "train.loc[train.shop_id == 0 , 'shop_id'] = 57\n",
    "test.loc[test.shop_id == 0 , 'shop_id'] = 57\n",
    "\n",
    "train.loc[train.shop_id == 1, 'shop_id'] = 58\n",
    "test.loc[test.shop_id == 1, 'shop_id'] = 58\n",
    "\n",
    "train.loc[train.shop_id == 10, 'shop_id'] = 11\n",
    "test.loc[test.shop_id == 10, 'shop_id'] = 11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#each shop_name starts with the city name\n",
    "#each category contains type and subtype in its name\n",
    "\n",
    "shops.loc[shops.shop_name == 'Сергиев Посад ТЦ \"7Я\"', 'shop_name'] = 'СергиевПосад ТЦ \"7Я\"'\n",
    "shops['city'] = shops['shop_name'].str.split(' ').map(lambda x: x[0])\n",
    "shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'\n",
    "shops['city_code'] = LabelEncoder().fit_transform(shops['city'])\n",
    "shops = shops[['shop_id','city_code']]\n",
    "\n",
    "cats['split'] = cats['item_category_name'].str.split('-')\n",
    "cats['type'] = cats['split'].map(lambda x: x[0].strip())\n",
    "cats['type_code'] = LabelEncoder().fit_transform(cats['type'])\n",
    "\n",
    "# if subtype is nan then type\n",
    "cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x) > 1 else x[0].strip())\n",
    "cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])\n",
    "cats = cats[['item_category_id','type_code', 'subtype_code']]\n",
    "\n",
    "items.drop(['item_name'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363\n",
      "5100\n",
      "214200\n"
     ]
    }
   ],
   "source": [
    "#number of items new in train as compaired to train set\n",
    "print(len(list(set(test.item_id) - set(train.item_id).intersection(set(test.item_id)))))\n",
    "\n",
    "#total items in test set\n",
    "print(len(list(set(test.item_id))))\n",
    "\n",
    "#total number of pairs\n",
    "print(len(test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature matrix generation\n",
    "### with respect to the 33 months provided to us ~ generating new necessary features ~ also filling the nans after clipping to (0,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#aggregating train set by shop/item pairs to calculate the target aggregates , then clipping it to (0,20) target value.\n",
    "matrix = []\n",
    "\n",
    "cols = ['date_block_num' , 'shop_id' , 'item_id']\n",
    "\n",
    "for i in range(34):\n",
    "    sales = train[train.date_block_num == i]\n",
    "    matrix.append(np.array(list(product([i] , sales.shop_id.unique() , sales.item_id.unique())) , dtype = 'int16'))\n",
    "    \n",
    "matrix = pd.DataFrame(np.vstack(matrix) , columns = cols)\n",
    "\n",
    "matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)\n",
    "matrix['shop_id'] = matrix['shop_id'].astype(np.int8)\n",
    "matrix['item_id'] = matrix['item_id'].astype(np.int16)\n",
    "\n",
    "matrix.sort_values(cols , inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating a revenue column of the item sold\n",
    "train['revenue'] = train['item_price']*train['item_cnt_day']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "group = train.groupby(['date_block_num','shop_id','item_id']).agg({'item_cnt_day': ['sum']})\n",
    "group.columns = ['item_cnt_month']\n",
    "group.reset_index(inplace = True)\n",
    "\n",
    "matrix = pd.merge(matrix , group , on=cols , how='left')\n",
    "matrix['item_cnt_month'] = (matrix['item_cnt_month'].fillna(0).clip(0,20).astype(np.float16))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month\n",
       "0               0        2       19             0.0\n",
       "1               0        2       27             1.0\n",
       "2               0        2       28             0.0\n",
       "3               0        2       29             0.0\n",
       "4               0        2       32             0.0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#lets view the damn matrix\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['date_block_num'] = 34\n",
    "test['date_block_num'] = test['date_block_num'].astype(np.int8)\n",
    "test['shop_id'] = test['shop_id'].astype(np.int8)\n",
    "test['item_id'] = test['item_id'].astype(np.int16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)\n",
    "matrix.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')\n",
    "matrix = pd.merge(matrix, items, on=['item_id'], how='left')\n",
    "matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')\n",
    "\n",
    "matrix['city_code'] = matrix['city_code'].astype(np.int8)\n",
    "matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)\n",
    "matrix['type_code'] = matrix['type_code'].astype(np.int8)\n",
    "matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  \n",
       "0                40         11             4  \n",
       "1                19          5            10  \n",
       "2                30          8            55  \n",
       "3                23          5            16  \n",
       "4                40         11             4  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dealing with the target lags (shifting and merging)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#target lags\n",
    "def lag_features(df , lags , col):\n",
    "    tmp = df[['date_block_num' , 'shop_id' , 'item_id' , col]]\n",
    "    for i in lags : \n",
    "        shifted = tmp.copy()\n",
    "        shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]\n",
    "        shifted['date_block_num']+=i\n",
    "        df = pd.merge(df , shifted , on=['date_block_num','shop_id','item_id'], how='left')\n",
    "    return df    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix = lag_features(matrix , [1,2,3,6,12] , 'item_cnt_month')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mean Encoding the features generated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>item_cnt_month_lag_3</th>\n",
       "      <th>item_cnt_month_lag_6</th>\n",
       "      <th>item_cnt_month_lag_12</th>\n",
       "      <th>date_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  item_cnt_month_lag_3  item_cnt_month_lag_6  \\\n",
       "0                   NaN                   NaN                   NaN   \n",
       "1                   NaN                   NaN                   NaN   \n",
       "2                   NaN                   NaN                   NaN   \n",
       "3                   NaN                   NaN                   NaN   \n",
       "4                   NaN                   NaN                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_12  date_avg_item_cnt_lag_1  \n",
       "0                    NaN                      NaN  \n",
       "1                    NaN                      NaN  \n",
       "2                    NaN                      NaN  \n",
       "3                    NaN                      NaN  \n",
       "4                    NaN                      NaN  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#mean encoded feature\n",
    "group = matrix.groupby(['date_block_num']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')\n",
    "matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_avg_item_cnt')\n",
    "matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>item_cnt_month_lag_3</th>\n",
       "      <th>item_cnt_month_lag_6</th>\n",
       "      <th>item_cnt_month_lag_12</th>\n",
       "      <th>date_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_avg_item_cnt_lag_2</th>\n",
       "      <th>date_item_avg_item_cnt_lag_3</th>\n",
       "      <th>date_item_avg_item_cnt_lag_6</th>\n",
       "      <th>date_item_avg_item_cnt_lag_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  item_cnt_month_lag_3  item_cnt_month_lag_6  \\\n",
       "0                   NaN                   NaN                   NaN   \n",
       "1                   NaN                   NaN                   NaN   \n",
       "2                   NaN                   NaN                   NaN   \n",
       "3                   NaN                   NaN                   NaN   \n",
       "4                   NaN                   NaN                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_12  date_avg_item_cnt_lag_1  \\\n",
       "0                    NaN                      NaN   \n",
       "1                    NaN                      NaN   \n",
       "2                    NaN                      NaN   \n",
       "3                    NaN                      NaN   \n",
       "4                    NaN                      NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_1  date_item_avg_item_cnt_lag_2  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_3  date_item_avg_item_cnt_lag_6  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_12  \n",
       "0                            NaN  \n",
       "1                            NaN  \n",
       "2                            NaN  \n",
       "3                            NaN  \n",
       "4                            NaN  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'item_id']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_item_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')\n",
    "matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')\n",
    "matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_item_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_avg_item_cnt_lag_2</th>\n",
       "      <th>date_item_avg_item_cnt_lag_3</th>\n",
       "      <th>date_item_avg_item_cnt_lag_6</th>\n",
       "      <th>date_item_avg_item_cnt_lag_12</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_item_avg_item_cnt_lag_1  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_2  date_item_avg_item_cnt_lag_3  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_6  date_item_avg_item_cnt_lag_12  \\\n",
       "0                           NaN                            NaN   \n",
       "1                           NaN                            NaN   \n",
       "2                           NaN                            NaN   \n",
       "3                           NaN                            NaN   \n",
       "4                           NaN                            NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_1  date_shop_avg_item_cnt_lag_2  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_3  date_shop_avg_item_cnt_lag_6  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_12  \n",
       "0                            NaN  \n",
       "1                            NaN  \n",
       "2                            NaN  \n",
       "3                            NaN  \n",
       "4                            NaN  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'shop_id']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_shop_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')\n",
    "matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')\n",
    "matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_item_avg_item_cnt_lag_2</th>\n",
       "      <th>date_item_avg_item_cnt_lag_3</th>\n",
       "      <th>date_item_avg_item_cnt_lag_6</th>\n",
       "      <th>date_item_avg_item_cnt_lag_12</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_item_avg_item_cnt_lag_2  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_3  date_item_avg_item_cnt_lag_6  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_12  date_shop_avg_item_cnt_lag_1  \\\n",
       "0                            NaN                           NaN   \n",
       "1                            NaN                           NaN   \n",
       "2                            NaN                           NaN   \n",
       "3                            NaN                           NaN   \n",
       "4                            NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_2  date_shop_avg_item_cnt_lag_3  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_6  date_shop_avg_item_cnt_lag_12  \\\n",
       "0                           NaN                            NaN   \n",
       "1                           NaN                            NaN   \n",
       "2                           NaN                            NaN   \n",
       "3                           NaN                            NaN   \n",
       "4                           NaN                            NaN   \n",
       "\n",
       "   date_cat_avg_item_cnt_lag_1  \n",
       "0                          NaN  \n",
       "1                          NaN  \n",
       "2                          NaN  \n",
       "3                          NaN  \n",
       "4                          NaN  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'item_category_id']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_cat_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')\n",
    "matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_cat_avg_item_cnt')\n",
    "matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_item_avg_item_cnt_lag_3</th>\n",
       "      <th>date_item_avg_item_cnt_lag_6</th>\n",
       "      <th>date_item_avg_item_cnt_lag_12</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_item_avg_item_cnt_lag_3  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_6  date_item_avg_item_cnt_lag_12  \\\n",
       "0                           NaN                            NaN   \n",
       "1                           NaN                            NaN   \n",
       "2                           NaN                            NaN   \n",
       "3                           NaN                            NaN   \n",
       "4                           NaN                            NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_1  date_shop_avg_item_cnt_lag_2  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_3  date_shop_avg_item_cnt_lag_6  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_12  date_cat_avg_item_cnt_lag_1  \\\n",
       "0                            NaN                          NaN   \n",
       "1                            NaN                          NaN   \n",
       "2                            NaN                          NaN   \n",
       "3                            NaN                          NaN   \n",
       "4                            NaN                          NaN   \n",
       "\n",
       "   date_shop_cat_avg_item_cnt_lag_1  \n",
       "0                               NaN  \n",
       "1                               NaN  \n",
       "2                               NaN  \n",
       "3                               NaN  \n",
       "4                               NaN  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = ['date_shop_cat_avg_item_cnt']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')\n",
    "matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_shop_cat_avg_item_cnt')\n",
    "matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_item_avg_item_cnt_lag_6</th>\n",
       "      <th>date_item_avg_item_cnt_lag_12</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_item_avg_item_cnt_lag_6  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_item_avg_item_cnt_lag_12  date_shop_avg_item_cnt_lag_1  \\\n",
       "0                            NaN                           NaN   \n",
       "1                            NaN                           NaN   \n",
       "2                            NaN                           NaN   \n",
       "3                            NaN                           NaN   \n",
       "4                            NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_2  date_shop_avg_item_cnt_lag_3  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_6  date_shop_avg_item_cnt_lag_12  \\\n",
       "0                           NaN                            NaN   \n",
       "1                           NaN                            NaN   \n",
       "2                           NaN                            NaN   \n",
       "3                           NaN                            NaN   \n",
       "4                           NaN                            NaN   \n",
       "\n",
       "   date_cat_avg_item_cnt_lag_1  date_shop_cat_avg_item_cnt_lag_1  \\\n",
       "0                          NaN                               NaN   \n",
       "1                          NaN                               NaN   \n",
       "2                          NaN                               NaN   \n",
       "3                          NaN                               NaN   \n",
       "4                          NaN                               NaN   \n",
       "\n",
       "   date_shop_type_avg_item_cnt_lag_1  \n",
       "0                                NaN  \n",
       "1                                NaN  \n",
       "2                                NaN  \n",
       "3                                NaN  \n",
       "4                                NaN  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'shop_id', 'type_code']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = ['date_shop_type_avg_item_cnt']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')\n",
    "matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_shop_type_avg_item_cnt')\n",
    "matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_item_avg_item_cnt_lag_12</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_subtype_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_item_avg_item_cnt_lag_12  \\\n",
       "0                   NaN  ...                            NaN   \n",
       "1                   NaN  ...                            NaN   \n",
       "2                   NaN  ...                            NaN   \n",
       "3                   NaN  ...                            NaN   \n",
       "4                   NaN  ...                            NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_1  date_shop_avg_item_cnt_lag_2  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_3  date_shop_avg_item_cnt_lag_6  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_12  date_cat_avg_item_cnt_lag_1  \\\n",
       "0                            NaN                          NaN   \n",
       "1                            NaN                          NaN   \n",
       "2                            NaN                          NaN   \n",
       "3                            NaN                          NaN   \n",
       "4                            NaN                          NaN   \n",
       "\n",
       "   date_shop_cat_avg_item_cnt_lag_1  date_shop_type_avg_item_cnt_lag_1  \\\n",
       "0                               NaN                                NaN   \n",
       "1                               NaN                                NaN   \n",
       "2                               NaN                                NaN   \n",
       "3                               NaN                                NaN   \n",
       "4                               NaN                                NaN   \n",
       "\n",
       "   date_shop_subtype_avg_item_cnt_lag_1  \n",
       "0                                   NaN  \n",
       "1                                   NaN  \n",
       "2                                   NaN  \n",
       "3                                   NaN  \n",
       "4                                   NaN  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = ['date_shop_subtype_avg_item_cnt']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')\n",
    "matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_shop_subtype_avg_item_cnt')\n",
    "matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>date_city_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_shop_avg_item_cnt_lag_1  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_2  date_shop_avg_item_cnt_lag_3  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_6  date_shop_avg_item_cnt_lag_12  \\\n",
       "0                           NaN                            NaN   \n",
       "1                           NaN                            NaN   \n",
       "2                           NaN                            NaN   \n",
       "3                           NaN                            NaN   \n",
       "4                           NaN                            NaN   \n",
       "\n",
       "   date_cat_avg_item_cnt_lag_1  date_shop_cat_avg_item_cnt_lag_1  \\\n",
       "0                          NaN                               NaN   \n",
       "1                          NaN                               NaN   \n",
       "2                          NaN                               NaN   \n",
       "3                          NaN                               NaN   \n",
       "4                          NaN                               NaN   \n",
       "\n",
       "   date_shop_type_avg_item_cnt_lag_1  date_shop_subtype_avg_item_cnt_lag_1  \\\n",
       "0                                NaN                                   NaN   \n",
       "1                                NaN                                   NaN   \n",
       "2                                NaN                                   NaN   \n",
       "3                                NaN                                   NaN   \n",
       "4                                NaN                                   NaN   \n",
       "\n",
       "   date_city_avg_item_cnt_lag_1  \n",
       "0                           NaN  \n",
       "1                           NaN  \n",
       "2                           NaN  \n",
       "3                           NaN  \n",
       "4                           NaN  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'city_code']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_city_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')\n",
    "matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_city_avg_item_cnt')\n",
    "matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_2</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_3</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>date_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_city_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_shop_avg_item_cnt_lag_2  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_3  date_shop_avg_item_cnt_lag_6  \\\n",
       "0                           NaN                           NaN   \n",
       "1                           NaN                           NaN   \n",
       "2                           NaN                           NaN   \n",
       "3                           NaN                           NaN   \n",
       "4                           NaN                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_12  date_cat_avg_item_cnt_lag_1  \\\n",
       "0                            NaN                          NaN   \n",
       "1                            NaN                          NaN   \n",
       "2                            NaN                          NaN   \n",
       "3                            NaN                          NaN   \n",
       "4                            NaN                          NaN   \n",
       "\n",
       "   date_shop_cat_avg_item_cnt_lag_1  date_shop_type_avg_item_cnt_lag_1  \\\n",
       "0                               NaN                                NaN   \n",
       "1                               NaN                                NaN   \n",
       "2                               NaN                                NaN   \n",
       "3                               NaN                                NaN   \n",
       "4                               NaN                                NaN   \n",
       "\n",
       "   date_shop_subtype_avg_item_cnt_lag_1  date_city_avg_item_cnt_lag_1  \\\n",
       "0                                   NaN                           NaN   \n",
       "1                                   NaN                           NaN   \n",
       "2                                   NaN                           NaN   \n",
       "3                                   NaN                           NaN   \n",
       "4                                   NaN                           NaN   \n",
       "\n",
       "   date_item_city_avg_item_cnt_lag_1  \n",
       "0                                NaN  \n",
       "1                                NaN  \n",
       "2                                NaN  \n",
       "3                                NaN  \n",
       "4                                NaN  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'item_id', 'city_code']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_item_city_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')\n",
    "matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_item_city_avg_item_cnt')\n",
    "matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "group = matrix.groupby(['date_block_num', 'type_code']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_type_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')\n",
    "matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_type_avg_item_cnt')\n",
    "matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_6</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>date_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_subtype_avg_item_cnt_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_shop_avg_item_cnt_lag_6  \\\n",
       "0                   NaN  ...                           NaN   \n",
       "1                   NaN  ...                           NaN   \n",
       "2                   NaN  ...                           NaN   \n",
       "3                   NaN  ...                           NaN   \n",
       "4                   NaN  ...                           NaN   \n",
       "\n",
       "   date_shop_avg_item_cnt_lag_12  date_cat_avg_item_cnt_lag_1  \\\n",
       "0                            NaN                          NaN   \n",
       "1                            NaN                          NaN   \n",
       "2                            NaN                          NaN   \n",
       "3                            NaN                          NaN   \n",
       "4                            NaN                          NaN   \n",
       "\n",
       "   date_shop_cat_avg_item_cnt_lag_1  date_shop_type_avg_item_cnt_lag_1  \\\n",
       "0                               NaN                                NaN   \n",
       "1                               NaN                                NaN   \n",
       "2                               NaN                                NaN   \n",
       "3                               NaN                                NaN   \n",
       "4                               NaN                                NaN   \n",
       "\n",
       "   date_shop_subtype_avg_item_cnt_lag_1  date_city_avg_item_cnt_lag_1  \\\n",
       "0                                   NaN                           NaN   \n",
       "1                                   NaN                           NaN   \n",
       "2                                   NaN                           NaN   \n",
       "3                                   NaN                           NaN   \n",
       "4                                   NaN                           NaN   \n",
       "\n",
       "   date_item_city_avg_item_cnt_lag_1  date_type_avg_item_cnt_lag_1  \\\n",
       "0                                NaN                           NaN   \n",
       "1                                NaN                           NaN   \n",
       "2                                NaN                           NaN   \n",
       "3                                NaN                           NaN   \n",
       "4                                NaN                           NaN   \n",
       "\n",
       "   date_subtype_avg_item_cnt_lag_1  \n",
       "0                              NaN  \n",
       "1                              NaN  \n",
       "2                              NaN  \n",
       "3                              NaN  \n",
       "4                              NaN  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group = matrix.groupby(['date_block_num', 'subtype_code']).agg({'item_cnt_month': ['mean']})\n",
    "group.columns = [ 'date_subtype_avg_item_cnt' ]\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')\n",
    "matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)\n",
    "matrix = lag_features(matrix, [1], 'date_subtype_avg_item_cnt')\n",
    "matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()\n",
    "\n",
    "#this ends the mean encodings with feature creation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EDA ~ dealing with the trends that could be observed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_shop_avg_item_cnt_lag_12</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>date_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>delta_price_lag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_shop_avg_item_cnt_lag_12  \\\n",
       "0                   NaN  ...                            NaN   \n",
       "1                   NaN  ...                            NaN   \n",
       "2                   NaN  ...                            NaN   \n",
       "3                   NaN  ...                            NaN   \n",
       "4                   NaN  ...                            NaN   \n",
       "\n",
       "   date_cat_avg_item_cnt_lag_1  date_shop_cat_avg_item_cnt_lag_1  \\\n",
       "0                          NaN                               NaN   \n",
       "1                          NaN                               NaN   \n",
       "2                          NaN                               NaN   \n",
       "3                          NaN                               NaN   \n",
       "4                          NaN                               NaN   \n",
       "\n",
       "   date_shop_type_avg_item_cnt_lag_1  date_shop_subtype_avg_item_cnt_lag_1  \\\n",
       "0                                NaN                                   NaN   \n",
       "1                                NaN                                   NaN   \n",
       "2                                NaN                                   NaN   \n",
       "3                                NaN                                   NaN   \n",
       "4                                NaN                                   NaN   \n",
       "\n",
       "   date_city_avg_item_cnt_lag_1  date_item_city_avg_item_cnt_lag_1  \\\n",
       "0                           NaN                                NaN   \n",
       "1                           NaN                                NaN   \n",
       "2                           NaN                                NaN   \n",
       "3                           NaN                                NaN   \n",
       "4                           NaN                                NaN   \n",
       "\n",
       "   date_type_avg_item_cnt_lag_1  date_subtype_avg_item_cnt_lag_1  \\\n",
       "0                           NaN                              NaN   \n",
       "1                           NaN                              NaN   \n",
       "2                           NaN                              NaN   \n",
       "3                           NaN                              NaN   \n",
       "4                           NaN                              NaN   \n",
       "\n",
       "   delta_price_lag  \n",
       "0              0.0  \n",
       "1              0.0  \n",
       "2              0.0  \n",
       "3              0.0  \n",
       "4              0.0  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#trend feature creation of the last 6 months ~\n",
    "\n",
    "group = train.groupby(['item_id']).agg({'item_price': ['mean']})\n",
    "group.columns = ['item_avg_item_price']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['item_id'], how='left')\n",
    "matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)\n",
    "\n",
    "group = train.groupby(['date_block_num','item_id']).agg({'item_price': ['mean']})\n",
    "group.columns = ['date_item_avg_item_price']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')\n",
    "matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)\n",
    "\n",
    "lags = [1,2,3,4,5,6]\n",
    "matrix = lag_features(matrix, lags, 'date_item_avg_item_price')\n",
    "\n",
    "for i in lags:\n",
    "    matrix['delta_price_lag_'+str(i)] = \\\n",
    "        (matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price']) / matrix['item_avg_item_price']\n",
    "\n",
    "def select_trend(row):\n",
    "    for i in lags:\n",
    "        if row['delta_price_lag_'+str(i)]:\n",
    "            return row['delta_price_lag_'+str(i)]\n",
    "    return 0\n",
    "    \n",
    "matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)\n",
    "matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)\n",
    "matrix['delta_price_lag'].fillna(0, inplace=True)\n",
    "\n",
    "features_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']\n",
    "for i in lags:\n",
    "    features_to_drop += ['date_item_avg_item_price_lag_'+str(i)]\n",
    "    features_to_drop += ['delta_price_lag_'+str(i)]\n",
    "\n",
    "matrix.drop(features_to_drop, axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assessing the trend in the last month after which the next month we have to predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "      <th>city_code</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>type_code</th>\n",
       "      <th>subtype_code</th>\n",
       "      <th>item_cnt_month_lag_1</th>\n",
       "      <th>item_cnt_month_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>date_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_cat_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_shop_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>date_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_item_city_avg_item_cnt_lag_1</th>\n",
       "      <th>date_type_avg_item_cnt_lag_1</th>\n",
       "      <th>date_subtype_avg_item_cnt_lag_1</th>\n",
       "      <th>delta_price_lag</th>\n",
       "      <th>delta_revenue_lag_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>8</td>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  shop_id  item_id  item_cnt_month  city_code  \\\n",
       "0               0        2       19             0.0          0   \n",
       "1               0        2       27             1.0          0   \n",
       "2               0        2       28             0.0          0   \n",
       "3               0        2       29             0.0          0   \n",
       "4               0        2       32             0.0          0   \n",
       "\n",
       "   item_category_id  type_code  subtype_code  item_cnt_month_lag_1  \\\n",
       "0                40         11             4                   NaN   \n",
       "1                19          5            10                   NaN   \n",
       "2                30          8            55                   NaN   \n",
       "3                23          5            16                   NaN   \n",
       "4                40         11             4                   NaN   \n",
       "\n",
       "   item_cnt_month_lag_2  ...  date_cat_avg_item_cnt_lag_1  \\\n",
       "0                   NaN  ...                          NaN   \n",
       "1                   NaN  ...                          NaN   \n",
       "2                   NaN  ...                          NaN   \n",
       "3                   NaN  ...                          NaN   \n",
       "4                   NaN  ...                          NaN   \n",
       "\n",
       "   date_shop_cat_avg_item_cnt_lag_1  date_shop_type_avg_item_cnt_lag_1  \\\n",
       "0                               NaN                                NaN   \n",
       "1                               NaN                                NaN   \n",
       "2                               NaN                                NaN   \n",
       "3                               NaN                                NaN   \n",
       "4                               NaN                                NaN   \n",
       "\n",
       "   date_shop_subtype_avg_item_cnt_lag_1  date_city_avg_item_cnt_lag_1  \\\n",
       "0                                   NaN                           NaN   \n",
       "1                                   NaN                           NaN   \n",
       "2                                   NaN                           NaN   \n",
       "3                                   NaN                           NaN   \n",
       "4                                   NaN                           NaN   \n",
       "\n",
       "   date_item_city_avg_item_cnt_lag_1  date_type_avg_item_cnt_lag_1  \\\n",
       "0                                NaN                           NaN   \n",
       "1                                NaN                           NaN   \n",
       "2                                NaN                           NaN   \n",
       "3                                NaN                           NaN   \n",
       "4                                NaN                           NaN   \n",
       "\n",
       "   date_subtype_avg_item_cnt_lag_1  delta_price_lag  delta_revenue_lag_1  \n",
       "0                              NaN              0.0                  NaN  \n",
       "1                              NaN              0.0                  NaN  \n",
       "2                              NaN              0.0                  NaN  \n",
       "3                              NaN              0.0                  NaN  \n",
       "4                              NaN              0.0                  NaN  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#trends in the last month of shop\n",
    "group = train.groupby(['date_block_num','shop_id']).agg({'revenue': ['sum']})\n",
    "group.columns = ['date_shop_revenue']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')\n",
    "matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)\n",
    "\n",
    "group = group.groupby(['shop_id']).agg({'date_shop_revenue': ['mean']})\n",
    "group.columns = ['shop_avg_revenue']\n",
    "group.reset_index(inplace=True)\n",
    "\n",
    "matrix = pd.merge(matrix, group, on=['shop_id'], how='left')\n",
    "matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)\n",
    "\n",
    "matrix['delta_revenue'] = (matrix['date_shop_revenue'] - matrix['shop_avg_revenue']) / matrix['shop_avg_revenue']\n",
    "matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)\n",
    "\n",
    "matrix = lag_features(matrix, [1], 'delta_revenue')\n",
    "\n",
    "matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)\n",
    "\n",
    "matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#extra special nuggies ~ \n",
    "matrix['month'] = matrix['date_block_num'] % 12\n",
    "\n",
    "days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])\n",
    "matrix['days'] = matrix['month'].map(days).astype(np.int8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a HashTable ~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1602.856077671051"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Months since the last sale for each shop/item pair and for item only. I use programing approach.\n",
    "\n",
    "#Create HashTable with key equals to {shop_id,item_id} and value equals to date_block_num. Iterate data from the top. Foreach row if {row.shop_id,row.item_id} is not present in the table, then add it to the table and set its value to row.date_block_num. if HashTable contains key, then calculate the difference beteween cached value and row.date_block_num.\n",
    "\n",
    "ts = time.time()\n",
    "cache = {}\n",
    "matrix['item_shop_last_sale'] = -1\n",
    "matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)\n",
    "for idx, row in matrix.iterrows():    \n",
    "    key = str(row.item_id)+' '+str(row.shop_id)\n",
    "    if key not in cache:\n",
    "        if row.item_cnt_month!=0:\n",
    "            cache[key] = row.date_block_num\n",
    "    else:\n",
    "        last_date_block_num = cache[key]\n",
    "        matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num\n",
    "        cache[key] = row.date_block_num         \n",
    "time.time() - ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "934.2788043022156"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts = time.time()\n",
    "cache = {}\n",
    "matrix['item_last_sale'] = -1\n",
    "matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)\n",
    "for idx, row in matrix.iterrows():    \n",
    "    key = row.item_id\n",
    "    if key not in cache:\n",
    "        if row.item_cnt_month!=0:\n",
    "            cache[key] = row.date_block_num\n",
    "    else:\n",
    "        last_date_block_num = cache[key]\n",
    "        if row.date_block_num>last_date_block_num:\n",
    "            matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num\n",
    "            cache[key] = row.date_block_num         \n",
    "time.time() - ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#months since the first sale for each shop/item pair and for item only\n",
    "matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')\n",
    "matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#final preparations\n",
    "\n",
    "#because of the using 12 as lag value drop first 12 months. Also drop all the columns with this month calculated values (other words which can not be calcucated for the test set)\n",
    "\n",
    "matrix = matrix[matrix.date_block_num > 11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the lags produced (nans) need to be filled up with 0s\n",
    "\n",
    "def fill_na(df):\n",
    "    for col in df.columns:\n",
    "        if ('_lag_' in col) & (df[col].isnull().any()):\n",
    "            if ('item_cnt' in col):\n",
    "                df[col].fillna(0, inplace=True)         \n",
    "    return df\n",
    "\n",
    "matrix = fill_na(matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 6639294 entries, 4488710 to 11128003\n",
      "Data columns (total 40 columns):\n",
      "date_block_num                          int8\n",
      "shop_id                                 int8\n",
      "item_id                                 int16\n",
      "item_cnt_month                          float16\n",
      "city_code                               int8\n",
      "item_category_id                        int8\n",
      "type_code                               int8\n",
      "subtype_code                            int8\n",
      "item_cnt_month_lag_1                    float16\n",
      "item_cnt_month_lag_2                    float16\n",
      "item_cnt_month_lag_3                    float16\n",
      "item_cnt_month_lag_6                    float16\n",
      "item_cnt_month_lag_12                   float16\n",
      "date_avg_item_cnt_lag_1                 float16\n",
      "date_item_avg_item_cnt_lag_1            float16\n",
      "date_item_avg_item_cnt_lag_2            float16\n",
      "date_item_avg_item_cnt_lag_3            float16\n",
      "date_item_avg_item_cnt_lag_6            float16\n",
      "date_item_avg_item_cnt_lag_12           float16\n",
      "date_shop_avg_item_cnt_lag_1            float16\n",
      "date_shop_avg_item_cnt_lag_2            float16\n",
      "date_shop_avg_item_cnt_lag_3            float16\n",
      "date_shop_avg_item_cnt_lag_6            float16\n",
      "date_shop_avg_item_cnt_lag_12           float16\n",
      "date_cat_avg_item_cnt_lag_1             float16\n",
      "date_shop_cat_avg_item_cnt_lag_1        float16\n",
      "date_shop_type_avg_item_cnt_lag_1       float16\n",
      "date_shop_subtype_avg_item_cnt_lag_1    float16\n",
      "date_city_avg_item_cnt_lag_1            float16\n",
      "date_item_city_avg_item_cnt_lag_1       float16\n",
      "date_type_avg_item_cnt_lag_1            float16\n",
      "date_subtype_avg_item_cnt_lag_1         float16\n",
      "delta_price_lag                         float16\n",
      "delta_revenue_lag_1                     float16\n",
      "month                                   int8\n",
      "days                                    int8\n",
      "item_shop_last_sale                     int8\n",
      "item_last_sale                          int8\n",
      "item_shop_first_sale                    int8\n",
      "item_first_sale                         int8\n",
      "dtypes: float16(27), int16(1), int8(12)\n",
      "memory usage: 481.2 MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "#printing all the columns\n",
    "print(matrix.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting the matrix into the pickle file and then deleting all necessary data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix.to_pickle('data.pkl')\n",
    "del matrix\n",
    "del cache\n",
    "del group\n",
    "del items\n",
    "del shops\n",
    "del cats\n",
    "del train\n",
    "\n",
    "# leave test for submission\n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_pickle('data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[[\n",
    "    'date_block_num',\n",
    "    'shop_id',\n",
    "    'item_id',\n",
    "    'item_cnt_month',\n",
    "    'city_code',\n",
    "    'item_category_id',\n",
    "    'type_code',\n",
    "    'subtype_code',\n",
    "    'item_cnt_month_lag_1',\n",
    "    'item_cnt_month_lag_2',\n",
    "    'item_cnt_month_lag_3',\n",
    "    'item_cnt_month_lag_6',\n",
    "    'item_cnt_month_lag_12',\n",
    "    'date_avg_item_cnt_lag_1',\n",
    "     'date_item_avg_item_cnt_lag_1',\n",
    "    'date_item_avg_item_cnt_lag_2',\n",
    "    'date_item_avg_item_cnt_lag_3',\n",
    "    'date_item_avg_item_cnt_lag_6',\n",
    "    'date_item_avg_item_cnt_lag_12',\n",
    "    'date_shop_avg_item_cnt_lag_1',\n",
    "    'date_shop_avg_item_cnt_lag_2',\n",
    "    'date_shop_avg_item_cnt_lag_3',\n",
    "    'date_shop_avg_item_cnt_lag_6',\n",
    "    'date_shop_avg_item_cnt_lag_12',\n",
    "    'date_cat_avg_item_cnt_lag_1',\n",
    "    'date_shop_cat_avg_item_cnt_lag_1',\n",
    "    #'date_shop_type_avg_item_cnt_lag_1',\n",
    "    #'date_shop_subtype_avg_item_cnt_lag_1',\n",
    "    'date_city_avg_item_cnt_lag_1',\n",
    "    'date_item_city_avg_item_cnt_lag_1',\n",
    "    #'date_type_avg_item_cnt_lag_1',\n",
    "    #'date_subtype_avg_item_cnt_lag_1',\n",
    "    'delta_price_lag',\n",
    "    'month',\n",
    "    'days',\n",
    "    'item_shop_last_sale',\n",
    "    'item_last_sale',\n",
    "    'item_shop_first_sale',\n",
    "    'item_first_sale',\n",
    "]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation ~ following the strategy below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#validation strategy is ~\n",
    "\n",
    "#34th month for the test set\n",
    "#33th month for the train set\n",
    "#13-33th month for the train.\n",
    "\n",
    "X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)\n",
    "Y_train = data[data.date_block_num < 33]['item_cnt_month']\n",
    "X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)\n",
    "Y_valid = data[data.date_block_num == 33]['item_cnt_month']\n",
    "X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "del data\n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model generation and fitting ~ ( XGBoost used with the given evaluation metric )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11:55:29] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.0.0\\src\\gbm\\gbtree.cc:138: Tree method is automatically selected to be 'approx' for faster speed. To use old behavior (exact greedy algorithm on single machine), set tree_method to 'exact'.\n",
      "[0]\tvalidation_0-rmse:1.05117\tvalidation_1-rmse:1.04150\n",
      "Multiple eval metrics have been passed: 'validation_1-rmse' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-rmse hasn't improved in 10 rounds.\n",
      "[1]\tvalidation_0-rmse:0.97137\tvalidation_1-rmse:0.98203\n",
      "[2]\tvalidation_0-rmse:0.91683\tvalidation_1-rmse:0.95044\n",
      "[3]\tvalidation_0-rmse:0.88316\tvalidation_1-rmse:0.93185\n",
      "[4]\tvalidation_0-rmse:0.86404\tvalidation_1-rmse:0.92559\n",
      "[5]\tvalidation_0-rmse:0.85318\tvalidation_1-rmse:0.92290\n",
      "[6]\tvalidation_0-rmse:0.84525\tvalidation_1-rmse:0.91835\n",
      "[7]\tvalidation_0-rmse:0.84057\tvalidation_1-rmse:0.91666\n",
      "[8]\tvalidation_0-rmse:0.83581\tvalidation_1-rmse:0.91353\n",
      "[9]\tvalidation_0-rmse:0.83249\tvalidation_1-rmse:0.91272\n",
      "[10]\tvalidation_0-rmse:0.82920\tvalidation_1-rmse:0.91201\n",
      "[11]\tvalidation_0-rmse:0.82553\tvalidation_1-rmse:0.91198\n",
      "[12]\tvalidation_0-rmse:0.82401\tvalidation_1-rmse:0.91145\n",
      "[13]\tvalidation_0-rmse:0.82258\tvalidation_1-rmse:0.91056\n",
      "[14]\tvalidation_0-rmse:0.82071\tvalidation_1-rmse:0.91002\n",
      "[15]\tvalidation_0-rmse:0.81909\tvalidation_1-rmse:0.90927\n",
      "[16]\tvalidation_0-rmse:0.81815\tvalidation_1-rmse:0.90928\n",
      "[17]\tvalidation_0-rmse:0.81737\tvalidation_1-rmse:0.90923\n",
      "[18]\tvalidation_0-rmse:0.81619\tvalidation_1-rmse:0.90918\n",
      "[19]\tvalidation_0-rmse:0.81452\tvalidation_1-rmse:0.90969\n",
      "[20]\tvalidation_0-rmse:0.81276\tvalidation_1-rmse:0.90994\n",
      "[21]\tvalidation_0-rmse:0.81200\tvalidation_1-rmse:0.90960\n",
      "[22]\tvalidation_0-rmse:0.80981\tvalidation_1-rmse:0.90739\n",
      "[23]\tvalidation_0-rmse:0.80902\tvalidation_1-rmse:0.90869\n",
      "[24]\tvalidation_0-rmse:0.80718\tvalidation_1-rmse:0.90903\n",
      "[25]\tvalidation_0-rmse:0.80605\tvalidation_1-rmse:0.90927\n",
      "[26]\tvalidation_0-rmse:0.80536\tvalidation_1-rmse:0.90882\n",
      "[27]\tvalidation_0-rmse:0.80459\tvalidation_1-rmse:0.90905\n",
      "[28]\tvalidation_0-rmse:0.80371\tvalidation_1-rmse:0.90983\n",
      "[29]\tvalidation_0-rmse:0.80326\tvalidation_1-rmse:0.91044\n",
      "[30]\tvalidation_0-rmse:0.80251\tvalidation_1-rmse:0.91027\n",
      "[31]\tvalidation_0-rmse:0.80184\tvalidation_1-rmse:0.91149\n",
      "[32]\tvalidation_0-rmse:0.80128\tvalidation_1-rmse:0.91108\n",
      "Stopping. Best iteration:\n",
      "[22]\tvalidation_0-rmse:0.80981\tvalidation_1-rmse:0.90739\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1089.6972961425781"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#implementing xgboost ~\n",
    "\n",
    "ts = time.time()\n",
    "\n",
    "model = XGBRegressor(\n",
    "    max_depth=8,\n",
    "    n_estimators=1000,\n",
    "    min_child_weight=300, \n",
    "    colsample_bytree=0.8, \n",
    "    subsample=0.8, \n",
    "    eta=0.3,    \n",
    "    seed=42)\n",
    "\n",
    "model.fit(\n",
    "    X_train, \n",
    "    Y_train, \n",
    "    eval_metric=\"rmse\", \n",
    "    eval_set=[(X_train, Y_train), (X_valid, Y_valid)], \n",
    "    verbose=True, \n",
    "    early_stopping_rounds = 10)\n",
    "\n",
    "time.time() - ts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction ~ clipping ~ Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "#finally the prediction and submission ~\n",
    "\n",
    "Y_pred = model.predict(X_valid).clip(0, 20)\n",
    "Y_test = model.predict(X_test).clip(0, 20)\n",
    "\n",
    "submission = pd.DataFrame({\n",
    "    \"ID\": test.index, \n",
    "    \"item_cnt_month\": Y_test\n",
    "})\n",
    "submission.to_csv('xgb_submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}