personalize_temporal_holdout/personalize_metadata_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Personalize with temporal evaluation on hold-out set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The notebook largely follows the basic notebook, with the additional tweaks of holding-out 1% of \"future\" data for every user and using item meta-data. Then, we set up an inference endpoint to bring recommendation and evaluate externally on the held-out data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tempfile, subprocess, urllib.request, zipfile\n",
    "import pandas as pd, numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import scipy.sparse as ss\n",
    "import json\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker.amazon.common as smac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download and process a dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tempfile.TemporaryDirectory() as tmpdir:\n",
    "    urllib.request.urlretrieve(\n",
    "        'http://files.grouplens.org/datasets/movielens/ml-20m.zip',\n",
    "        tmpdir + '/ml-20m.zip')\n",
    "    zipfile.ZipFile(tmpdir + '/ml-20m.zip').extractall(tmpdir)\n",
    "    df = pd.read_csv(tmpdir + '/ml-20m/ratings.csv')\n",
    "    movies = pd.read_csv(tmpdir + '/ml-20m/movies.csv', index_col='movieId')\n",
    "    vocab_size = df.movieId.max() + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "131263"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_time_ratio = 0.01\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## hold out the last bit of data in time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfo = df.copy()\n",
    "df = df[df.timestamp < df.timestamp.max() * (1-test_time_ratio) + df.timestamp.min() * test_time_ratio]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## convert into Personalize format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = ['USER_ID','ITEM_ID','EVENT_VALUE','TIMESTAMP']\n",
    "df['EVENT_TYPE']='RATING'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USER_ID</th>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th>EVENT_VALUE</th>\n",
       "      <th>TIMESTAMP</th>\n",
       "      <th>EVENT_TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112486027</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484676</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484819</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484727</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1112484580</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   USER_ID  ITEM_ID  EVENT_VALUE   TIMESTAMP EVENT_TYPE\n",
       "0        1        2          3.5  1112486027     RATING\n",
       "1        1       29          3.5  1112484676     RATING\n",
       "2        1       32          3.5  1112484819     RATING\n",
       "3        1       47          3.5  1112484727     RATING\n",
       "4        1       50          3.5  1112484580     RATING"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for demo we may want to upload a small dataset\n",
    "#df=df.loc[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('interactions.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## process item metadata into Personalize formats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = movies.reset_index()\n",
    "\n",
    "del movies['title']\n",
    "\n",
    "movies.columns=['ITEM_ID','GENRE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th>GENRE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ITEM_ID                                        GENRE\n",
       "0        1  Adventure|Animation|Children|Comedy|Fantasy\n",
       "1        2                   Adventure|Children|Fantasy\n",
       "2        3                               Comedy|Romance\n",
       "3        4                         Comedy|Drama|Romance\n",
       "4        5                                       Comedy"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.to_csv('item_metadata.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# upload data to s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "make_bucket: demo-temporal-holdout-metadata-56924\r\n"
     ]
    }
   ],
   "source": [
    "os.environ['AWS_DEFAULT_REGION']=\"us-east-1\"\n",
    "suffix = str(np.random.uniform())[4:9]\n",
    "bucket = \"demo-temporal-holdout-metadata-\"+suffix     # replace with the name of your S3 bucket\n",
    "!aws s3 mb s3://{bucket}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize = boto3.client(service_name='personalize', endpoint_url='https://personalize.us-east-1.amazonaws.com')\n",
    "personalize_runtime = boto3.client(service_name='personalize-runtime', endpoint_url='https://personalize-runtime.us-east-1.amazonaws.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "interactions_filename = 'interactions.csv'\n",
    "boto3.Session().resource('s3').Bucket(bucket).Object(interactions_filename).upload_file(interactions_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_metadata_file = 'item_metadata.csv'\n",
    "boto3.Session().resource('s3').Bucket(bucket).Object(item_metadata_file).upload_file(item_metadata_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create schemas for our two types of data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_name=\"DEMO-temporal-metadata-schema-\"+suffix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"schemaArn\": \"arn:aws:personalize:us-east-1:261294318658:schema/DEMO-temporal-metadata-schema-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"c2ae0ef6-ccdb-4b12-afb7-956a4552dfea\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:39:41 GMT\",\n",
      "      \"x-amzn-requestid\": \"c2ae0ef6-ccdb-4b12-afb7-956a4552dfea\",\n",
      "      \"content-length\": \"101\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "schema = {\n",
    "    \"type\": \"record\",\n",
    "    \"name\": \"Interactions\",\n",
    "    \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    "    \"fields\": [\n",
    "        {\n",
    "            \"name\": \"USER_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"ITEM_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"EVENT_VALUE\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"TIMESTAMP\",\n",
    "            \"type\": \"long\"\n",
    "        },\n",
    "        { \n",
    "            \"name\": \"EVENT_TYPE\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "    ],\n",
    "    \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_schema_response = personalize.create_schema(\n",
    "    name = schema_name,\n",
    "    schema = json.dumps(schema)\n",
    ")\n",
    "\n",
    "schema_arn = create_schema_response['schemaArn']\n",
    "print(json.dumps(create_schema_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_schema_name=\"DEMO-temporal-metadata-metadataschema-\"+suffix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"schemaArn\": \"arn:aws:personalize:us-east-1:261294318658:schema/DEMO-temporal-metadata-metadataschema-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"0d6a5d42-bb7d-43aa-951a-cbec53a2d82b\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:39:42 GMT\",\n",
      "      \"x-amzn-requestid\": \"0d6a5d42-bb7d-43aa-951a-cbec53a2d82b\",\n",
      "      \"content-length\": \"109\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "metadata_schema = {\n",
    " \"type\": \"record\",\n",
    " \"name\": \"Items\",\n",
    " \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    " \"fields\": [\n",
    " {\n",
    " \"name\": \"ITEM_ID\",\n",
    " \"type\": \"string\"\n",
    " },\n",
    " {\n",
    " \"name\": \"GENRE\",\n",
    " \"type\": \"string\",\n",
    " \"categorical\": True\n",
    " }\n",
    " ],\n",
    " \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_metadata_schema_response = personalize.create_schema(\n",
    "    name = metadata_schema_name,\n",
    "    schema = json.dumps(metadata_schema)\n",
    ")\n",
    "\n",
    "metadata_schema_arn = create_metadata_schema_response['schemaArn']\n",
    "print(json.dumps(create_metadata_schema_response, indent=2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create a dataset group where we are going to add the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetGroupArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-group/DEMO-temporal-metadata-dataset-group-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"d3cece12-3787-431b-9bd9-7b659560668f\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:39:42 GMT\",\n",
      "      \"x-amzn-requestid\": \"d3cece12-3787-431b-9bd9-7b659560668f\",\n",
      "      \"content-length\": \"121\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_group_name = \"DEMO-temporal-metadata-dataset-group-\" + suffix\n",
    "\n",
    "create_dataset_group_response = personalize.create_dataset_group(\n",
    "    name = dataset_group_name\n",
    ")\n",
    "\n",
    "dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n",
    "print(json.dumps(create_dataset_group_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetGroup: CREATE PENDING\n",
      "DatasetGroup: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_group_response = personalize.describe_dataset_group(\n",
    "        datasetGroupArn = dataset_group_arn\n",
    "    )\n",
    "    status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n",
    "    print(\"DatasetGroup: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## add our two datasets into the dataset group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset/DEMO-temporal-metadata-dataset-group-56924/INTERACTIONS\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"dbb2934b-fa62-4285-8a9a-d01237bdc9fa\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:40:02 GMT\",\n",
      "      \"x-amzn-requestid\": \"dbb2934b-fa62-4285-8a9a-d01237bdc9fa\",\n",
      "      \"content-length\": \"123\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_type = \"INTERACTIONS\"\n",
    "create_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = schema_arn,\n",
    "    name = \"DEMO-temporal-metadata-dataset-interactions-\" + suffix\n",
    ")\n",
    "\n",
    "interactions_dataset_arn = create_dataset_response['datasetArn']\n",
    "print(json.dumps(create_dataset_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset/DEMO-temporal-metadata-dataset-group-56924/ITEMS\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"d3c3a4c5-9a02-4210-b4d4-0425f071b80e\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:40:02 GMT\",\n",
      "      \"x-amzn-requestid\": \"d3c3a4c5-9a02-4210-b4d4-0425f071b80e\",\n",
      "      \"content-length\": \"116\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_type = \"ITEMS\"\n",
    "create_metadata_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = metadata_schema_arn,\n",
    "    name = \"DEMO-temporal-metadata-dataset-items-\" + suffix\n",
    ")\n",
    "\n",
    "metadata_dataset_arn = create_metadata_dataset_response['datasetArn']\n",
    "print(json.dumps(create_metadata_dataset_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## add approprate permissions to the s3 bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "s3 = boto3.client(\"s3\")\n",
    "\n",
    "policy = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Principal\": {\n",
    "                \"Service\": \"personalize.amazonaws.com\"\n",
    "            },\n",
    "            \"Action\": [\n",
    "                \"s3:GetObject\",\n",
    "                \"s3:ListBucket\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                \"arn:aws:s3:::{}\".format(bucket),\n",
    "                \"arn:aws:s3:::{}/*\".format(bucket)\n",
    "            ]\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy));\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from botocore.exceptions import ClientError"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create a role with permissions we need"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "iam = boto3.client(\"iam\")\n",
    "\n",
    "role_name = \"PersonalizeS3Role-\"+suffix\n",
    "assume_role_policy_document = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "          \"Effect\": \"Allow\",\n",
    "          \"Principal\": {\n",
    "            \"Service\": \"personalize.amazonaws.com\"\n",
    "          },\n",
    "          \"Action\": \"sts:AssumeRole\"\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "try:\n",
    "    create_role_response = iam.create_role(\n",
    "        RoleName = role_name,\n",
    "        AssumeRolePolicyDocument = json.dumps(assume_role_policy_document)\n",
    "    );\n",
    "\n",
    "    iam.attach_role_policy(\n",
    "        RoleName = role_name,\n",
    "        PolicyArn = \"arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess\"\n",
    "    );\n",
    "\n",
    "    role_arn = create_role_response[\"Role\"][\"Arn\"]\n",
    "except ClientError as e:\n",
    "    if e.response['Error']['Code'] == 'EntityAlreadyExists':\n",
    "        role_arn = iam.get_role(RoleName=role_name)['Role']['Arn']\n",
    "    else:\n",
    "        raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:iam::261294318658:role/PersonalizeS3Role-56924\n"
     ]
    }
   ],
   "source": [
    "print(role_arn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import data into the created dataset group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sometimes need to wait a bit for the role to be created\n",
    "time.sleep(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetImportJobArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-import-job/DEMO-temporal-dataset-import-job-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"0ce2e431-fb21-4892-a9d1-3b130b622692\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:44:46 GMT\",\n",
      "      \"x-amzn-requestid\": \"0ce2e431-fb21-4892-a9d1-3b130b622692\",\n",
      "      \"content-length\": \"126\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"DEMO-temporal-dataset-import-job-\"+suffix,\n",
    "    datasetArn = interactions_dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, 'interactions.csv')\n",
    "    },\n",
    "    roleArn = role_arn\n",
    ")\n",
    "\n",
    "dataset_import_job_arn = create_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetImportJobArn\": \"arn:aws:personalize:us-east-1:261294318658:dataset-import-job/DEMO-temporal-metadata-dataset-import-job-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"2a7664f2-ca20-47f2-a2e7-a837c2ccc005\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 01:44:50 GMT\",\n",
      "      \"x-amzn-requestid\": \"2a7664f2-ca20-47f2-a2e7-a837c2ccc005\",\n",
      "      \"content-length\": \"135\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_metadata_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"DEMO-temporal-metadata-dataset-import-job-\"+suffix,\n",
    "    datasetArn = metadata_dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, 'item_metadata.csv')\n",
    "    },\n",
    "    roleArn = role_arn\n",
    ")\n",
    "\n",
    "metadata_dataset_import_job_arn = create_metadata_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_metadata_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = describe_dataset_import_job_response[\"datasetImportJob\"]\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetImportJob: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = metadata_dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = describe_dataset_import_job_response[\"datasetImportJob\"]\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## create a solution which uses meta-data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:personalize:::recipe/aws-hrnn\n",
      "arn:aws:personalize:::recipe/aws-hrnn-coldstart\n",
      "arn:aws:personalize:::recipe/aws-hrnn-metadata\n",
      "arn:aws:personalize:::recipe/aws-personalized-ranking\n",
      "arn:aws:personalize:::recipe/aws-popularity-count\n",
      "arn:aws:personalize:::recipe/aws-sims\n"
     ]
    }
   ],
   "source": [
    "recipe_list = personalize.list_recipes()\n",
    "for recipe in recipe_list['recipes']:\n",
    "    print(recipe['recipeArn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe_arn = \"arn:aws:personalize:::recipe/aws-hrnn-metadata\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-temporal-metadata-solution-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"a95ffd0b-b349-41fb-b574-744504eb86e1\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 02:03:56 GMT\",\n",
      "      \"x-amzn-requestid\": \"a95ffd0b-b349-41fb-b574-744504eb86e1\",\n",
      "      \"content-length\": \"107\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_response = personalize.create_solution(\n",
    "    name = \"DEMO-temporal-metadata-solution-\"+suffix,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    recipeArn = recipe_arn\n",
    ")\n",
    "\n",
    "solution_arn = create_solution_response['solutionArn']\n",
    "print(json.dumps(create_solution_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-temporal-metadata-solution-56924/9f802675\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"78d76834-4c5b-47d6-8ac4-5bc5c0f7b406\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 02:03:56 GMT\",\n",
      "      \"x-amzn-requestid\": \"78d76834-4c5b-47d6-8ac4-5bc5c0f7b406\",\n",
      "      \"content-length\": \"123\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_version_response = personalize.create_solution_version(\n",
    "    solutionArn = solution_arn\n",
    ")\n",
    "\n",
    "solution_version_arn = create_solution_version_response['solutionVersionArn']\n",
    "print(json.dumps(create_solution_version_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SolutionVersion: CREATE PENDING\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_solution_version_response = personalize.describe_solution_version(\n",
    "        solutionVersionArn = solution_version_arn\n",
    "    )\n",
    "    status = describe_solution_version_response[\"solutionVersion\"][\"status\"]\n",
    "    print(\"SolutionVersion: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## get metrics for the solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-east-1:261294318658:solution/DEMO-temporal-metadata-solution-56924/9f802675\",\n",
      "  \"metrics\": {\n",
      "    \"coverage\": 0.2187,\n",
      "    \"mean_reciprocal_rank_at_25\": 0.1056,\n",
      "    \"normalized_discounted_cumulative_gain_at_10\": 0.1431,\n",
      "    \"normalized_discounted_cumulative_gain_at_25\": 0.172,\n",
      "    \"normalized_discounted_cumulative_gain_at_5\": 0.1216,\n",
      "    \"precision_at_10\": 0.0212,\n",
      "    \"precision_at_25\": 0.0132,\n",
      "    \"precision_at_5\": 0.0297\n",
      "  },\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"63818c96-8b7f-4abc-a827-b16922fe87ba\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 04:34:18 GMT\",\n",
      "      \"x-amzn-requestid\": \"63818c96-8b7f-4abc-a827-b16922fe87ba\",\n",
      "      \"content-length\": \"420\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "get_solution_metrics_response = personalize.get_solution_metrics(\n",
    "    solutionVersionArn = solution_version_arn\n",
    ")\n",
    "\n",
    "print(json.dumps(get_solution_metrics_response, indent=2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a campaign from the solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"campaignArn\": \"arn:aws:personalize:us-east-1:261294318658:campaign/DEMO-metadata-campaign-56924\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"3e454f0d-421f-48f8-9b79-05a900df1f59\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Sat, 01 Jun 2019 04:35:18 GMT\",\n",
      "      \"x-amzn-requestid\": \"3e454f0d-421f-48f8-9b79-05a900df1f59\",\n",
      "      \"content-length\": \"98\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_campaign_response = personalize.create_campaign(\n",
    "    name = \"DEMO-metadata-campaign-\"+suffix,\n",
    "    solutionVersionArn = solution_version_arn,\n",
    "    minProvisionedTPS = 2,    \n",
    ")\n",
    "\n",
    "campaign_arn = create_campaign_response['campaignArn']\n",
    "print(json.dumps(create_campaign_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Campaign: CREATE PENDING\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_campaign_response = personalize.describe_campaign(\n",
    "        campaignArn = campaign_arn\n",
    "    )\n",
    "    status = describe_campaign_response[\"campaign\"][\"status\"]\n",
    "    print(\"Campaign: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the held out data, to compute metrics externally from the system as well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USER_ID</th>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th>EVENT_VALUE</th>\n",
       "      <th>TIMESTAMP</th>\n",
       "      <th>EVENT_TYPE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3534</th>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1424733487</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3535</th>\n",
       "      <td>31</td>\n",
       "      <td>110</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1424733473</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3536</th>\n",
       "      <td>31</td>\n",
       "      <td>260</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1424733433</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3537</th>\n",
       "      <td>31</td>\n",
       "      <td>364</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1424844807</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3538</th>\n",
       "      <td>31</td>\n",
       "      <td>527</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1424733598</td>\n",
       "      <td>RATING</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      USER_ID  ITEM_ID  EVENT_VALUE   TIMESTAMP EVENT_TYPE\n",
       "3534       31        1          3.0  1424733487     RATING\n",
       "3535       31      110          5.0  1424733473     RATING\n",
       "3536       31      260          5.0  1424733433     RATING\n",
       "3537       31      364          3.0  1424844807     RATING\n",
       "3538       31      527          0.5  1424733598     RATING"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dfo.copy()\n",
    "df = df[df.timestamp >= df.timestamp.max() * (1-test_time_ratio) + df.timestamp.min() * test_time_ratio]\n",
    "df.columns = ['USER_ID','ITEM_ID','EVENT_VALUE','TIMESTAMP']\n",
    "df['EVENT_TYPE']='RATING'\n",
    "test_users = df['USER_ID'].unique()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import numpy as np\n",
    "from metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1833720d3e4748eabdf78a5ba3c43ee6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3448), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "relevance = []\n",
    "for user_id in tqdm_notebook(test_users):\n",
    "    true_items = set(df[df['USER_ID']==user_id]['ITEM_ID'].values)\n",
    "    rec_response = personalize_runtime.get_recommendations(\n",
    "        campaignArn = campaign_arn,\n",
    "        userId = str(user_id)\n",
    "    )\n",
    "    rec_items = [int(x['itemId']) for x in rec_response['itemList']]\n",
    "    relevance.append([int(x in true_items) for x in rec_items])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean_reciprocal_rank 0.11662028311307289\n",
      "precision_at_5 0.12784222737819023\n",
      "precision_at_10 0.12975638051044083\n",
      "precision_at_25 0.08581206496519722\n",
      "normalized_discounted_cumulative_gain_at_5 0.16459408903295208\n",
      "normalized_discounted_cumulative_gain_at_10 0.23502802046231244\n",
      "normalized_discounted_cumulative_gain_at_25 0.30649821765257435\n"
     ]
    }
   ],
   "source": [
    "print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in relevance]))\n",
    "print('precision_at_5', np.mean([precision_at_k(r, 5) for r in relevance]))\n",
    "print('precision_at_10', np.mean([precision_at_k(r, 10) for r in relevance]))\n",
    "print('precision_at_25', np.mean([precision_at_k(r, 25) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in relevance]))\n",
    "print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in relevance]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ubuntu/AWSConciergeHRNNExamples/movielens_benchmarks'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_mxnet_p36)",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}