train_personalize_with_customer_and_contact_records.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download the data files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "interactions_filename = 'INTERACTIONS.csv'\n",
    "users_filename = 'USERS.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The user dataset in the next cell is originally downloaded from a public dataset from keggel website. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-11-17 01:06:49--  https://aws-contact-center-blog.s3.amazonaws.com/predict-customer-intent/customer11k.csv\n",
      "Resolving aws-contact-center-blog.s3.amazonaws.com (aws-contact-center-blog.s3.amazonaws.com)... 52.92.129.201\n",
      "Connecting to aws-contact-center-blog.s3.amazonaws.com (aws-contact-center-blog.s3.amazonaws.com)|52.92.129.201|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 456961 (446K) [text/csv]\n",
      "Saving to: ‘USERS.csv’\n",
      "\n",
      "USERS.csv           100%[===================>] 446.25K  --.-KB/s    in 0.01s   \n",
      "\n",
      "2021-11-17 01:06:49 (39.9 MB/s) - ‘USERS.csv’ saved [456961/456961]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://aws-contact-center-blog.s3.amazonaws.com/predict-customer-intent/customer11k.csv  -O USERS.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Organize Python imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import json\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "from array import array\n",
    "from calendar import monthrange\n",
    "from random import seed\n",
    "import random\n",
    "from random import randint\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv\n",
    "import string \n",
    "import time\n",
    "from sagemaker import get_execution_role\n",
    "import sagemaker"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate customer interactions Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section we use the same customer IDs as available in the user data set and synthetically generate the interactions events for those customers to be used as trainind data. For real-world use cases, you will get these interaction events from customer interaction channels such as ISV, Web, Mobile apps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################\n",
    "## Declare constants \n",
    "###################\n",
    "\n",
    "# customer ID start, as per the user dataset we have\n",
    "starting_cust_id=30908\n",
    "\n",
    "# customer ID end, as per the user dataset we have\n",
    "ending_cust_id=30908+11000\n",
    "\n",
    "# how many customers we need with one or more events\n",
    "max_interactions_sequence_count=8000\n",
    "\n",
    "# starting date of the dataset\n",
    "event_start_date = datetime.now() - timedelta(days=380) # more than one year back\n",
    "\n",
    "# convert to epoch time\n",
    "epoch_event_start_date = int(event_start_date.strftime('%s'))\n",
    "\n",
    "event_time_range = 31536000 # We will spread the events over an year. This is number of seconds in a year. \n",
    "\n",
    "max_time_between_events = 1059200 # over 12 days\n",
    "min_time_between_events = 3900 # about half days. \n",
    "\n",
    "\n",
    "# lets createa a set of possible interactions \n",
    "# Lets say there are 6 different types of customer intents. \n",
    "# each intent can have a sequence of calls of different types\n",
    "# we will use these fixed set of interaction sequence to generate synthetic data\n",
    "\n",
    "intent_type=6 # number denoting the primary intent\n",
    "interaction_sequence_length = 5 # this number is denoting secondary intent within a primary intent\n",
    "\n",
    "# So we will have a 6 x 5 matrix of intent and sub-intents \n",
    "# Let's name it call_matrix - as in our case each call has an intent and a sub-intent. \n",
    "call_matrix  = [[0 for x in range(interaction_sequence_length)] for y in range(intent_type)]\n",
    "\n",
    "\n",
    "# Intents e.g. HL is an intent e.g., Home Loan related.  HL_NE is new account opening in Home Loan product. \n",
    "# Similarly, \n",
    "\n",
    "call_matrix[0] = ['HL_NE','HL_AS','HL_DS','HL_RP','HL_CS']  # sub-intent sequence for primary intent HL (home loan)\n",
    "call_matrix[1] = ['CC_NE','CC_CA','CC_AA','CC_LC','CC_CS']  # sub-intent sequence for primary intent CC (credit card)\n",
    "call_matrix[2] = ['AC_NE','AC_AS','AC_PA','AC_AA','AC_CS']  # sub-intent sequence for primary intent AC (retail account)\n",
    "call_matrix[3] = ['FD_NE','FD_NA','FD_AD','FD_PW','FD_CS']  # sub-intent sequence for primary intent FD (fixed deposit)\n",
    "call_matrix[4] = ['LI_NE','LI_AS','LI_PA','LI_SW','LI_CS']  # sub-intent sequence for primary intent LI (life insurance)\n",
    "call_matrix[5] = ['TA_NE','TA_AS','TA_AA','TA_FN','TA_CS']  # sub-intent sequence for primary intent TA (trading account)\n",
    "\n",
    "# We will pick randomly from one of these number as number of times a customer contacts for a primary intent.\n",
    "# This constricts a sequence of calls with sub-intents for the same primary intent. E.g., FD_NE followed by FD_NA etc.\n",
    "random_sequence_length = ['1','1','1','2','2','2','2','2','2','3','3','3','3','3','3','4','4','5','5','6','7']\n",
    "\n",
    "###################\n",
    "## Generate interactions event sequences\n",
    "###################\n",
    "\n",
    "\n",
    "#data column headers \n",
    "# USER_ID (string), ITEM_ID (string), TIMESTAMP (long), IMPRESSION\n",
    "\n",
    "\n",
    "with open(interactions_filename, 'w', newline='') as file:\n",
    "    writer = csv.writer(file, delimiter=',')\n",
    "    writer.writerow([\"USER_ID\", \"ITEM_ID\", \"TIMESTAMP\"])\n",
    "\n",
    "    ## Loop for interaction sequences \n",
    "    interactions_sequence_count=0 \n",
    "    while ( interactions_sequence_count <= max_interactions_sequence_count ):\n",
    "\n",
    "        #pick a customer id randomly from the whole range\n",
    "        customer_id = randint(starting_cust_id, ending_cust_id)\n",
    "\n",
    "        #pick a primary intent for the calling events for that customer\n",
    "        intent_id =  randint(0, intent_type-1)\n",
    "\n",
    "        # randomly select how many calling events will be recorded for a particular sequence of events \n",
    "        # involving this particular primary intent \n",
    "        max_sequence_length = int(random.choice(random_sequence_length))\n",
    "\n",
    "        sequence_length = 0  ## sequence counter \n",
    "        \n",
    "        # select a random start time for the next event, within the time range created earlier. \n",
    "        time_lapse_between_event = epoch_event_start_date+event_time_range\n",
    "        next_event_time =randint(epoch_event_start_date , time_lapse_between_event)\n",
    "\n",
    "        while (sequence_length <= max_sequence_length):\n",
    "            \n",
    "            # next we will pick the sub-intents from the call_matrix\n",
    "            \n",
    "            # make sure we do not exceed the max length of the interactions array\n",
    "            if sequence_length >= interaction_sequence_length:\n",
    "                event_sequence_index = interaction_sequence_length-1  \n",
    "            \n",
    "            else:\n",
    "                event_sequence_index = sequence_length\n",
    "                writer.writerow([customer_id, call_matrix[intent_id][event_sequence_index] ,next_event_time])\n",
    "            \n",
    "            sequence_length+=1\n",
    "            next_event_time += randint(min_time_between_events, max_time_between_events)\n",
    "\n",
    "        interactions_sequence_count+=1\n",
    "\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Describe the user dataset info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_df=pd.read_csv(users_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 11162 entries, 0 to 11161\n",
      "Data columns (total 6 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   USER_ID    11162 non-null  int64 \n",
      " 1   deposit    11162 non-null  object\n",
      " 2   age        11162 non-null  int64 \n",
      " 3   job        11162 non-null  object\n",
      " 4   marital    11162 non-null  object\n",
      " 5   education  11162 non-null  object\n",
      "dtypes: int64(2), object(4)\n",
      "memory usage: 523.3+ KB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(users_df.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USER_ID</th>\n",
       "      <th>deposit</th>\n",
       "      <th>age</th>\n",
       "      <th>job</th>\n",
       "      <th>marital</th>\n",
       "      <th>education</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30908</td>\n",
       "      <td>yes</td>\n",
       "      <td>59</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30909</td>\n",
       "      <td>yes</td>\n",
       "      <td>56</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30910</td>\n",
       "      <td>yes</td>\n",
       "      <td>41</td>\n",
       "      <td>technician</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30911</td>\n",
       "      <td>yes</td>\n",
       "      <td>55</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30912</td>\n",
       "      <td>yes</td>\n",
       "      <td>54</td>\n",
       "      <td>admin.</td>\n",
       "      <td>married</td>\n",
       "      <td>tertiary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   USER_ID deposit  age         job  marital  education\n",
       "0    30908     yes   59      admin.  married  secondary\n",
       "1    30909     yes   56      admin.  married  secondary\n",
       "2    30910     yes   41  technician  married  secondary\n",
       "3    30911     yes   55    services  married  secondary\n",
       "4    30912     yes   54      admin.  married   tertiary"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Describe the interactions dataset info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "interactions_df=pd.read_csv(interactions_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            USER_ID     TIMESTAMP\n",
      "count  29627.000000  2.962700e+04\n",
      "mean   36497.523408  1.620739e+09\n",
      "std     3164.050405  9.076235e+06\n",
      "min    30909.000000  1.604280e+09\n",
      "25%    33773.000000  1.612899e+09\n",
      "50%    36534.000000  1.620637e+09\n",
      "75%    39248.000000  1.628548e+09\n",
      "max    41905.000000  1.638956e+09\n"
     ]
    }
   ],
   "source": [
    "print(interactions_df.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>USER_ID</th>\n",
       "      <th>ITEM_ID</th>\n",
       "      <th>TIMESTAMP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36168</td>\n",
       "      <td>FD_NE</td>\n",
       "      <td>1618655080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36168</td>\n",
       "      <td>FD_NA</td>\n",
       "      <td>1618820839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36168</td>\n",
       "      <td>FD_AD</td>\n",
       "      <td>1618887060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36168</td>\n",
       "      <td>FD_PW</td>\n",
       "      <td>1619290920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41412</td>\n",
       "      <td>FD_NE</td>\n",
       "      <td>1625258874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>41412</td>\n",
       "      <td>FD_NA</td>\n",
       "      <td>1625780865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>41412</td>\n",
       "      <td>FD_AD</td>\n",
       "      <td>1625819144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>41412</td>\n",
       "      <td>FD_PW</td>\n",
       "      <td>1626146061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>41412</td>\n",
       "      <td>FD_CS</td>\n",
       "      <td>1626827324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>32235</td>\n",
       "      <td>CC_NE</td>\n",
       "      <td>1612652289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>32235</td>\n",
       "      <td>CC_CA</td>\n",
       "      <td>1613386518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>32235</td>\n",
       "      <td>CC_AA</td>\n",
       "      <td>1614003091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>32235</td>\n",
       "      <td>CC_LC</td>\n",
       "      <td>1615041415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>32235</td>\n",
       "      <td>CC_CS</td>\n",
       "      <td>1615559237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>32105</td>\n",
       "      <td>TA_NE</td>\n",
       "      <td>1634463036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>32105</td>\n",
       "      <td>TA_AS</td>\n",
       "      <td>1634702868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>32105</td>\n",
       "      <td>TA_AA</td>\n",
       "      <td>1634781939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>32105</td>\n",
       "      <td>TA_FN</td>\n",
       "      <td>1635584356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>39436</td>\n",
       "      <td>AC_NE</td>\n",
       "      <td>1631240620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>39436</td>\n",
       "      <td>AC_AS</td>\n",
       "      <td>1632254075</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    USER_ID ITEM_ID   TIMESTAMP\n",
       "0     36168   FD_NE  1618655080\n",
       "1     36168   FD_NA  1618820839\n",
       "2     36168   FD_AD  1618887060\n",
       "3     36168   FD_PW  1619290920\n",
       "4     41412   FD_NE  1625258874\n",
       "5     41412   FD_NA  1625780865\n",
       "6     41412   FD_AD  1625819144\n",
       "7     41412   FD_PW  1626146061\n",
       "8     41412   FD_CS  1626827324\n",
       "9     32235   CC_NE  1612652289\n",
       "10    32235   CC_CA  1613386518\n",
       "11    32235   CC_AA  1614003091\n",
       "12    32235   CC_LC  1615041415\n",
       "13    32235   CC_CS  1615559237\n",
       "14    32105   TA_NE  1634463036\n",
       "15    32105   TA_AS  1634702868\n",
       "16    32105   TA_AA  1634781939\n",
       "17    32105   TA_FN  1635584356\n",
       "18    39436   AC_NE  1631240620\n",
       "19    39436   AC_AS  1632254075"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "interactions_df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload data files to S3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a random string to be used as a suffix when naming the programatically created AWS artifacts such as S3 buckets and Personalize resources. This way subsequent runs of the notebook will not have conflict with a previously created resource with the same name. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l0mid\n"
     ]
    }
   ],
   "source": [
    "random_suffix = ''.join(np.random.choice(list(string.ascii_lowercase + string.digits), 5))\n",
    "print(random_suffix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored 'random_suffix' (str)\n"
     ]
    }
   ],
   "source": [
    "%store random_suffix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "make_bucket: personalize-connect-blog-data-l0mid\r\n"
     ]
    }
   ],
   "source": [
    "bucket = \"personalize-connect-blog-data-\"+random_suffix    \n",
    "!aws s3 mb s3://{bucket}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize = boto3.client(service_name='personalize')\n",
    "personalize_runtime = boto3.client(service_name='personalize-runtime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "boto3.Session().resource('s3').Bucket(bucket).Object(interactions_filename).upload_file(interactions_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "boto3.Session().resource('s3').Bucket(bucket).Object(users_filename).upload_file(users_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Personalize service artifacts using Amazon boto3 APIs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Personalize dataset schemas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "personalize-connect-blog-users-schema-l0mid\n"
     ]
    }
   ],
   "source": [
    "users_schema_name='personalize-connect-blog-users-schema-'+random_suffix\n",
    "interactions_schema_name='personalize-connect-blog-interactions-schema-'+random_suffix\n",
    "print(users_schema_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"schemaArn\": \"arn:aws:personalize:us-west-2:311381347798:schema/personalize-connect-blog-users-schema-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"9acfcd8d-bf0f-4ef6-a5ed-bb281a6ba93d\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:06:51 GMT\",\n",
      "      \"x-amzn-requestid\": \"9acfcd8d-bf0f-4ef6-a5ed-bb281a6ba93d\",\n",
      "      \"content-length\": \"109\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "users_schema = {\n",
    "    \"type\": \"record\",\n",
    "    \"name\": \"Users\",\n",
    "    \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    "    \"fields\": [\n",
    "        {\n",
    "            \"name\": \"USER_ID\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"deposit\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"age\",\n",
    "            \"type\": \"long\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"job\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"marital\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"education\",\n",
    "            \"type\": \"string\"\n",
    "        }\n",
    "    ],\n",
    "    \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_schema_response = personalize.create_schema(\n",
    "    name = users_schema_name,\n",
    "    schema = json.dumps(users_schema)\n",
    ")\n",
    "\n",
    "users_schema_arn = create_schema_response['schemaArn']\n",
    "print(json.dumps(create_schema_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"schemaArn\": \"arn:aws:personalize:us-west-2:311381347798:schema/personalize-connect-blog-interactions-schema-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"853aac6d-7efd-4a47-ac2e-84fefe284540\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:06:52 GMT\",\n",
      "      \"x-amzn-requestid\": \"853aac6d-7efd-4a47-ac2e-84fefe284540\",\n",
      "      \"content-length\": \"116\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "interactions_schema =  {\n",
    "  \"type\": \"record\",\n",
    "  \"name\": \"Interactions\",\n",
    "  \"namespace\": \"com.amazonaws.personalize.schema\",\n",
    "  \"fields\": [\n",
    "      {\n",
    "          \"name\": \"USER_ID\",\n",
    "          \"type\": \"string\"\n",
    "      },\n",
    "      {\n",
    "          \"name\": \"ITEM_ID\",\n",
    "          \"type\": \"string\"\n",
    "      },\n",
    "      {\n",
    "          \"name\": \"TIMESTAMP\",\n",
    "          \"type\": \"long\"\n",
    "      }\n",
    "  ],\n",
    "  \"version\": \"1.0\"\n",
    "}\n",
    "\n",
    "create_schema_response = personalize.create_schema(\n",
    "    name = interactions_schema_name,\n",
    "    schema = json.dumps(interactions_schema)\n",
    ")\n",
    "\n",
    "interactions_schema_arn = create_schema_response['schemaArn']\n",
    "\n",
    "print(json.dumps(create_schema_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create dataset group and dataset resources in Personalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create dataset group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetGroupArn\": \"arn:aws:personalize:us-west-2:311381347798:dataset-group/personalize-connect-blog-dataset-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"b6f44102-8ebe-433b-acdf-2df90da985de\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:06:51 GMT\",\n",
      "      \"x-amzn-requestid\": \"b6f44102-8ebe-433b-acdf-2df90da985de\",\n",
      "      \"content-length\": \"117\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_dataset_group_response = personalize.create_dataset_group(\n",
    "    name = \"personalize-connect-blog-dataset-\"+random_suffix\n",
    ")\n",
    "\n",
    "dataset_group_arn = create_dataset_group_response['datasetGroupArn']\n",
    "print(json.dumps(create_dataset_group_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check status of dataset group creation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetGroup: CREATE PENDING\n",
      "DatasetGroup: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_group_response = personalize.describe_dataset_group(\n",
    "        datasetGroupArn = dataset_group_arn\n",
    "    )\n",
    "    status = describe_dataset_group_response[\"datasetGroup\"][\"status\"]\n",
    "    print(\"DatasetGroup: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create INTERACTIONS dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetArn\": \"arn:aws:personalize:us-west-2:311381347798:dataset/personalize-connect-blog-dataset-l0mid/INTERACTIONS\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"9916af03-829a-4103-942f-2b6eb93bfdd7\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:07:52 GMT\",\n",
      "      \"x-amzn-requestid\": \"9916af03-829a-4103-942f-2b6eb93bfdd7\",\n",
      "      \"content-length\": \"119\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_type = \"INTERACTIONS\"\n",
    "create_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = interactions_schema_arn,\n",
    "    name = \"personalize-connect-blog-interactions-dataset-\"+random_suffix\n",
    ")\n",
    "\n",
    "interactions_dataset_arn = create_dataset_response['datasetArn']\n",
    "print(json.dumps(create_dataset_response, indent=2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check INTERACTIONS dateset creation status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: CREATE PENDING\n",
      "dataset: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_response = personalize.describe_dataset(\n",
    "        datasetArn = interactions_dataset_arn\n",
    "    )\n",
    "    status = describe_dataset_response[\"dataset\"][\"status\"]\n",
    "    print(\"dataset: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create USERS dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetArn\": \"arn:aws:personalize:us-west-2:311381347798:dataset/personalize-connect-blog-dataset-l0mid/USERS\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"0c711f1c-0468-47a2-948a-90511e501c3b\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:08:52 GMT\",\n",
      "      \"x-amzn-requestid\": \"0c711f1c-0468-47a2-948a-90511e501c3b\",\n",
      "      \"content-length\": \"112\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "dataset_type = \"USERS\"\n",
    "create_dataset_response = personalize.create_dataset(\n",
    "    datasetType = dataset_type,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    schemaArn = users_schema_arn,\n",
    "    name = \"personalize-connect-blog-users-dataset-\"+random_suffix\n",
    ")\n",
    "\n",
    "users_dataset_arn = create_dataset_response['datasetArn']\n",
    "print(json.dumps(create_dataset_response, indent=2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check USERS dateset creation status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: CREATE PENDING\n",
      "dataset: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_dataset_response = personalize.describe_dataset(\n",
    "        datasetArn = users_dataset_arn\n",
    "    )\n",
    "    status = describe_dataset_response[\"dataset\"][\"status\"]\n",
    "    print(\"dataset: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add S3 bucket policy for allowing access from personalize service"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code below adds necessary access policy to the S3 bucket so that AWS Personalize service can have read access to the bucket. This is necessary for Personalize service to read and import the training data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.client(\"s3\")\n",
    "\n",
    "policy = {\n",
    "    \"Version\": \"2012-10-17\",\n",
    "    \"Id\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "    \"Statement\": [\n",
    "        {\n",
    "            \"Sid\": \"PersonalizeS3BucketAccessPolicy\",\n",
    "            \"Effect\": \"Allow\",\n",
    "            \"Principal\": {\n",
    "                \"Service\": \"personalize.amazonaws.com\"\n",
    "            },\n",
    "            \"Action\": [\n",
    "                \"s3:GetObject\",\n",
    "                \"s3:ListBucket\"\n",
    "            ],\n",
    "            \"Resource\": [\n",
    "                \"arn:aws:s3:::{}\".format(bucket),\n",
    "                \"arn:aws:s3:::{}/*\".format(bucket)\n",
    "            ]\n",
    "        }\n",
    "    ]\n",
    "}\n",
    "\n",
    "s3.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the ARN for Personalize IAM Role "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Amazon Resource Names (ARNs) are unique IDs of resources. Our CloudFormation template had earlier created an IAM Service Role specifically with permissions to execute Amazon Personalize APIs, have access to S3 etc. This role was associated to this SageMaker notebook so that we can execute the Personalize related tasks from this SageMaker notebook. In order for Personalize service to import data from S3, this role provides the necessary permissions. We pass this role ARN as a reference to Personalize.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "sagemaker_session = sagemaker.Session()\n",
    "role_arn = sagemaker.get_execution_role(sagemaker_session=sagemaker_session)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'arn:aws:iam::311381347798:role/service-role/predict-ci-nb-CustomPersonalizeServiceRole-10HRPWAGXBH5V'"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "role_arn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create data import job in Personalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are long-running jobs run by the Personalize service to import the prepared datasets from specified S3 location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://personalize-connect-blog-data-l0mid/INTERACTIONS.csv\n",
      "arn:aws:personalize:us-west-2:311381347798:dataset/personalize-connect-blog-dataset-l0mid/INTERACTIONS\n"
     ]
    }
   ],
   "source": [
    "print(\"s3://{}/{}\".format(bucket, interactions_filename))\n",
    "print(interactions_dataset_arn)\n",
    "interactions_dataset_import_job_arn = 'arn:aws:personalize:us-west-2:311381347798:dataset-import-job/interactions-dataset-import-zhc0r'\n",
    "users_dataset_import_job_arn = 'arn:aws:personalize:us-west-2:311381347798:dataset-import-job/users-dataset-import-zhc0r'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetImportJobArn\": \"arn:aws:personalize:us-west-2:311381347798:dataset-import-job/interactions-dataset-import-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"ae00507c-44f8-41bb-98de-41ac57a21ad0\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:09:53 GMT\",\n",
      "      \"x-amzn-requestid\": \"ae00507c-44f8-41bb-98de-41ac57a21ad0\",\n",
      "      \"content-length\": \"121\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_interactions_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"interactions-dataset-import-\"+random_suffix,\n",
    "    datasetArn = interactions_dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, interactions_filename)\n",
    "        \n",
    "    }, roleArn = role_arn\n",
    ")\n",
    "\n",
    "interactions_dataset_import_job_arn = create_interactions_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_interactions_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"datasetImportJobArn\": \"arn:aws:personalize:us-west-2:311381347798:dataset-import-job/users-dataset-import-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"6a7fcba4-3458-4f6e-aaa8-22c91f1115c1\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:09:53 GMT\",\n",
      "      \"x-amzn-requestid\": \"6a7fcba4-3458-4f6e-aaa8-22c91f1115c1\",\n",
      "      \"content-length\": \"114\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_users_dataset_import_job_response = personalize.create_dataset_import_job(\n",
    "    jobName = \"users-dataset-import-\"+random_suffix,\n",
    "    datasetArn = users_dataset_arn,\n",
    "    dataSource = {\n",
    "        \"dataLocation\": \"s3://{}/{}\".format(bucket, users_filename)\n",
    "        \n",
    "    }, roleArn = role_arn\n",
    ")\n",
    "\n",
    "users_dataset_import_job_arn = create_users_dataset_import_job_response['datasetImportJobArn']\n",
    "print(json.dumps(create_users_dataset_import_job_response, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the status of the dataset creation jobs. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetImportJob: CREATE PENDING\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: CREATE IN_PROGRESS\n",
      "DatasetImportJob: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_users_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = users_dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = describe_users_dataset_import_job_response[\"datasetImportJob\"]\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetImportJob: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "        \n",
    "    interactions_dataset_import_job_response = personalize.describe_dataset_import_job(\n",
    "        datasetImportJobArn = interactions_dataset_import_job_arn\n",
    "    )\n",
    "    \n",
    "    dataset_import_job = interactions_dataset_import_job_response[\"datasetImportJob\"]\n",
    "#     print(dataset_import_job)\n",
    "    if \"latestDatasetImportJobRun\" not in dataset_import_job:\n",
    "        status = dataset_import_job[\"status\"]\n",
    "        print(\"DatasetImportJob: {}\".format(status))\n",
    "    else:\n",
    "        status = dataset_import_job[\"latestDatasetImportJobRun\"][\"status\"]\n",
    "        print(\"LatestDatasetImportJobRun: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a solution in Personalize "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the appropriate recipe for this use case. One can try with different options. Please refer to https://docs.aws.amazon.com/personalize/latest/dg/working-with-predefined-recipes.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arn:aws:personalize:::recipe/aws-hrnn\n",
      "arn:aws:personalize:::recipe/aws-hrnn-coldstart\n",
      "arn:aws:personalize:::recipe/aws-hrnn-metadata\n",
      "arn:aws:personalize:::recipe/aws-personalized-ranking\n",
      "arn:aws:personalize:::recipe/aws-popularity-count\n",
      "arn:aws:personalize:::recipe/aws-similar-items\n",
      "arn:aws:personalize:::recipe/aws-sims\n",
      "arn:aws:personalize:::recipe/aws-user-personalization\n"
     ]
    }
   ],
   "source": [
    "recipe_list = personalize.list_recipes()\n",
    "for recipe in recipe_list['recipes']:\n",
    "    print(recipe['recipeArn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe_arn = \"arn:aws:personalize:::recipe/aws-user-personalization\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionArn\": \"arn:aws:personalize:us-west-2:311381347798:solution/predict-ci-user-personalization-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"a032b8fa-e606-4b95-b1b6-2d45c0375bbc\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:14:54 GMT\",\n",
      "      \"x-amzn-requestid\": \"a032b8fa-e606-4b95-b1b6-2d45c0375bbc\",\n",
      "      \"content-length\": \"107\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_response = personalize.create_solution(\n",
    "    name='predict-ci-user-personalization-'+ random_suffix,\n",
    "    datasetGroupArn = dataset_group_arn,\n",
    "    recipeArn= recipe_arn\n",
    ")\n",
    "\n",
    "solution_arn = create_solution_response['solutionArn']\n",
    "print(json.dumps(create_solution_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"solutionVersionArn\": \"arn:aws:personalize:us-west-2:311381347798:solution/predict-ci-user-personalization-l0mid/a31e0210\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"aa5c0cd6-9e5b-486f-9947-293dac9d6691\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:14:54 GMT\",\n",
      "      \"x-amzn-requestid\": \"aa5c0cd6-9e5b-486f-9947-293dac9d6691\",\n",
      "      \"content-length\": \"123\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_solution_version_response = personalize.create_solution_version(\n",
    "    solutionArn = solution_arn,\n",
    "    trainingMode='FULL'\n",
    ")\n",
    "\n",
    "solution_version_arn = create_solution_version_response['solutionVersionArn']\n",
    "print(json.dumps(create_solution_version_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SolutionVersion: CREATE PENDING\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: CREATE IN_PROGRESS\n",
      "SolutionVersion: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_solution_version_response = personalize.describe_solution_version(\n",
    "        solutionVersionArn = solution_version_arn\n",
    "    )\n",
    "    status = describe_solution_version_response[\"solutionVersion\"][\"status\"]\n",
    "    print(\"SolutionVersion: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploy a campaign in Personalize for this solution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a list of Properties and Hyperparameters available for the user personalization recipe, please refer to: https://docs.aws.amazon.com/personalize/latest/dg/native-recipe-new-item-USER_PERSONALIZATION.html#bandit-hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"campaignArn\": \"arn:aws:personalize:us-west-2:311381347798:campaign/predict-ci-user-personalization-l0mid\",\n",
      "  \"ResponseMetadata\": {\n",
      "    \"RequestId\": \"927f7b78-c150-47bf-97c6-1de5bd6490bc\",\n",
      "    \"HTTPStatusCode\": 200,\n",
      "    \"HTTPHeaders\": {\n",
      "      \"content-type\": \"application/x-amz-json-1.1\",\n",
      "      \"date\": \"Wed, 17 Nov 2021 01:37:56 GMT\",\n",
      "      \"x-amzn-requestid\": \"927f7b78-c150-47bf-97c6-1de5bd6490bc\",\n",
      "      \"content-length\": \"107\",\n",
      "      \"connection\": \"keep-alive\"\n",
      "    },\n",
      "    \"RetryAttempts\": 0\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "create_campaign_response = personalize.create_campaign(\n",
    "    name = \"predict-ci-user-personalization-\"+random_suffix,\n",
    "    solutionVersionArn = solution_version_arn,\n",
    "    minProvisionedTPS = 2,\n",
    "    campaignConfig = {\"itemExplorationConfig\": {\"explorationItemAgeCutOff\": \"2000\"}}\n",
    ")\n",
    "\n",
    "campaign_arn = create_campaign_response['campaignArn']\n",
    "print(json.dumps(create_campaign_response, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Campaign: CREATE PENDING\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: CREATE IN_PROGRESS\n",
      "Campaign: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "status = None\n",
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_campaign_response = personalize.describe_campaign(\n",
    "        campaignArn = campaign_arn\n",
    "    )\n",
    "    status = describe_campaign_response[\"campaign\"][\"status\"]\n",
    "    print(\"Campaign: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Customer Intent Predicted "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update new interaction data in real-time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using personalize APIs for updating user interaction data in real-time. This is done using the put_events API offered by Amazon Personalize service."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'eventTrackerArn': 'arn:aws:personalize:us-west-2:311381347798:event-tracker/8c5eb3c9',\n",
       " 'trackingId': '9224e121-c78f-4f6b-aa63-fa3809e71a68',\n",
       " 'ResponseMetadata': {'RequestId': '778391b8-e4d0-4d49-b5db-adff1d47d7b2',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n",
       "   'date': 'Wed, 17 Nov 2021 01:48:57 GMT',\n",
       "   'x-amzn-requestid': '778391b8-e4d0-4d49-b5db-adff1d47d7b2',\n",
       "   'content-length': '139',\n",
       "   'connection': 'keep-alive'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "event_tracker = 'CallInentTracker'\n",
    "response = personalize.create_event_tracker(\n",
    "    name=event_tracker,\n",
    "    datasetGroupArn=dataset_group_arn\n",
    ")\n",
    "tracking_id = response['trackingId']\n",
    "tracking_arn= response ['eventTrackerArn']\n",
    "response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a random session ID for sending interaction events "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize_events = boto3.client(service_name='personalize-events')\n",
    "\n",
    "letters = string.ascii_lowercase\n",
    "session_id= ''.join(random.choice(letters) for i in range(10)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Record new events for an existing customer "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check status of the event tracker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eventTracker: CREATE PENDING\n",
      "eventTracker: ACTIVE\n"
     ]
    }
   ],
   "source": [
    "max_time = time.time() + 3*60*60 # 3 hours\n",
    "while time.time() < max_time:\n",
    "    describe_event_tracker_response = personalize.describe_event_tracker(\n",
    "        eventTrackerArn = tracking_arn\n",
    "    )\n",
    "    status = describe_event_tracker_response[\"eventTracker\"][\"status\"]\n",
    "    print(\"eventTracker: {}\".format(status))\n",
    "    \n",
    "    if status == \"ACTIVE\" or status == \"CREATE FAILED\":\n",
    "        break\n",
    "        \n",
    "    time.sleep(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize_events.put_events(\n",
    "    trackingId = tracking_id,\n",
    "    userId= '35739',\n",
    "    sessionId = session_id,\n",
    "    eventList = [{\n",
    "        'sentAt': int(time.time()-276480),\n",
    "        'eventType': 'call',\n",
    "        'properties': \"{\\\"itemId\\\": \\\"AC_NE\\\"}\"\n",
    "        },{\n",
    "        'sentAt': int(time.time()-86400),\n",
    "        'eventType': 'call',\n",
    "        'properties': \"{\\\"itemId\\\": \\\"AC_AS\\\"}\"\n",
    "        },{\n",
    "        'sentAt': int(time.time()-1000),\n",
    "        'eventType': 'call',\n",
    "        'properties': \"{\\\"itemId\\\": \\\"AC_PA\\\"}\"\n",
    "        }]\n",
    ")\n",
    "time.sleep(120) # allowing two mins for personalize service to update the recommendaion based on real-time events"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Customer intent prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A. High confidence score example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A deployed Personalized campaign offers Personalize runtime APIs for getting real-time personalized recommendations. In section of the notebook, we will use Boto APIs to interact with the Personalize runtime APIs. We would provide a customer's unique ID and receive that customers next intent predicted by Personalize. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommended items - score \n",
      "AC_AA 0.9439064\n",
      "CC_NE 0.0138092\n",
      "LI_NE 0.0085333\n",
      "HL_NE 0.007464\n",
      "TA_NE 0.0071827\n",
      "FD_NE 0.00682\n",
      "AC_NE 0.0066388\n",
      "AC_PA 0.000894\n",
      "TA_AA 0.0006131\n",
      "TA_FN 0.0005504\n",
      "HL_RP 0.0004061\n",
      "HL_CS 0.0003783\n",
      "AC_CS 0.0003213\n",
      "LI_AS 0.0003\n",
      "TA_AS 0.0001959\n",
      "CC_CA 0.0001781\n",
      "CC_LC 0.0001759\n",
      "AC_AS 0.0001737\n",
      "LI_CS 0.0001436\n",
      "CC_CS 0.0001373\n",
      "HL_DS 0.0001343\n",
      "FD_NA 0.0001288\n",
      "TA_CS 0.0001215\n",
      "CC_AA 0.0001116\n"
     ]
    }
   ],
   "source": [
    "response = personalize_runtime.get_recommendations(\n",
    "    campaignArn = campaign_arn,\n",
    "    userId = '35739' )\n",
    "\n",
    "print(\"Recommended items - score \")\n",
    "for item in response['itemList']:\n",
    "     print (item['itemId'], item['score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### B. Low confidence score example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This returns a low confidence score as the customer ID is new. Personalize does not have any contact history for this customer. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommended items - score \n",
      "HL_NE 0.2017641\n",
      "AC_NE 0.1764564\n",
      "FD_NE 0.1739267\n",
      "CC_NE 0.1654633\n",
      "TA_NE 0.1425011\n",
      "LI_NE 0.1273979\n",
      "HL_DS 0.0011574\n",
      "FD_CS 0.0007902\n",
      "CC_CS 0.0007501\n",
      "AC_AA 0.0007381\n",
      "TA_AA 0.0006833\n",
      "FD_PW 0.0006245\n",
      "HL_CS 0.0005332\n",
      "LI_CS 0.0005196\n",
      "LI_AS 0.0004616\n",
      "FD_AD 0.0004566\n",
      "HL_AS 0.0004515\n",
      "AC_CS 0.0004445\n",
      "TA_CS 0.0004393\n",
      "HL_RP 0.0004161\n",
      "CC_AA 0.0004146\n",
      "CC_LC 0.000384\n",
      "LI_SW 0.0003837\n",
      "LI_PA 0.0003789\n"
     ]
    }
   ],
   "source": [
    "response = personalize_runtime.get_recommendations(\n",
    "    campaignArn = campaign_arn,\n",
    "    userId = '89789' )\n",
    "\n",
    "print(\"Recommended items - score \")\n",
    "for item in response['itemList']:\n",
    "     print (item['itemId'], item['score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### C. Record  events for a new customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "personalize_events.put_events(\n",
    "    trackingId = tracking_id,\n",
    "    userId= '88889',\n",
    "    sessionId = session_id,\n",
    "    eventList = [{\n",
    "        'sentAt': int(time.time()-86400),\n",
    "        'eventType': 'call',\n",
    "        'properties': \"{\\\"itemId\\\": \\\"HL_NE\\\"}\"\n",
    "        }]\n",
    ")\n",
    "time.sleep(120) # allowing two mins for personalize service to update the recommendaion based on real-time events"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### D. start getting intent prediction in real-time for the new customer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommended items - scores\n",
      "HL_AS 0.9971374\n",
      "LI_NE 0.0005079\n",
      "HL_DS 0.0004015\n",
      "CC_NE 0.0002941\n",
      "HL_NE 0.0002761\n",
      "FD_NE 0.0002717\n",
      "AC_NE 0.0002577\n",
      "TA_NE 0.0002323\n",
      "TA_AA 9.24e-05\n",
      "CC_CS 6.93e-05\n",
      "FD_NA 5.61e-05\n",
      "HL_RP 5.49e-05\n",
      "AC_AS 4.17e-05\n",
      "FD_AD 3.24e-05\n",
      "FD_PW 3.11e-05\n",
      "AC_CS 2.7e-05\n",
      "HL_CS 2.21e-05\n",
      "CC_AA 1.91e-05\n",
      "FD_CS 1.66e-05\n",
      "CC_CA 1.43e-05\n",
      "LI_CS 1.33e-05\n",
      "TA_CS 1.31e-05\n",
      "TA_FN 1.19e-05\n",
      "LI_AS 8.1e-06\n"
     ]
    }
   ],
   "source": [
    "response = personalize_runtime.get_recommendations(\n",
    "    campaignArn = campaign_arn,\n",
    "    userId = '88889' )\n",
    "\n",
    "print(\"Recommended items - scores\")\n",
    "\n",
    "for item in response['itemList']:\n",
    "     print (item['itemId'], item['score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'arn:aws:personalize:us-west-2:311381347798:campaign/predict-ci-user-personalization-l0mid'"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "campaign_arn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'9224e121-c78f-4f6b-aa63-fa3809e71a68'"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tracking_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}