5.0 - Transfer learning with BERT models on likes prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "\n",
    "#import xgboost as xgb\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras import layers\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Activation\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "tqdm().pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this is the raw dataset\n",
    "#data16 = pd.read_csv(r'/Users/cairo/Google Drive/wechat data/016.csv', sep=',', error_bad_lines=False, index_col=False, dtype='unicode')\n",
    "#data16_sample = pd.read_csv(r'/Users/cairo/Google Drive/wechat data/data16_sample.csv', sep=',', error_bad_lines=False, index_col=False, dtype='unicode')\n",
    "\n",
    "df = pd.read_csv(r'/Users/cairo/Google Drive/wechat data/ad data/AntzbBackData10k.csv', sep=',', error_bad_lines=False, index_col=False, dtype='unicode')\n",
    "\n",
    "df=df[[\"account\", 'advertisement','title', \"content\", \"clicksCount\", \"orderNum\", \"originalFlag\", \"likeCount\", \"publicTime\"]]\n",
    "\n",
    "\n",
    "df = df[pd.notnull(df['title'])]\n",
    "df = df[pd.notnull(df['content'])]\n",
    "df = df[pd.notnull(df['advertisement'])]\n",
    "df = df[pd.notnull(df['likeCount'])]\n",
    "\n",
    "df[\"publicTime\"] = pd.to_datetime(df[\"publicTime\"])\n",
    "#df[\"publicTime\"] = dd[\"publicTime\"].dt.to_period('D')\n",
    "df[\"clicksCount\"] = df[\"clicksCount\"].astype(float)\n",
    "df[\"likeCount\"] = df[\"likeCount\"].astype(float)\n",
    "df[\"orderNum\"] = df[\"orderNum\"].astype(float)\n",
    "df[\"originalFlag\"] = df[\"originalFlag\"].astype(float)\n",
    "\n",
    "#topicdata = pd.read_csv(r'C:/Users/Junhao/Google Drive/wechat data/TopicOutcomeAll20Topic.csv', sep=',', error_bad_lines=False, index_col=False, dtype='unicode')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cutword1 = cutword.head(1000).wordsCut\n",
    "#cutword3 = cutword.head(1000).title  #i can also try to use only the article titles as bert input \n",
    "df[\"combined_content\"] = df.title + df.title + df.title + df.content\n",
    "\n",
    "cutword2 = df.combined_content.dropna()\n",
    "\n",
    "#cutword2 = df.head(500).content.dropna() #use 1000 article content from rawdata as bert input\n",
    "#I need to cut each article to the length of 100 characters because the tokenizer can only deal with a max length of 512\n",
    "#the computation takes too long if the kept article length is long\n",
    "\n",
    "cutword3 = []\n",
    "for i in cutword2:\n",
    "    dd = i[:512]\n",
    "    cutword3.append(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-63-4faa14ba2eef>:12: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  for sentence in tqdm(sentences):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1edbdd860ed4f659a2cd15b875b5dc5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9995 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "bert_model_name = 'bert-base-chinese'\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(bert_model_name)\n",
    "MAX_LEN = 512\n",
    "\n",
    "def tokenize_sentences(sentences, tokenizer, max_seq_len = 512):\n",
    "    tokenized_sentences = []\n",
    "\n",
    "    for sentence in tqdm(sentences):\n",
    "        tokenized_sentence = tokenizer.encode(\n",
    "                            sentence,                  # Sentence to encode.\n",
    "                            add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "                            max_length = max_seq_len,  # Truncate all sentences.\n",
    "                    )\n",
    "        \n",
    "        tokenized_sentences.append(tokenized_sentence)\n",
    "\n",
    "    return tokenized_sentences\n",
    "\n",
    "def create_attention_masks(tokenized_and_padded_sentences):\n",
    "    attention_masks = []\n",
    "\n",
    "    for sentence in tokenized_and_padded_sentences:\n",
    "        att_mask = [int(token_id > 0) for token_id in sentence]\n",
    "        attention_masks.append(att_mask)\n",
    "\n",
    "    return np.asarray(attention_masks)\n",
    "\n",
    "input_ids = tokenize_sentences(cutword3, tokenizer, MAX_LEN)\n",
    "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", value=0, truncating=\"post\", padding=\"post\")\n",
    "attention_masks = create_attention_masks(input_ids)\n",
    "#attention_masks = torch.tensor(attention_masks)\n",
    "#attention_masks = attention_masks.tolist()\n",
    "#attention_masks.to_tensor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#likeCount = df.head(500).likeCount #use likecount as dependent variable \n",
    "\n",
    "likeCount = df.likeCount\n",
    "\n",
    "#df = df[[\"likeCount\", \"combined_content\"]].head(500)\n",
    "\n",
    "df = df[[\"likeCount\", \"combined_content\"]]\n",
    "\n",
    "#removeinds = pd.isnull(df).any(1).to_numpy().nonzero()[0]\n",
    "\n",
    "#likeCount = [i for j, i in enumerate(likeCount) if j not in removeinds] # remove the two likecount where content is empty\n",
    "#likeCount = [float(i) for i in likeCount]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9997"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(likeCount)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create training and validation set\n",
    "\n",
    "type_ids = np.zeros((len(likeCount), 512))\n",
    "\n",
    "attention_val = attention_masks[-int(0.2 * len(likeCount)):]\n",
    "input_ids_val = input_ids[-int(0.2 * len(likeCount)):]\n",
    "type_ids_val = type_ids[-int(0.2 * len(likeCount)):]\n",
    "\n",
    "attention_train = attention_masks[:-int(0.2 * len(likeCount))]\n",
    "input_ids_train = input_ids[:-int(0.2 * len(likeCount))]\n",
    "type_ids_train = type_ids[:-int(0.2 * len(likeCount))]\n",
    "\n",
    "\n",
    "x_train = [input_ids_train, type_ids_train, attention_train]\n",
    "x_val = [input_ids_val, type_ids_val, attention_val]\n",
    "\n",
    "y_val = np.asarray(likeCount[-int(0.2 * len(likeCount)):])\n",
    "y_train = np.asarray(likeCount[:-int(0.2 * len(likeCount))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  101,   711,   784,   720,  5739,  4909,  4518,  1914,  2168,\n",
       "        1555,  8024,  3791,  4909,  4518,  1316,  2768,   749,  7484,\n",
       "        1462,  2157,  4638,  1921,  1828,  8043,   711,   784,   720,\n",
       "        5739,  4909,  4518,  1914,  2168,  1555,  8024,  3791,  4909,\n",
       "        4518,  1316,  2768,   749,  7484,  1462,  2157,  4638,  1921,\n",
       "        1828,  8043,   711,   784,   720,  5739,  4909,  4518,  1914,\n",
       "        2168,  1555,  8024,  3791,  4909,  4518,  1316,  2768,   749,\n",
       "        7484,  1462,  2157,  4638,  1921,  1828,  8043,   677,  3862,\n",
       "         697,  1920,  4909,  4518,  8038,  4636,  2399,  1184,  4638,\n",
       "         671,  1767,   100,  1920,  3124,  2424,   100,   680,   100,\n",
       "        2207,  3124,  2424,   100,  4638,  1169,  2428,  4993,   751,\n",
       "         511,  3341,  3975,  8038,   782,  4868,  1066,  1939,  8020,\n",
       "         100,  8038,  8228,  8291, 11017,  8797,  9450,  8778,  8021,\n",
       "         868,  5442,  8038,  3330,  1157,   122,   120,   127,   697,\n",
       "         702,  4909,  4518, 10116,  8156,  2399,  8024,  3791,  1744,\n",
       "        7674,   818,  7566,   752,  3130,   860,  2225,  2828,  3791,\n",
       "        1744,  7566,   752,  7667,  2128,  5390,  1762,   677,  3862,\n",
       "        5439,  1814,  1334,  1469,  5739,  4909,  4518,   722,  7313,\n",
       "        4638,   671,  2792,  3136,  1828,  7027,   511,  7390,  1400,\n",
       "        8024,   800,   809,  3791,  1744,  1555,   782,  6206,  3724,\n",
       "         743,  1765,  2456,  2791,  2094,   711,  4507,  8024,  2990,\n",
       "        1139,  6392,  4989,  3791,  1744,  4909,  4518,  4638,  6206,\n",
       "        3724,   511,   677,  3862,  6887,  1378,  2151,  2710,   719,\n",
       "        2400,   679,  6230,  2533,  6821,  3221,   702,   784,   720,\n",
       "        1920,   752,  1036,  8024,   852,   711,   749,  2208,  2685,\n",
       "        7937,  4172,  8024,  6820,  3221,  2456,  6379,  3130,   860,\n",
       "        2225,  2828,  3791,  1744,  7566,   752,  7667,  2456,  1168,\n",
       "        5739,  1744,  4909,  4518,   511,  6821,   855,  3791,  1744,\n",
       "        1912,   769,  2135,  7561,  3198,  2697,  1168,  5404,  6802,\n",
       "        8038,   100,  2769,  1828,  1828,  3791,  1065,  6205,  1912,\n",
       "         769,  2135,  8024,   872,  2233,  4197,  6375,  2769,   743,\n",
       "        2247,   754,  5739,  1744,  4638,  1759,  1765,  8043,   100,\n",
       "        7937,  4172,  3188,  4197,  6719,   679,  2458,  8024,  6929,\n",
       "        2218,  1372,  3300,  4500,  5439,  1215,  3791,   749,   511,\n",
       "        2151,  2710,   719,  2199,   677,  3862,  5439,  1814,  1469,\n",
       "        3817,  3814,  3853,   722,  7313,  6818,  1283,   774,  4638,\n",
       "        1759,  1765,  1153,  2768,   749,  3791,  1744,  4909,  4518,\n",
       "         511,   800,  2400,   679,  6230,  2533,  6821,  3300,   784,\n",
       "         720,   679,  1980,  8024,  2902,  3926,  3124,  2424,  4638,\n",
       "        3124,  5032,  8024,  3188,  4197,  6628,   679,  6624,  6821,\n",
       "        2376,  6037,  2094,  8024,  4415,   738,  6382,   679,  6858,\n",
       "        8024,   679,  1963,  1153,   671,  1779,  1765,  3175,  8024,\n",
       "        6375,   800,   812,  5632,  2769,  5052,  4415,   511,  5445,\n",
       "         684,  8024,  4909,  4518,  4638,  2692,  2590,  3221,  1038,\n",
       "        6387,   872,   857,   678,  3341,  8024,   852,  6821,   763,\n",
       "        1765,  3175,  4385,  1762,  1372,  3300,   671,   763,  4788,\n",
       "        5770,  2238,  8024,  1071,   865,  6963,  3221,  3300,   712,\n",
       "         782,  4638,  1093,  4506,  8024,   872,  2582,   720,  3833,\n",
       "         678,  1343,  8024,  2769,  2218,   679,  5052,   749,   511,\n",
       "         791,  1921,  6158,  6371,   711,  3221,   700,  3326,  6802,\n",
       "        1744,  4638,   100,  3780,  1912,  3791,  3326,   100,  8024,\n",
       "        1762,  2496,  3198,  4692,  3341,  8024,   788,   788,  3221,\n",
       "         671,  4905,   837,  5320,  4638,   510,  7151,  2190,  6037,\n",
       "        1929,  4638,   100,  5396,   100,   100,  3124,  5032,   511,\n",
       "         794,  3634,  8024,   677,  3862,  3300,   749,   697,   702,\n",
       "         671,  1298,   671,  1266,   697,   702,  3683,  6943,  4638,\n",
       "         102,     0,     0,     0,     0,     0,     0,     0])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train[1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at bert-base-chinese were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFBertModel were initialized from the model checkpoint at bert-base-chinese.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
      "Model: \"model_7\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_22 (InputLayer)           [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_24 (InputLayer)           [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_23 (InputLayer)           [(None, 512)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tf_bert_model_7 (TFBertModel)   TFBaseModelOutputWit 102267648   input_22[0][0]                   \n",
      "                                                                 input_24[0][0]                   \n",
      "                                                                 input_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "start_logits0 (Dense)           (None, 512, 1)       768         tf_bert_model_7[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 102,268,416\n",
      "Trainable params: 768\n",
      "Non-trainable params: 102,267,648\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from transformers import TFBertForSequenceClassification, TFBertModel, BertConfig, TFBertForPreTraining\n",
    "\n",
    "max_len = 512\n",
    "\n",
    "def create_model():\n",
    "    ## BERT encoder\n",
    "    encoder = TFBertModel.from_pretrained(\"bert-base-chinese\")\n",
    "\n",
    "    ## QA Model\n",
    "    input_ids = layers.Input(shape=(max_len,), dtype=tf.int32)\n",
    "    token_type_ids = layers.Input(shape=(max_len,), dtype=tf.int32)\n",
    "    attention_mask = layers.Input(shape=(max_len,), dtype=tf.int32)\n",
    "    embedding = encoder(\n",
    "        input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)[0]\n",
    " \n",
    "    start_logits0 = layers.Dense(1, name=\"start_logits0\", use_bias=False)(embedding)\n",
    "    #start_logits1 = (Activation('sigmoid'))(start_logits0)\n",
    "    #start_logits = layers.Flatten()(start_logits0)\n",
    "\n",
    "    #end_logits = layers.Dense(1, name=\"end_logit\", use_bias=False)(embedding)\n",
    "    #end_logits = layers.Flatten()(end_logits)\n",
    "\n",
    "    #start_probs = layers.Activation(keras.activations.softmax)(start_logits)\n",
    "    #end_probs = layers.Activation(keras.activations.softmax)(end_logits)\n",
    "\n",
    "    model = keras.Model(\n",
    "        inputs=[input_ids, token_type_ids, attention_mask],\n",
    "        outputs=[start_logits0],\n",
    "    )\n",
    "    #loss = keras.losses.MSE(y_pred= model.outputs, y_true = y_val)\n",
    "    #loss = keras.losses.SparseCategoricalCrossentropy(from_logits=False)\n",
    "    optimizer = keras.optimizers.Adam(lr=5e-5)\n",
    "    \n",
    "    model.layers[3].trainable = False  #make the bert layer not trainable\n",
    "    model.compile(optimizer=optimizer, loss=\"mean_squared_error\", metrics = tf.keras.metrics.MeanSquaredError(name='mse'))\n",
    "    return model\n",
    "\n",
    "mymodel = create_model()\n",
    "mymodel.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "125/125 [==============================] - ETA: 0s - loss: 1002122.5000 - mse: 1002122.5000  WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
      "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
      "125/125 [==============================] - 11818s 94s/step - loss: 1002122.5000 - mse: 1002122.5000 - val_loss: 811935.0625 - val_mse: 811935.0625\n",
      "Epoch 2/3\n",
      "125/125 [==============================] - 10984s 88s/step - loss: 1001569.8125 - mse: 1001569.8125 - val_loss: 811322.3125 - val_mse: 811322.3125\n",
      "Epoch 3/3\n",
      "125/125 [==============================] - 10839s 87s/step - loss: 1001024.8750 - mse: 1001024.8750 - val_loss: 810718.6875 - val_mse: 810718.6875\n"
     ]
    }
   ],
   "source": [
    "history = mymodel.fit(x_train, y_train, batch_size=64, epochs=3, validation_data=(x_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': [1002122.5, 1001569.8125, 1001024.875],\n",
       " 'mse': [1002122.5, 1001569.8125, 1001024.875],\n",
       " 'val_loss': [811935.0625, 811322.3125, 810718.6875],\n",
       " 'val_mse': [811935.0625, 811322.3125, 810718.6875]}"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}