train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "454a41dd-f032-4364-9771-d21c191eb175",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import BertForSequenceClassification\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from degree_inference.cah_data import CAHData\n",
    "from degree_inference.predict import predict\n",
    "from degree_inference.train import train\n",
    "\n",
    "load_dotenv() \n",
    "\n",
    "from google.cloud import bigquery\n",
    "\n",
    "%load_ext google.cloud.bigquery"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32ef18d3-ffb4-4c50-a118-69953c595923",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = CAHData(include_ilr=True, include_gpt_inferences=False, augment=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86867561-419e-4a8c-8dd2-0ad3d155dbca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "counts = c.df['label'].value_counts()\n",
    "plt.hist(counts, bins=15)\n",
    "plt.title(\"Distibution of training examples\")\n",
    "plt.xlabel(\"# of examples\")\n",
    "plt.ylabel(\"CAH3 codes with this many examples\")\n",
    "# counts, bins = np.histogram(counts)\n",
    "# plt.stairs(counts, bins)\n",
    "# counts.plot(kind='bar').set(xlabel=None)\n",
    "# ax = plt.gca()\n",
    "# ax.get_xaxis().set_visible(False)\n",
    "# plt.title(\"Frequency of CAH3 codes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebb94eaf-abd8-4cb6-8d1a-8d9375e1016e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bigquery degree_name_to_hecos\n",
    "SELECT\n",
    "  LOWER(cq.subject) AS text,\n",
    "  cq.degree_subject_cah_l3 as label\n",
    "FROM\n",
    "  `rugged-abacus-218110.dataform_ABS_2_dev.application_choice_details`\n",
    "LEFT JOIN\n",
    "  UNNEST(candidate_qualifications) AS cq\n",
    "LEFT JOIN `rugged-abacus-218110.dfe_reference_data.cah_categories_l3_v2` AS cah_codes ON cah_codes.id = degree_subject_cah_l3\n",
    "WHERE degree_level IS NOT NULL AND degree_level !='unknown'\n",
    "AND degree_subject_cah_l3 IS NOT NULL\n",
    "GROUP BY \n",
    "text,\n",
    "label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb9f54ff-de82-4dc0-98da-7662d9862b83",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = CAHData(include_ilr=False, augment=False, include_gpt_inferences=False)\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(data.df['label'].unique()))\n",
    "mps_device = torch.device(\"mps\")\n",
    "model.to(mps_device)\n",
    "\n",
    "trainer = train(model,data,epochs=30,comment=\"cah\")\n",
    "\n",
    "model.save_pretrained(\"./models/cah\")\n",
    "\n",
    "data = CAHData(include_ilr=True, augment=False, include_gpt_inferences=False)\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(data.df['label'].unique()))\n",
    "mps_device = torch.device(\"mps\")\n",
    "model.to(mps_device)\n",
    "\n",
    "trainer = train(model,data,epochs=16,comment=\"cah-ilr\")\n",
    "model.save_pretrained(\"./models/cah-ilr\")\n",
    "# model.to(\"cpu\")\n",
    "\n",
    "data = CAHData(include_ilr=True, augment=False, include_gpt_inferences=True)\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(data.df['label'].unique()))\n",
    "mps_device = torch.device(\"mps\")\n",
    "model.to(mps_device)\n",
    "\n",
    "trainer = train(model,data,epochs=12,comment=\"cah-ilr-gpt\")\n",
    "\n",
    "model.save_pretrained(\"./models/cah-ilr-gpt\")\n",
    "\n",
    "data = CAHData(include_ilr=True, augment=True, include_gpt_inferences=True)\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=len(data.df['label'].unique()))\n",
    "mps_device = torch.device(\"mps\")\n",
    "model.to(mps_device)\n",
    "\n",
    "trainer = train(model,data,epochs=12,comment=\"cah-ilr-gpt-augmented\")\n",
    "\n",
    "model.save_pretrained(\"./models/cah-ilr-gpt-augmented\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c3dedf0-0191-4fe1-ac60-162b1bfd3a7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "out = predict(data, model,list(degree_name_to_hecos['text']))\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a31e7783-e540-4291-a3e1-95d5b62f250e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"./models/cah-ilr-gpt-augmented\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ea2d9f5-cd7a-4a73-b607-4fced868d361",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"./cah-ilr-gpt-augmented\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8b7ad1a-fc48-4b2f-916f-1d12aa05a1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.df['label'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85e85f67-552e-4b74-9fba-d48f599f97a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bigquery unlabelled\n",
    "SELECT\n",
    "        cq.subject AS degree_subject,\n",
    "        cq.degree_subject_cah_l3,\n",
    "        cah_codes.name AS cah_category_name\n",
    "    FROM\n",
    "        `rugged-abacus-218110.dataform_ABS_2_dev.application_choice_details`\n",
    "    LEFT JOIN\n",
    "        UNNEST(candidate_qualifications) AS cq\n",
    "    LEFT JOIN `rugged-abacus-218110.dfe_reference_data.cah_categories_l3_v2` AS cah_codes ON cah_codes.id = degree_subject_cah_l3\n",
    "    WHERE degree_level IS NOT NULL AND degree_level !='unknown'\n",
    "    AND nationality_group = \"British\"\n",
    "    AND degree_subject_cah_l3 IS NULL\n",
    "    GROUP BY\n",
    "        degree_subject,\n",
    "        degree_subject_cah_l3,\n",
    "        cah_category_name\n",
    "    ORDER BY RAND()\n",
    "    LIMIT 100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acd7c309-5b43-4b6c-b135-bda77738332c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inference\n",
    "\n",
    "# load pretrained model\n",
    "model = BertForSequenceClassification.from_pretrained('./30-epoch-gpt2-ilr-augmented-1e-5/', num_labels=len(data.df['label'].unique()))\n",
    "model.to(\"cpu\")\n",
    "\n",
    "out = predict(data, model,list(unlabelled.degree_subject))\n",
    "for row in out:\n",
    "    print(row)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "transformers"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}