fda_exploration.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"data/product.txt\", sep=\"\\t\", encoding=\"unicode_escape\")\n",
    "#filter on single word in prescription category\n",
    "df2 = df[(df['PROPRIETARYNAME'].str.count(' ') == 0) & (df['PRODUCTTYPENAME'] == 'HUMAN PRESCRIPTION DRUG')]\n",
    "#convert to upper and remove any name with non-alpha chars\n",
    "df2.loc[:, 'PROPRIETARYNAME'] = df2.loc[:, 'PROPRIETARYNAME'].str.upper().replace('[^A-Z]', '', regex=True)\n",
    "#drop dups\n",
    "unique_drugs = df2.drop_duplicates(subset=['PROPRIETARYNAME'])\n",
    "#save\n",
    "unique_drugs['PROPRIETARYNAME'].to_csv(\"data/drug_names.txt\", header=False, index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TF imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.keras.optimizers import rmsprop_v2\n",
    "from tensorflow.python.keras.callbacks import LambdaCallback\n",
    "from tensorflow.python.keras.models import Sequential\n",
    "from tensorflow.python.keras.layers import Dense\n",
    "from tensorflow.python.keras.layers import LSTM\n",
    "import numpy as np"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reading data/prep vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/drug_names.txt\", encoding='utf-8') as f:\n",
    "    text = f.read().lower()\n",
    "\n",
    "print('num chars: ', len(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = sorted(list(set(text)))\n",
    "print(len(vocab))\n",
    "print(vocab)\n",
    "\n",
    "lines = text.split('\\n')\n",
    "lines = [line for line in lines if len(line) != 0]\n",
    "print(\"total lines: \", len(lines))\n",
    "max_length = len(max(lines, key=len))\n",
    "\n",
    "char_ind = dict((c, i) for i, c in enumerate(vocab))\n",
    "ind_char = dict((i, c) for i, c in enumerate(vocab))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "substrings = []\n",
    "target_char = []\n",
    "\n",
    "for line in lines:\n",
    "    # pre-padding with zeros\n",
    "    s = (max_length - len(line))*'0' + line\n",
    "    substrings.append(s)\n",
    "    target_char.append('\\n')\n",
    "    for it,j in enumerate(line):\n",
    "        if (it >= len(line)-1):\n",
    "            continue\n",
    "        s = (max_length - len(line[:-1-it]))*'0' + line[:-1-it]\n",
    "        substrings.append(s)\n",
    "        target_char.append(line[-1-it])\n",
    "\n",
    "print(len(substrings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.zeros((len(substrings), max_length, len(vocab)), dtype=float)\n",
    "Y = np.zeros((len(substrings), len(vocab)), dtype=float)\n",
    "for i, substr in enumerate(substrings):\n",
    "    for t, char in enumerate(substr):\n",
    "        if char != '0':\n",
    "            X[i, t, char_ind[char]] = 1\n",
    "    Y[i, char_ind[target_char[i]]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((X, Y))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Batch prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "BUFFER_SIZE = 10000\n",
    "\n",
    "dataset = (\n",
    "    dataset\n",
    "    .shuffle(BUFFER_SIZE)\n",
    "    .batch(BATCH_SIZE, drop_remainder=True)\n",
    "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create the model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### String lookup functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ids_from_chars = tf.keras.layers.StringLookup(\n",
    "    num_oov_indices=0, vocabulary=list(vocab), mask_token=None)\n",
    "\n",
    "chars_from_ids = tf.keras.layers.StringLookup(\n",
    "    num_oov_indices=0, vocabulary=ids_from_chars.get_vocabulary(), invert=True, mask_token=None)\n",
    "\n",
    "one_hot_chars = tf.keras.layers.StringLookup(\n",
    "    num_oov_indices=0, vocabulary=list(vocab), mask_token='0', output_mode='one_hot')\n",
    "\n",
    "def text_from_ids(ids):\n",
    "  return tf.strings.reduce_join(chars_from_ids(ids), axis=-1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Class definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DrugNameModel(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, chars_from_ids, lstm_units, max_length):\n",
    "        super().__init__(self)\n",
    "        self.lstm = tf.keras.layers.LSTM(lstm_units,\n",
    "                                         input_shape=(1, max_length, vocab_size))\n",
    "        self.dense = tf.keras.layers.Dense(vocab_size, \n",
    "                                           activation='softmax')\n",
    "        self.vocab_size = vocab_size\n",
    "        self.chars_from_ids = chars_from_ids\n",
    "\n",
    "    @tf.function\n",
    "    def __call__(self, inputs, training=False):\n",
    "        x = inputs\n",
    "        x = self.lstm(x, training=training)\n",
    "        x = self.dense(x, training=training)\n",
    "        return x"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(vocab)\n",
    "lstm_units = 64\n",
    "\n",
    "model = DrugNameModel(\n",
    "            max_length=max_length,\n",
    "            vocab_size=vocab_size,\n",
    "            lstm_units=lstm_units,\n",
    "            chars_from_ids=chars_from_ids)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Test untrained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for samp_X, samp_Y in dataset.take(1):\n",
    "    preds = model(inputs=samp_X, training=False)\n",
    "    \n",
    "print(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_ids = tf.random.categorical(preds, num_samples=1)\n",
    "sampled_ids = tf.squeeze(sampled_ids, axis=-1)\n",
    "sampled_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Next Char Predictions:\\n\", text_from_ids(sampled_ids))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Attatch optimizer, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.losses import CategoricalCrossentropy\n",
    "from tensorflow.keras.optimizers import RMSprop\n",
    "loss = CategoricalCrossentropy()\n",
    "opt = RMSprop(learning_rate=0.005)\n",
    "model.compile(optimizer=opt, loss=loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_mean_loss = loss(samp_Y, preds)\n",
    "print(\"prediction shape: \", batch_mean_loss.shape)\n",
    "print(\"mean loss: \", batch_mean_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sanity check--the exponential mean loss should be approx equal to vocabulary size (~27)\n",
    "tf.exp(batch_mean_loss)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(dataset, \n",
    "                    epochs=8, \n",
    "                    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.saved_model.save(model, 'drug_model')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SavedModel API syntax"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Low level api format\n",
    "save: `tf.saved_model.save(model, 'model_name')`  \n",
    "load: `model = tf.saved_model.load('model_name')`\n",
    "\n",
    "##### High level api format\n",
    "save: `model_subclassed.save('model_name')`  \n",
    "load: `model = tf.keras.models.load_model('model_name')`"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Name generator subclass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OneName(tf.keras.Model):\n",
    "    def __init__(self, model, one_hot_chars, chars_from_ids):\n",
    "        super().__init__()\n",
    "        self.model = model\n",
    "        self.one_hot_chars = one_hot_chars\n",
    "        self.chars_from_ids = chars_from_ids\n",
    "\n",
    "    def vectorize_input(self, curr_string, i):\n",
    "        split = tf.strings.unicode_split(curr_string, 'UTF-8')\n",
    "        one_hot = one_hot_chars(split)\n",
    "        paddings = ([31 - i, 0], [0, 0])\n",
    "        x_pad = tf.pad(one_hot, paddings, \"CONSTANT\")\n",
    "        one_hot_x = tf.reshape(x_pad, [1, 31, 27])\n",
    "        return one_hot_x\n",
    "    \n",
    "    @tf.function(input_signature=[tf.TensorSpec(shape=(), dtype=tf.string)])\n",
    "    def generate_word(self, inputs):\n",
    "        name = inputs\n",
    "        if (tf.strings.length(name) == 0):\n",
    "            rand_int = tf.random.uniform(shape=[1], dtype=tf.int32, minval=1, maxval=27)[0]\n",
    "            name += chars_from_ids(rand_int)\n",
    "            \n",
    "        length = tf.strings.length(name)\n",
    "        for i in tf.range(length, 25):\n",
    "            x = self.vectorize_input(name, i)\n",
    "            y = self.model(x, training=False)\n",
    "            y = tf.squeeze(y)\n",
    "            samples = tf.math.top_k(y, k=3)\n",
    "            samples = samples.indices\n",
    "            id = tf.random.shuffle(samples)[0]\n",
    "            x_char = chars_from_ids(id)\n",
    "            if (x_char == '\\n'):\n",
    "                break\n",
    "            name += x_char\n",
    "        return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_generator = OneName(model, one_hot_chars, chars_from_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.saved_model.save(name_generator, 'drug_generator')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Develop Drugs!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator = tf.saved_model.load('drug_generator')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_drugs = set()\n",
    "prefix = ''\n",
    "for i in range(500):\n",
    "    name = generator.generate_word(prefix).numpy().decode('utf-8')\n",
    "    if (len(name) >= 12): continue\n",
    "    new_drugs.add(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('fake_drugs.txt', 'w') as f:\n",
    "    for s in new_drugs:\n",
    "        f.write(f'{s}\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}