src/3_map_topics_to_categories.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-06-04T09:58:00.375095Z",
     "start_time": "2024-06-04T09:58:00.336300Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "from dataset.dataset_utils import *\n",
    "from utils.runctx import RunCtx"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-04T09:58:01.456611Z",
     "start_time": "2024-06-04T09:58:00.767263Z"
    }
   },
   "id": "1bcbdae191b51514",
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "run_ctx = RunCtx(\"<YOUR_TOKEN>\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-04T09:58:01.490587Z",
     "start_time": "2024-06-04T09:58:01.458966Z"
    }
   },
   "id": "7439abfe4589c6dc",
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "all_topic_names = run_ctx.state.load(\"all_topic_names.json\") \n",
    "results_v1_1 = run_ctx.state.load(\"categorization_results_v2.json\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-04T09:58:01.544494Z",
     "start_time": "2024-06-04T09:58:01.494164Z"
    }
   },
   "id": "3eb6663046183e74",
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "from pipeline.pipeline import async_topic_mapping_pipeline\n",
    "\n",
    "async def async_run_pipeline(model, prompt_fn, dataset, output_name):\n",
    "    \"\"\"\n",
    "    This function runs a specified prompt and model with topic_mapping_pipeline against the batch of topics\n",
    "    :param model: model to use \n",
    "    :param prompt_fn: prompt to use\n",
    "    :param dataset: dataset to run on\n",
    "    :param output_name: base name for results files\n",
    "    \"\"\"\n",
    "    pipeline_fn = async_topic_mapping_pipeline(model, results_v1_1, prompt_fn)\n",
    "    results = await run_ctx.run_async(pipeline_fn, dataset)\n",
    "    run_ctx.state.save(results, output_name + \".json\", overwrite=True)\n",
    "    return results"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-04T09:58:51.400463Z",
     "start_time": "2024-06-04T09:58:51.362554Z"
    }
   },
   "id": "ca3f4b0a074a702d",
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "from prompts.prompts import map_category_v1\n",
    "topic_mapping_all = await async_run_pipeline(run_ctx.model_gpt3_5, map_category_v1, all_topic_names, \"topic_mapping_all\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-04T09:59:22.903066Z",
     "start_time": "2024-06-04T09:58:52.214633Z"
    }
   },
   "id": "4041560d53358fab",
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "from dataset.dataset_utils import build_topic_sentiments_df\n",
    "\n",
    "all_reviews_df = load_reviews(run_ctx.data_dir + \"/us_reviews.csv\")\n",
    "topic_extraction_results = run_ctx.state.load(\"results_all_gpt_4o_mini_final.json\")\n",
    " \n",
    "topic_sentiments_df = build_topic_sentiments_df(\n",
    "    all_reviews_df,\n",
    "    topic_extraction_results, \n",
    "    topic_mapping_all\n",
    ")\n",
    "topic_sentiments_df.to_csv(run_ctx.data_dir + \"/topic_sentiments.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-03T10:20:53.409513Z",
     "start_time": "2024-06-03T10:20:53.061979Z"
    }
   },
   "id": "48f5f06feaae1a3e",
   "execution_count": 31
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}