data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7f76c2f4-9ffe-4e2f-b4a7-2ae2368dadc4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import sys\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d5734c9e-f696-478f-82d1-895c671c14cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def process(example, step: int):\n",
    "    function_key = \"Function: \"\n",
    "    observe_key = \"Observation: \"\n",
    "    answer_key = \"Final Answer: \"\n",
    "    new_data = {}\n",
    "    if len(example) <= 3:\n",
    "        new_data[\"instruction\"] = str(example[1][\"content\"])\n",
    "        new_data[\"input\"] = \"\"\n",
    "        new_data[\"output\"] = answer_key + str(example[2][\"content\"])\n",
    "    else:\n",
    "        name = example[2][\"function_call\"][\"name\"]\n",
    "        args = example[2][\"function_call\"][\"arguments\"]\n",
    "        args = json.loads(str(example[2][\"function_call\"][\"arguments\"]))\n",
    "        for (key, val) in args.items():\n",
    "            function_key += name + '(' + key + '=' + '\"' + val + '\"' + ')'\n",
    "        \n",
    "        function = function_key\n",
    "        observe = observe_key + str(example[3][\"content\"])\n",
    "        answer = answer_key + str(example[4][\"content\"])\n",
    "        \n",
    "        if step == 0:\n",
    "            new_data[\"instruction\"] = str(example[1][\"content\"])\n",
    "            new_data[\"input\"] = \"\"\n",
    "            new_data[\"output\"] = f\"{function}\\nObservation: \"\n",
    "        else:\n",
    "            new_data[\"instruction\"] = str(example[1][\"content\"])\n",
    "            new_data[\"input\"] = function + '\\n' + observe + '\\n'\n",
    "            new_data[\"output\"] = answer\n",
    "            \n",
    "    return new_data\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b4e5afa5-3628-4acc-b421-fcccd3f81c62",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "out_dir = \"/media/2T_SSD1/tungtt/llm/instructions\"\n",
    "last_file = 155\n",
    "dataset = []\n",
    "\n",
    "for idx in range(last_file):\n",
    "    data_path = os.path.join(out_dir, f'{idx}.json')\n",
    "    example = json.load(open(data_path))\n",
    "    for step in range(2):\n",
    "        if len(example) <= 3 and step == 1:\n",
    "            continue\n",
    "        dataset.append(process(example, step))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "86b97f9e-d68b-4675-93ad-90e4b61a4bde",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('rikkei_data.json', 'w', encoding='utf8') as json_file:\n",
    "    json.dump(dataset, json_file, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "9a54f344-2e75-494d-9b67-ca16a4858a95",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "307\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3fdbed6-4519-437c-a4aa-f3071b060bc0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}