code prompt generation

README.md

@@ -0,0 +1 @@
+# conbot

coding_prompt_generation.ipynb

@@ -0,0 +1,285 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Notebook for generating prompts for code generation using local data\n",
+    "* Use most recently looked at files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "from jupyter_notebook_parser import JupyterNotebookParser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "project_dir = \"/Users/connorparish/code/hindsight_parsing/\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files_to_use = ['notebooks/annotation_export.ipynb', 'annotation_db.py']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "f = files_to_use[0]\n",
+    "f_path = os.path.join(project_dir, f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def parse_notebook(path):\n",
+    "    parsed = JupyterNotebookParser(f_path)\n",
+    "    cells = parsed.get_code_cells()\n",
+    "    code = \"\"\n",
+    "    for cell in cells:\n",
+    "        code += \"\".join(cell['source']) + \"\\n\"\n",
+    "    return code"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "code_context = \"\"\n",
+    "for f in files_to_use:\n",
+    "    f_path = os.path.join(project_dir, f)\n",
+    "    code_context += f\"Code from {f}\" + \"\\n\"\n",
+    "    if f.split('.')[-1] == \"ipynb\":\n",
+    "        code_context += parse_notebook(f_path)\n",
+    "    else:\n",
+    "        with open(f_path, 'r') as infile:\n",
+    "            code_context += infile.read()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "task = \"Create a script annotate.py that runs uses a yolo model to create annotations for a list of images and inserts them into the database.\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = \"Project Context:\\n\" + code_context + \"\\n\\nUsing the above project context perform the following task. \" + task"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Project Context:\n",
+      "Code from notebooks/annotation_export.ipynb\n",
+      "import os\n",
+      "import json\n",
+      "import shutil\n",
+      "\n",
+      "from ultralytics import YOLO\n",
+      "from PIL import Image\n",
+      "\n",
+      "import sys\n",
+      "sys.path.insert(0, \"../../hindsight/hindsight_server/\")\n",
+      "\n",
+      "from db import HindsightDB\n",
+      "from utils import make_dir\n",
+      "db = HindsightDB()\n",
+      "frames = db.get_frames(impute_applications=False)\n",
+      "frames = frames.loc[frames['application'] == \"Twitter\"]\n",
+      "sel_frames = frames.sample(n=100)\n",
+      "dest_dir = \"../data/annotations/twitter/rand_upload_2/\"\n",
+      "make_dir(dest_dir)\n",
+      "for f in sel_frames['path']:\n",
+      "    dest_f = os.path.join(dest_dir, os.path.basename(f))\n",
+      "    shutil.copy(f, dest_dir)\n",
+      "trained_model = YOLO(\"./runs/detect/train15/weights/best.pt\")\n",
+      "pred_images_dir = dest_dir \n",
+      "images = list()\n",
+      "for f in os.listdir(pred_images_dir):\n",
+      "    im = Image.open(os.path.join(pred_images_dir, f))\n",
+      "    images.append(im)\n",
+      "results = trained_model(images)\n",
+      "all_preds = list()\n",
+      "for result in results:\n",
+      "    image_preds_d = {}\n",
+      "    image_path = f\"/data/local-files/?d={result.path}\"\n",
+      "    image_preds_d['data'] = {\"image\" : image_path}\n",
+      "    predictions_d = {\"model_version\": \"train13\", \"score\": 0.55}\n",
+      "    org_width = result.orig_shape[1]\n",
+      "    org_height = result.orig_shape[0]\n",
+      "    result_d_template = {\"type\": \"rectanglelabels\",        \n",
+      "            \"from_name\": \"label\", \"to_name\": \"image\",\n",
+      "            \"original_width\": org_width, \"original_height\": org_height,\n",
+      "            \"image_rotation\": 0}\n",
+      "    converted_results = list()\n",
+      "    for i, box in enumerate(result.boxes):\n",
+      "        result_d = result_d_template.copy()\n",
+      "        result_d['id'] = f\"result{i}\"\n",
+      "        value_d = {\"rotation\" : 0,\n",
+      "                \"x\" : ((float(box.xyxyn[0][0])) * 100),\n",
+      "                \"y\" : (float(box.xyxyn[0][1])) * 100,\n",
+      "                \"width\" : ((float(box.xyxyn[0][2]) - float(box.xyxyn[0][0])) * 100), \n",
+      "                \"height\": (float(box.xyxyn[0][3]) - float(box.xyxyn[0][1])) * 100, \n",
+      "                \"rectanglelabels\": [result.names[int(box.cls[0])]]}\n",
+      "        result_d['value'] = value_d\n",
+      "        converted_results.append(result_d)\n",
+      "        \n",
+      "    predictions_d[\"result\"] = converted_results\n",
+      "    image_preds_d['predictions'] = [predictions_d]\n",
+      "    all_preds.append(image_preds_d)\n",
+      "with open(\"rand_upload_2.json\", 'w') as outfile:\n",
+      "    json.dump(all_preds, outfile)\n",
+      "\n",
+      "\n",
+      "Code from annotation_db.py\n",
+      "import os\n",
+      "import sqlite3\n",
+      "from pathlib import Path\n",
+      "\n",
+      "import portalocker\n",
+      "\n",
+      "BASE_DIR = Path(os.path.dirname(os.path.abspath(__file__)))\n",
+      "\n",
+      "DATA_DIR = BASE_DIR / \"data\"\n",
+      "ANNOTATIONS_DB_FILE = DATA_DIR / \"hindsight_annotations.db\"\n",
+      "\n",
+      "class HindsightAnnotationsDB:\n",
+      "    def __init__(self, db_file=ANNOTATIONS_DB_FILE):\n",
+      "        self.db_file = db_file\n",
+      "        self.lock_file = db_file + '.lock'\n",
+      "        self.create_tables()\n",
+      "\n",
+      "    def get_connection(self):\n",
+      "        \"\"\"Get a new connection every time for thread safety.\"\"\"\n",
+      "        connection = sqlite3.connect(self.db_file, timeout=50)\n",
+      "        connection.execute('PRAGMA journal_mode=WAL;')\n",
+      "        connection.execute('PRAGMA busy_timeout = 10000;')\n",
+      "        return connection\n",
+      "    \n",
+      "    def with_lock(func):\n",
+      "        \"\"\"Decorator to handle database locking.\"\"\"\n",
+      "        def wrapper(self, *args, **kwargs):\n",
+      "            with open(self.lock_file, 'a') as lock_file:\n",
+      "                portalocker.lock(lock_file, portalocker.LOCK_EX)\n",
+      "                try:\n",
+      "                    result = func(self, *args, **kwargs)\n",
+      "                finally:\n",
+      "                    portalocker.unlock(lock_file)\n",
+      "                return result\n",
+      "        return wrapper\n",
+      "    \n",
+      "    @with_lock\n",
+      "    def create_tables(self):\n",
+      "        with self.get_connection() as conn:\n",
+      "            cursor = conn.cursor()\n",
+      "            cursor.execute('''CREATE TABLE IF NOT EXISTS object_detection_annotations (\n",
+      "                            id INTEGER PRIMARY KEY,\n",
+      "                            frame_id INTEGER NOT NULL,\n",
+      "                            x DOUBLE NOT NULL,\n",
+      "                            y DOUBLE NOT NULL,\n",
+      "                            w DOUBLE NOT NULL,\n",
+      "                            h DOUBLE NOT NULL,\n",
+      "                            rotation DOUBLE DEFAULT 0,\n",
+      "                            label TEXT,\n",
+      "                            conf DOUBLE NOT NULL,\n",
+      "                            model_name TEXT NOT NULL,\n",
+      "                            model_version TEXT\n",
+      "                           )\n",
+      "            ''')\n",
+      "\n",
+      "    @with_lock\n",
+      "    def insert_annotation(self, frame_id, x, y, w, h, rotation, label, conf, model_name, model_version=None):\n",
+      "        \"\"\"Insert a new annotation into the database.\"\"\"\n",
+      "        with self.get_connection() as conn:\n",
+      "            cursor = conn.cursor()\n",
+      "            cursor.execute('''INSERT INTO object_detection_annotations \n",
+      "                            (frame_id, x, y, w, h, rotation, label, conf, model_name, model_version) \n",
+      "                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''',\n",
+      "                            (frame_id, x, y, w, h, rotation, label, conf, model_name, model_version))\n",
+      "            conn.commit()\n",
+      "\n",
+      "    @with_lock\n",
+      "    def get_annotations_by_frame(self, frame_id):\n",
+      "        \"\"\"Retrieve annotations by frame ID.\"\"\"\n",
+      "        with self.get_connection() as conn:\n",
+      "            cursor = conn.cursor()\n",
+      "            cursor.execute('''SELECT id, x, y, w, h, rotation, label, conf, model_name, model_version\n",
+      "                              FROM object_detection_annotations \n",
+      "                              WHERE frame_id = ?''', (frame_id,))\n",
+      "            return cursor.fetchall()\n",
+      "\n",
+      "Using the above project context perform the following task. Create a script annotate.py that runs uses a yolo model to create annotations for a list of images and inserts them into the database.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hindsight_exp",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}