OpenAI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5c74b6ae-bd17-43e1-b45e-432142d46a7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import openai\n",
    "import faiss\n",
    "import json\n",
    "import numpy as np\n",
    "\n",
    "from markdownify import markdownify\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "client = openai.OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3459648e-f045-4f87-94a3-5f223b551f65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get documentation from OBP\n",
    "obp_base_url = \"https://test.openbankproject.com\"\n",
    "obp_version = \"v5.1.0\"\n",
    "\n",
    "# swagger docs\n",
    "swagger_url = \"{}/obp/v5.1.0/resource-docs/{}/swagger\".format(obp_base_url, obp_version)\n",
    "swagger_response = requests.get(swagger_url)\n",
    "swagger_json = swagger_response.json()\n",
    "\n",
    "# glossary\n",
    "glossary_url = \"{}/obp/{}/api/glossary\".format(obp_base_url, obp_version)\n",
    "glossary_response = requests.get(glossary_url)\n",
    "glossary_json = glossary_response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c777ee8e-57d4-4f19-a0c8-638ca0948073",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "no reference or unresolvable response body\n",
      "{\n",
      "  \"path\": \"/obp/v5.1.0/account/check/scheme/iban\",\n",
      "  \"method\": \"post\",\n",
      "  \"summary\": \"Validate and check IBAN\",\n",
      "  \"description\": \"Validate and check IBAN for errors\\n\\nAuthentication is Optional\\n\\n\",\n",
      "  \"responses\": [\n",
      "    {\n",
      "      \"code\": \"201\",\n",
      "      \"body\": {\n",
      "        \"is_valid\": {\n",
      "          \"type\": \"boolean\",\n",
      "          \"example\": \"true\"\n",
      "        },\n",
      "        \"details\": {\n",
      "          \"required\": [\n",
      "            \"bank_routings\",\n",
      "            \"city\",\n",
      "            \"postcode\",\n",
      "            \"branch\",\n",
      "            \"country\",\n",
      "            \"attributes\",\n",
      "            \"bank\",\n",
      "            \"address\",\n",
      "            \"phone\"\n",
      "          ],\n",
      "          \"properties\": {\n",
      "            \"bank_routings\": {\n",
      "              \"type\": \"array\",\n",
      "              \"items\": {\n",
      "                \"required\": [\n",
      "                  \"scheme\",\n",
      "                  \"address\"\n",
      "                ],\n",
      "                \"properties\": {\n",
      "                  \"scheme\": {\n",
      "                    \"type\": \"string\",\n",
      "                    \"example\": \"scheme value\"\n",
      "                  },\n",
      "                  \"address\": {\n",
      "                    \"type\": \"string\",\n",
      "                    \"example\": \"\"\n",
      "                  }\n",
      "                }\n",
      "              }\n",
      "            },\n",
      "            \"city\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"Frankfurt am Main\"\n",
      "            },\n",
      "            \"postcode\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"60311\"\n",
      "            },\n",
      "            \"branch\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"\"\n",
      "            },\n",
      "            \"country\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"Germany\"\n",
      "            },\n",
      "            \"attributes\": {\n",
      "              \"type\": \"array\",\n",
      "              \"items\": {\n",
      "                \"required\": [\n",
      "                  \"name\",\n",
      "                  \"value\"\n",
      "                ],\n",
      "                \"properties\": {\n",
      "                  \"name\": {\n",
      "                    \"type\": \"string\",\n",
      "                    \"example\": \"country_iso\"\n",
      "                  },\n",
      "                  \"value\": {\n",
      "                    \"type\": \"string\",\n",
      "                    \"example\": \"\"\n",
      "                  }\n",
      "                }\n",
      "              }\n",
      "            },\n",
      "            \"bank\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"Societe Generale\"\n",
      "            },\n",
      "            \"address\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"Neue mainzer strasse 46-50\"\n",
      "            },\n",
      "            \"phone\": {\n",
      "              \"type\": \"string\",\n",
      "              \"example\": \"\"\n",
      "            }\n",
      "          }\n",
      "        }\n",
      "      }\n",
      "    },\n",
      "    {\n",
      "      \"code\": \"400\",\n",
      "      \"body\": {\n",
      "        \"message\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"OBP-50000: Unknown Error.\"\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  ],\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"address\": {\n",
      "        \"type\": \"string\",\n",
      "        \"example\": \"DE75512108001245126199\"\n",
      "      }\n",
      "    },\n",
      "    \"required\": [\n",
      "      \"address\"\n",
      "    ]\n",
      "  }\n",
      "}\n",
      "{\n",
      "  \"path\": \"/obp/v5.1.0/accounts/public\",\n",
      "  \"method\": \"get\",\n",
      "  \"summary\": \"Get Public Accounts at all Banks\",\n",
      "  \"description\": \"Get public accounts at all banks (Anonymous access).  \\nReturns accounts that contain at least one public view (a view where is\\\\_public is true)  \\nFor each account the API returns the ID and the available views.\\n\\nAuthentication is Optional\\n\\n\",\n",
      "  \"responses\": [\n",
      "    {\n",
      "      \"code\": \"200\",\n",
      "      \"body\": {\n",
      "        \"accounts\": {\n",
      "          \"type\": \"array\",\n",
      "          \"items\": {\n",
      "            \"required\": [\n",
      "              \"id\",\n",
      "              \"label\",\n",
      "              \"bank_id\",\n",
      "              \"views_available\"\n",
      "            ],\n",
      "            \"properties\": {\n",
      "              \"id\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"8ca8a7e4-6d02-48e3-a029-0b2bf89de9f0\"\n",
      "              },\n",
      "              \"label\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"NoneLabel\"\n",
      "              },\n",
      "              \"bank_id\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"gh.29.uk\"\n",
      "              },\n",
      "              \"views_available\": {\n",
      "                \"type\": \"array\",\n",
      "                \"items\": {\n",
      "                  \"required\": [\n",
      "                    \"id\",\n",
      "                    \"short_name\",\n",
      "                    \"is_public\"\n",
      "                  ],\n",
      "                  \"properties\": {\n",
      "                    \"id\": {\n",
      "                      \"type\": \"string\",\n",
      "                      \"example\": \"1\"\n",
      "                    },\n",
      "                    \"short_name\": {\n",
      "                      \"type\": \"string\",\n",
      "                      \"example\": \"HHH\"\n",
      "                    },\n",
      "                    \"is_public\": {\n",
      "                      \"type\": \"boolean\",\n",
      "                      \"example\": \"true\"\n",
      "                    }\n",
      "                  }\n",
      "                }\n",
      "              }\n",
      "            }\n",
      "          }\n",
      "        }\n",
      "      }\n",
      "    },\n",
      "    {\n",
      "      \"code\": \"400\",\n",
      "      \"body\": {\n",
      "        \"message\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"OBP-30066: Could not get accounts.\"\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  ],\n",
      "  \"parameters\": {\n",
      "    \"type\": \"object\",\n",
      "    \"properties\": {\n",
      "      \"jsonString\": {\n",
      "        \"type\": \"string\",\n",
      "        \"example\": \"{}\"\n",
      "      }\n",
      "    },\n",
      "    \"required\": [\n",
      "      \"jsonString\"\n",
      "    ]\n",
      "  }\n",
      "}\n",
      "{\n",
      "  \"path\": \"/obp/v5.1.0/adapter\",\n",
      "  \"method\": \"get\",\n",
      "  \"summary\": \"Get Adapter Info\",\n",
      "  \"description\": \"Get basic information about the Adapter.\\n\\nAuthentication is Optional\\n\\nAuthentication is Mandatory\\n\\n\",\n",
      "  \"responses\": [\n",
      "    {\n",
      "      \"code\": \"200\",\n",
      "      \"body\": {\n",
      "        \"name\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"ACCOUNT_MANAGEMENT_FEE\"\n",
      "        },\n",
      "        \"backend_messages\": {\n",
      "          \"type\": \"array\",\n",
      "          \"items\": {\n",
      "            \"required\": [\n",
      "              \"source\",\n",
      "              \"text\",\n",
      "              \"errorCode\",\n",
      "              \"status\"\n",
      "            ],\n",
      "            \"properties\": {\n",
      "              \"duration\": {\n",
      "                \"type\": \"string\",\n",
      "                \"format\": \"double\",\n",
      "                \"example\": \"5.123\"\n",
      "              },\n",
      "              \"source\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"\"\n",
      "              },\n",
      "              \"text\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"\"\n",
      "              },\n",
      "              \"errorCode\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"\"\n",
      "              },\n",
      "              \"status\": {\n",
      "                \"type\": \"string\",\n",
      "                \"example\": \"\"\n",
      "              }\n",
      "            }\n",
      "          }\n",
      "        },\n",
      "        \"total_duration\": {\n",
      "          \"type\": \"string\",\n",
      "          \"format\": \"double\",\n",
      "          \"example\": \"5.123\"\n",
      "        },\n",
      "        \"version\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"\"\n",
      "        },\n",
      "        \"date\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"2020-01-27\"\n",
      "        },\n",
      "        \"git_commit\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"59623811dd8a41f6ffe67be46954eee11913dc28\"\n",
      "        }\n",
      "      }\n",
      "    },\n",
      "    {\n",
      "      \"code\": \"400\",\n",
      "      \"body\": {\n",
      "        \"message\": {\n",
      "          \"type\": \"string\",\n",
      "          \"example\": \"OBP-20001: User not logged in. Authentication is required!\"\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  ],\n",
      "  \"parameters\": null\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "def resolve_reference(ref, definitions, resolved={}):\n",
    "    \"\"\"\n",
    "    Resolves a $ref to its definition, avoiding circular references.\n",
    "    \"\"\"\n",
    "    ref_name = ref.split('/')[-1]\n",
    "    if ref_name in resolved:\n",
    "        return resolved[ref_name]\n",
    "\n",
    "    if ref_name in definitions.keys():\n",
    "        definition = definitions[ref_name]\n",
    "    else:\n",
    "        definition = {}\n",
    "    resolved[ref_name] = definition\n",
    "    properties = definition.get('properties', {})\n",
    "    resolved_properties = resolve_properties(properties, definitions, resolved)\n",
    "    return {**definition, 'properties': resolved_properties}\n",
    "\n",
    "def resolve_properties(properties, definitions, resolved):\n",
    "    \"\"\"\n",
    "    Resolves nested references in properties, avoiding circular references.\n",
    "    \"\"\"\n",
    "    resolved_properties = {}\n",
    "    for prop_name, prop_details in properties.items():\n",
    "        if '$ref' in prop_details:\n",
    "            resolved_properties[prop_name] = resolve_reference(prop_details['$ref'], definitions, resolved)\n",
    "        elif prop_details.get('type') == 'array' and 'items' in prop_details and '$ref' in prop_details['items']:\n",
    "            resolved_properties[prop_name] = {\n",
    "                \"type\": \"array\",\n",
    "                \"items\": resolve_reference(prop_details['items']['$ref'], definitions, resolved)\n",
    "            }\n",
    "        else:\n",
    "            resolved_properties[prop_name] = prop_details\n",
    "    return resolved_properties\n",
    "\n",
    "def parse_swagger(swagger_json):\n",
    "    paths = swagger_json['paths']\n",
    "    definitions = swagger_json['definitions']\n",
    "\n",
    "    endpoints = []\n",
    "    for path, methods in paths.items():\n",
    "        for method, details in methods.items():\n",
    "            endpoint_info = {\n",
    "                'path': path,\n",
    "                'method': method,\n",
    "                'summary': details.get('summary'),\n",
    "                'description': markdownify(details.get('description', '')),\n",
    "                'responses': []\n",
    "            }\n",
    "\n",
    "            if ('parameters' in details) & (details['parameters'] != []):\n",
    "                endpoint_info[\"parameters\"] = {\n",
    "                        \"type\": \"object\",\n",
    "                        \"properties\": {},\n",
    "                        \"required\": []\n",
    "                    }\n",
    "                for param in details['parameters']:\n",
    "                    if param['in'] == 'body' and '$ref' in param['schema']:\n",
    "                        ref = param['schema']['$ref']\n",
    "                        definition_name = ref.split('/')[-1]\n",
    "                        definition = resolve_reference(ref, definitions)\n",
    "                        \n",
    "                        endpoint_info['parameters']['required'].extend(definition.get('required', []))\n",
    "                        endpoint_info['parameters']['properties'].update(\n",
    "                            resolve_properties(definition.get('properties', {}), definitions, {})\n",
    "                        )\n",
    "                    elif param['in'] == 'body' and '$ref' not in param['schema']:\n",
    "                        endpoint_info[\"parameters\"] = param[\"schema\"]\n",
    "                        \n",
    "                    elif param['in'] == 'path':\n",
    "                        endpoint_info['parameters']['required'].append(param['name'])\n",
    "                        endpoint_info['parameters']['properties'][param['name']] = {\n",
    "                            \"type\": param['type'],\n",
    "                            \"description\": param.get('description', '')\n",
    "                        }\n",
    "                    elif param['in'] == 'query':\n",
    "                        endpoint_info['parameters']['properties'][param['name']] = {\n",
    "                            \"type\": param['type'],\n",
    "                            \"description\": param.get('description', '')\n",
    "                        }\n",
    "                        if param.get('required', False):\n",
    "                            endpoint_info['parameters']['required'].append(param['name'])\n",
    "\n",
    "            \n",
    "            elif details['parameters'] == []:\n",
    "                    endpoint_info['parameters'] = None\n",
    "            \n",
    "            if 'responses' in details:\n",
    "                for code, response in details['responses'].items():\n",
    "                    if \"schema\" in response.keys() and (\"$ref\" in response['schema']):\n",
    "                        ref = response['schema']['$ref']\n",
    "                        definition_name = ref.split('/')[-1]\n",
    "                        definition = resolve_reference(ref, definitions)\n",
    "\n",
    "                        response_resolved = {\n",
    "                            \"code\": code,\n",
    "                            \"body\": resolve_properties(definition.get('properties', {}), definitions, {})\n",
    "                        }\n",
    "\n",
    "                        endpoint_info[\"responses\"].append(response_resolved)\n",
    "                    else:\n",
    "                        print(\"no reference or unresolvable response body\")\n",
    "                        \n",
    "                    \n",
    "                    \n",
    "                    \n",
    "            \n",
    "            endpoints.append(endpoint_info)\n",
    "    return endpoints\n",
    "\n",
    "endpoints = parse_swagger(swagger_json)\n",
    "for endpoint in endpoints[:3]:\n",
    "    print(json.dumps(endpoint, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b1bc2a76-f532-4510-9d54-287fd7fd8fcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'title': 'API',\n",
       "  'description': 'The terms `API` (Application Programming Interface) and `Endpoint` are used somewhat interchangeably.\\n\\nHowever, an API normally refers to a group of Endpoints.\\n\\nAn endpoint has a unique URL path and HTTP verb (GET, POST, PUT, DELETE etc).\\n\\nWhen we POST a Swagger file to the Create Endpoint endpoint, we are in fact creating a set of Endpoints that have a common Tag. Tags are used to group Endpoints in the API Explorer and filter the Endpoints in the Resource Doc endpoints.\\n\\nEndpoints can also be grouped together in Collections.\\n\\nSee also [Endpoint](/glossary#Endpoint)\\n\\n\\t\\t\\t\\t '},\n",
       " {'title': 'API Collection',\n",
       "  'description': 'An API Collection is a collection of endpoints grouped together for a certain purpose.\\n\\nHaving read access to a Collection does not constitute execute access on the endpoints in the Collection.\\n\\n(Execute access is governed by Entitlements to Roles - and in some cases, Views.)\\n\\nCollections can be created and shared. You can make a collection non-sharable but the default is sharable.\\n\\nYour \"Favourites\" in API Explorer is actually a collection you control named \"Favourites\".\\n\\nTo share a Collection (e.g. your Favourites) just click on your Favourites in the API Explorer and share the URL in the browser. If you want to share the Collection via an API, just share the collection_id with a developer.\\n\\nIf you share a Collection it can\\'t be modified by anyone else, but anyone can use it as a basis for their own Favourites or another collection.\\n\\nThere are over 13 endpoints for controlling Collections.\\nSome of these endpoints require Entitlements to Roles and some operate on your own personal collections such as your favourites.\\n\\n '}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parse_glossary(glossary_json):\n",
    "    glossary_items = glossary_json['glossary_items']\n",
    "    parsed_items = []\n",
    "    \n",
    "    for item in glossary_items:\n",
    "        title = item.get('title', 'No title')\n",
    "        description_info = item.get('description', {})\n",
    "        \n",
    "        # Get markdown description or else return no description\n",
    "        description = description_info.get('markdown', 'No description')\n",
    "        # do not add descriptions if they are empty\n",
    "        if description == \"\":\n",
    "            continue\n",
    "        \n",
    "        parsed_items.append({\n",
    "            'title': title,\n",
    "            'description': description\n",
    "        })\n",
    "    \n",
    "    return parsed_items\n",
    "\n",
    "glossary_items = parse_glossary(glossary_json)\n",
    "glossary_items[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ea8b589b-56bc-4645-a1f4-e852e1656aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create vector embeddings\n",
    "def get_embeddings(texts):\n",
    "    response = client.embeddings.create(\n",
    "        model=\"text-embedding-3-small\",\n",
    "        input=texts\n",
    "    )\n",
    "    \n",
    "    return [e.embedding for e in response.data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "68efce72-595c-4109-ab43-e4f07af607d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_and_save_embedding_faiss(formatted_texts: list, json_metadata: list, filename: str):\n",
    "    \"\"\"\n",
    "    Creates and saves text embeddings and metadata for a given list of texts \n",
    "\n",
    "    texts: formatted list of texts for creating embeddings\n",
    "    json_metadata: dict to pass as json metadata (ie. [{'title': 'API', 'description': '<API DESCRIPTION>'}, ...] for glossary)\n",
    "    filename: prefix to attach to saved index and metadata i.e. 'glossary' for saving 'glossary_index.faiss' and 'glossary_metadata.json'\n",
    "    \"\"\"\n",
    "    embeddings = get_embeddings(formatted_texts)\n",
    "    \n",
    "    # Convert embeddings to a numpy array\n",
    "    embeddings_np = np.array(embeddings).astype('float32')\n",
    "    \n",
    "    # Create a FAISS index\n",
    "    index = faiss.IndexFlatL2(embeddings_np.shape[1])  # L2 distance index\n",
    "    index.add(embeddings_np)\n",
    "    \n",
    "    # Optionally, save the index to disk for later use\n",
    "    faiss.write_index(index, f\"{filename}_index.faiss\")\n",
    "    \n",
    "    # Save metadata for retrieval\n",
    "    with open(f\"{filename}_metadata.json\", 'w') as f:\n",
    "        json.dump(json_metadata, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e9680bc1-075d-4371-9167-1ea96a3d376a",
   "metadata": {},
   "outputs": [],
   "source": [
    "endpoint_texts = [f\"{e['method'].upper()} {e['path']} - {e['description']}\" for e in endpoints]\n",
    "create_and_save_embedding_faiss(endpoint_texts, endpoints, \"endpoint\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9955272a-cd14-4bfd-82f2-9e9051da6eae",
   "metadata": {},
   "outputs": [],
   "source": [
    "glossary_texts = [f\"{g['title']} - {g['description']}\" for g in glossary_items]\n",
    "create_and_save_embedding_faiss(glossary_texts, glossary_items, \"glossary\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8de6a1a8-771d-4d86-87e4-8ca5656ac344",
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_endpoints(query):\n",
    "    query_embedding = get_embeddings([query])[0]\n",
    "    query_embedding_np = np.array([query_embedding]).astype('float32')\n",
    "\n",
    "    \n",
    "    \n",
    "    # Perform the search\n",
    "    distances, indices = index.search(query_embedding_np, k=5)\n",
    "    \n",
    "    # Load metadata\n",
    "    with open('endpoints_metadata.json', 'r') as f:\n",
    "        endpoints = json.load(f)\n",
    "    \n",
    "    # Retrieve matching endpoints\n",
    "    matches = [endpoints[i] for i in indices[0]]\n",
    "    return matches\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}