Logan/merge next

.github/workflows/unit_test.yml

@@ -8,6 +8,8 @@ on:
 
 env:
   POETRY_VERSION: "1.6.1"
+  LLAMA_CLOUD_BASE_URL: ${{ secrets.PLATFORM_BASE_URL }}
+  LLAMA_CLOUD_API_KEY: ${{ secrets.PLATFORM_API_KEY }}
 
 jobs:
   test:

docs/examples/evaluation/faithfulness_eval.ipynb

@@ -19,12 +19,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip install llama-index-llms-openai"
+    "%pip install llama-index-llms-openai pandas[jinja2] spacy"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "4a8304f2",
    "metadata": {},
    "outputs": [],
@@ -37,28 +37,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "9080b39e",
+   "execution_count": 2,
+   "id": "190a6684",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# configuring logger to INFO level\n",
-    "import logging\n",
-    "import sys\n",
-    "\n",
-    "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
-    "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
+    "import os\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "8d0b2364-4806-4656-81e7-3f6e4b910b5b",
    "metadata": {},
    "outputs": [],
    "source": [
     "from llama_index.core import (\n",
-    "    TreeIndex,\n",
     "    VectorStoreIndex,\n",
     "    SimpleDirectoryReader,\n",
     "    Response,\n",
@@ -81,7 +76,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "b9b98f89-d5b8-4d29-92f6-ad76d5060e9f",
    "metadata": {},
    "outputs": [],
@@ -94,7 +89,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "1298bbb4-c99e-431e-93ef-eb32c0a2fc2a",
    "metadata": {},
    "outputs": [],
@@ -104,7 +99,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "41f0e53f-77a6-40d5-94ae-3f81b01af75c",
    "metadata": {},
    "outputs": [],
@@ -118,13 +113,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "af730b2e-6949-4865-b7af-bb2bc60a9173",
    "metadata": {},
    "outputs": [],
    "source": [
+    "from llama_index.core.evaluation import EvaluationResult\n",
+    "\n",
     "# define jupyter display function\n",
-    "def display_eval_df(response: Response, eval_result: str) -> None:\n",
+    "def display_eval_df(response: Response, eval_result: EvaluationResult) -> None:\n",
     "    if response.source_nodes == []:\n",
     "        print(\"no response!\")\n",
     "        return\n",
@@ -133,6 +130,7 @@
     "            \"Response\": str(response),\n",
     "            \"Source\": response.source_nodes[0].node.text[:1000] + \"...\",\n",
     "            \"Evaluation Result\": \"Pass\" if eval_result.passing else \"Fail\",\n",
+    "            \"Reasoning\": eval_result.feedback,\n",
     "        },\n",
     "        index=[0],\n",
     "    )\n",
@@ -156,7 +154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "180a5d2e-9286-477b-9cd0-a5976d18d845",
    "metadata": {},
    "outputs": [],
@@ -168,40 +166,42 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "c764b8b3-69b1-4ac8-b88b-3f9e204b8bfb",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
        "<style type=\"text/css\">\n",
-       "#T_db5e8_row0_col0, #T_db5e8_row0_col1 {\n",
+       "#T_21e1a_row0_col0, #T_21e1a_row0_col1 {\n",
        "  inline-size: 600px;\n",
        "  overflow-wrap: break-word;\n",
        "}\n",
        "</style>\n",
-       "<table id=\"T_db5e8\">\n",
+       "<table id=\"T_21e1a\">\n",
        "  <thead>\n",
        "    <tr>\n",
        "      <th class=\"blank level0\" >&nbsp;</th>\n",
-       "      <th id=\"T_db5e8_level0_col0\" class=\"col_heading level0 col0\" >Response</th>\n",
-       "      <th id=\"T_db5e8_level0_col1\" class=\"col_heading level0 col1\" >Source</th>\n",
-       "      <th id=\"T_db5e8_level0_col2\" class=\"col_heading level0 col2\" >Evaluation Result</th>\n",
+       "      <th id=\"T_21e1a_level0_col0\" class=\"col_heading level0 col0\" >Response</th>\n",
+       "      <th id=\"T_21e1a_level0_col1\" class=\"col_heading level0 col1\" >Source</th>\n",
+       "      <th id=\"T_21e1a_level0_col2\" class=\"col_heading level0 col2\" >Evaluation Result</th>\n",
+       "      <th id=\"T_21e1a_level0_col3\" class=\"col_heading level0 col3\" >Reasoning</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th id=\"T_db5e8_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
-       "      <td id=\"T_db5e8_row0_col0\" class=\"data row0 col0\" >New York City got its name from the English explorer Henry Hudson, who rediscovered New York Harbor in 1609 while searching for the Northwest Passage. He named the area New York after the Duke of York, who later became King James II of England.</td>\n",
-       "      <td id=\"T_db5e8_row0_col1\" class=\"data row0 col1\" >He claimed the area for France and named it Nouvelle Angoulême (New Angoulême).A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio ('Saint Anthony's River').The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed by Gomes' expedition and labeled the northeastern United States as Tierra de Esteban Gómez in his honor.In 1609, the English explorer Henry Hudson rediscovered New York Harbor while searching for the Northwest Passage to the Orient for the Dutch East India Company.He proceeded to sail up what the Dutch would name the North River (now the Hudson River), named first by Hudson as the Mauritius after Maurice, Prince of Orange.Hudson's first mate described the harbor as \"a very good Harbour for all windes\" and the river as \"a mile broad\" and \"full of fish\".Hud...</td>\n",
-       "      <td id=\"T_db5e8_row0_col2\" class=\"data row0 col2\" >Fail</td>\n",
+       "      <th id=\"T_21e1a_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "      <td id=\"T_21e1a_row0_col0\" class=\"data row0 col0\" >New York City got its name when it came under British control in 1664. It was renamed New York after King Charles II of England granted the lands to his brother, the Duke of York.</td>\n",
+       "      <td id=\"T_21e1a_row0_col1\" class=\"data row0 col1\" >The city came under British control in 1664 and was renamed New York after King Charles II of England granted the lands to his brother, the Duke of York. The city was regained by the Dutch in July 1673 and was renamed New Orange for one year and three months; the city has been continuously named New York since November 1674. New York City was the capital of the United States from 1785 until 1790, and has been the largest U.S. city since 1790. The Statue of Liberty greeted millions of immigrants as they came to the U.S. by ship in the late 19th and early 20th centuries, and is a symbol of the U.S. and its ideals of liberty and peace. In the 21st century, New York City has emerged as a global node of creativity, entrepreneurship, and as a symbol of freedom and cultural diversity. The New York Times has won the most Pulitzer Prizes for journalism and remains the U.S. media's \"newspaper of record\". In 2019, New York City was voted the greatest city in the world in a survey of over 30,000 p...</td>\n",
+       "      <td id=\"T_21e1a_row0_col2\" class=\"data row0 col2\" >Pass</td>\n",
+       "      <td id=\"T_21e1a_row0_col3\" class=\"data row0 col3\" >YES</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n"
       ],
       "text/plain": [
-       "<pandas.io.formats.style.Styler at 0x7f991be1bc40>"
+       "<pandas.io.formats.style.Styler at 0x169353d10>"
       ]
      },
      "metadata": {},
@@ -224,30 +224,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "90a8cd4d",
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "WARNING:llama_index.indices.service_context:chunk_size_limit is deprecated, please specify chunk_size instead\n",
-      "chunk_size_limit is deprecated, please specify chunk_size instead\n",
-      "chunk_size_limit is deprecated, please specify chunk_size instead\n"
+      "/Users/loganmarkewich/giant_change/llama_index/llama-index-core/llama_index/core/evaluation/dataset_generation.py:212: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
+      "  return cls(\n",
+      "/Users/loganmarkewich/giant_change/llama_index/llama-index-core/llama_index/core/evaluation/dataset_generation.py:309: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
+      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n"
      ]
     },
     {
      "data": {
       "text/plain": [
        "['What is the population of New York City as of 2020?',\n",
-       " 'Which borough of New York City is home to the headquarters of the United Nations?',\n",
-       " 'How many languages are spoken in New York City, making it the most linguistically diverse city in the world?',\n",
-       " 'Who founded the trading post on Manhattan Island that would later become New York City?',\n",
-       " 'What was New York City named after in 1664?']"
+       " 'Which city is the second-largest in the United States?',\n",
+       " 'How many people live within 250 miles of New York City?',\n",
+       " 'What are the five boroughs of New York City?',\n",
+       " 'What is the gross metropolitan product of the New York metropolitan area?']"
       ]
      },
-     "execution_count": null,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -263,7 +264,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "810ee913",
    "metadata": {},
    "outputs": [],
@@ -289,44 +290,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "8a7ca4b8",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=35 request_id=b36e17a843c31e827f0b7034e603cf28 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=35 request_id=b36e17a843c31e827f0b7034e603cf28 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=35 request_id=b36e17a843c31e827f0b7034e603cf28 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=35 request_id=5acb726518065db9312da9f23beef411 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=35 request_id=5acb726518065db9312da9f23beef411 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=35 request_id=5acb726518065db9312da9f23beef411 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=46 request_id=4af43bfbe4e24fdae0ec33312ee7491e response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=46 request_id=4af43bfbe4e24fdae0ec33312ee7491e response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=46 request_id=4af43bfbe4e24fdae0ec33312ee7491e response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=37 request_id=e30413546fe5f96d3890606767f2ec53 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=37 request_id=e30413546fe5f96d3890606767f2ec53 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=37 request_id=e30413546fe5f96d3890606767f2ec53 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=33 request_id=01f0a8dada4dae80c97a9a412f03b84f response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=33 request_id=01f0a8dada4dae80c97a9a412f03b84f response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/embeddings processing_ms=33 request_id=01f0a8dada4dae80c97a9a412f03b84f response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=282 request_id=ed7b1f8ba68ae32b1d8e24e0d0764e86 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=282 request_id=ed7b1f8ba68ae32b1d8e24e0d0764e86 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=282 request_id=ed7b1f8ba68ae32b1d8e24e0d0764e86 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=820 request_id=b4532c6d665b6cfd644861ed69819cb9 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=820 request_id=b4532c6d665b6cfd644861ed69819cb9 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=820 request_id=b4532c6d665b6cfd644861ed69819cb9 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=847 request_id=4d9bbc71a95b7e0bb69a048e251772c8 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=847 request_id=4d9bbc71a95b7e0bb69a048e251772c8 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=847 request_id=4d9bbc71a95b7e0bb69a048e251772c8 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=952 request_id=d1657940d881929d500b1fddc46b5866 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=952 request_id=d1657940d881929d500b1fddc46b5866 response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=952 request_id=d1657940d881929d500b1fddc46b5866 response_code=200\n",
-      "INFO:openai:message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=1482 request_id=c4456f75580d227f846d3a044e5eef1b response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=1482 request_id=c4456f75580d227f846d3a044e5eef1b response_code=200\n",
-      "message='OpenAI API response' path=https://api.openai.com/v1/chat/completions processing_ms=1482 request_id=c4456f75580d227f846d3a044e5eef1b response_code=200\n",
       "finished query\n",
       "score: 5/5\n"
      ]
@@ -355,7 +326,8 @@
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3"
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
   }
  },
  "nbformat": 4,