Merge branch 'main' of https://github.com/evidentlyai/evidently

truefoundry · Sep 10, 2024 · 97aa3d8 · 97aa3d8
2 parents 64b0e27 + 29dca00
commit 97aa3d8
Show file tree

Hide file tree

Showing 128 changed files with 1,961 additions and 4,739 deletions.
diff --git a/.github/share-actions/ui-node-pnpm-install/action.yml b/.github/share-actions/ui-node-pnpm-install/action.yml
@@ -1,4 +1,10 @@
 name: UI install
+inputs:
+  args:
+    type: string
+    description: ""
+    required: false
+    default: ""
 runs:
   using: "composite"
   steps:
@@ -15,5 +21,5 @@ runs:
 
     - name: 📥 Install node dependencies
       working-directory: ui
-      run: pnpm i --frozen-lockfile --ignore-scripts
+      run: pnpm i --frozen-lockfile --ignore-scripts ${{ inputs.args }}
       shell: bash
diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml
@@ -57,6 +57,23 @@ jobs:
           echo "One or more evidently_python file(s) has changed."
           echo "List all the files that have changed: ${{ steps.changed-files.outputs.evidently_python_all_changed_and_modified_files }}"
 
+  code-check:
+    name: UI code-check
+    runs-on: ubuntu-22.04
+    needs: changed_files
+    if: ${{ github.event.pull_request.draft == false && (needs.changed_files.outputs.ui_any_modified == 'true' || needs.changed_files.outputs.evidently_python_any_modified == 'true') }}
+
+    steps:
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+
+      - name: 📥 Install ui dependencies
+        uses: ./.github/share-actions/ui-node-pnpm-install
+
+      - name: 🔬 Check code quality
+        working-directory: ui
+        run: pnpm code-check
+
   ui-type-check:
     name: UI type-check
     runs-on: ubuntu-22.04
@@ -150,16 +167,8 @@ jobs:
         working-directory: ui
         run: pnpm dlx [email protected] install --with-deps
 
-      - uses: actions/cache@v3
-        id: cache-bikes-dataset
-        env:
-          cache-name: cache-bikes-dataset
-        with:
-          path: Bike-Sharing-Dataset.zip
-          key: cache-bikes-dataset
-      - name: Download test data
-        if: ${{ steps.cache-bikes-dataset.outputs.cache-hit != 'true' }}
-        run: curl -k https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip -o Bike-Sharing-Dataset.zip
+      - name: 🔍 Get bikes dataset cached
+        uses: ./.github/share-actions/get-bikes-dataset-cached
 
       - name: Run UI
         run: EXPERIMENTAL_DETERMINISTIC_UUID="true" evidently ui --port 8000 --workspace workspace-for-visual-testing --demo-projects all &

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,7 +19,8 @@ repos:
         args: [--exit-non-zero-on-fix, --fix]
       - id: ruff-format
 
-#  - repo: https://github.com/pre-commit/mirrors-prettier
-#    rev: "v3.0.3"
-#    hooks:
-#      - id: prettier
+  - repo: https://github.com/biomejs/pre-commit
+    rev: "v0.4.0"
+    hooks:
+      - id: biome-check
+        additional_dependencies: ["@biomejs/[email protected]"]
diff --git a/biome.jsonc b/biome.jsonc
@@ -0,0 +1,53 @@
+{
+  "files": {
+    // `endpoints.d.ts` is autogenerated
+    // `JsonParser.ts` copied from https://github.com/douglascrockford/JSON-js
+    "ignore": [
+      "ui/packages/evidently-ui-lib/src/**/JsonParser.ts",
+      "ui/packages/evidently-ui-lib/src/**/endpoints.d.ts",
+      "ui/packages/evidently-ui-lib/.tsc-dts/**"
+    ],
+    "include": ["ui/*/src/**", "ui/packages/*/src/**"]
+  },
+  "formatter": {
+    "enabled": true,
+    "formatWithErrors": false,
+    "indentStyle": "space",
+    "indentWidth": 2,
+    "lineEnding": "lf",
+    "lineWidth": 100,
+    "attributePosition": "auto"
+  },
+  "css": {
+    "linter": {
+      "enabled": true
+    },
+    "formatter": {
+      "enabled": true
+    }
+  },
+  "javascript": {
+    "formatter": {
+      "arrowParentheses": "always",
+      "bracketSameLine": false,
+      "bracketSpacing": true,
+      "jsxQuoteStyle": "single",
+      "quoteStyle": "single",
+      "quoteProperties": "asNeeded",
+      "semicolons": "asNeeded",
+      "trailingCommas": "none"
+    }
+  },
+  "json": {
+    "formatter": {
+      "trailingCommas": "none"
+    }
+  },
+  "organizeImports": { "enabled": true },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": true
+    }
+  }
+}
diff --git a/examples/how_to_questions/how_to_evaluate_llm_with_text_descriptors.ipynb b/examples/how_to_questions/how_to_evaluate_llm_with_text_descriptors.ipynb
@@ -50,7 +50,7 @@
     "from evidently.metrics import ColumnSummaryMetric, ColumnDistributionMetric, ColumnDriftMetric, DataDriftTable, TextDescriptorsDistribution, ColumnCategoryMetric\n",
     "from evidently.tests import TestColumnValueMin, TestColumnValueMean, TestCategoryShare, TestShareOfOutRangeValues\n",
     "\n",
-    "from evidently.metric_preset import DataDriftPreset, DataQualityPreset, TextOverviewPreset, TextEvals\n",
+    "from evidently.metric_preset import DataDriftPreset, DataQualityPreset, TextEvals\n",
     "\n",
     "from evidently.descriptors import HuggingFaceModel, HuggingFaceToxicityModel, OpenAIPrompting \n",
     "from evidently.descriptors import RegExp, BeginsWith, EndsWith, Contains, DoesNotContain, IncludesWords, ExcludesWords\n",

diff --git a/examples/how_to_questions/how_to_use_llm_judge_template.ipynb b/examples/how_to_questions/how_to_use_llm_judge_template.ipynb
@@ -15,7 +15,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from evidently.descriptors import LLMEval, NegativityLLMEval, PIILLMEval, DeclineLLMEval, BiasLLMEval, ToxicityLLMEval"
+    "from evidently.descriptors import LLMEval, NegativityLLMEval, PIILLMEval, DeclineLLMEval, BiasLLMEval, ToxicityLLMEval, ContextQualityLLMEval\n",
+    "from evidently.descriptors import SemanticSimilarity \n",
+    "from evidently.descriptors import CustomColumnEval, CustomPairColumnEval"
    ]
   },
   {
@@ -52,7 +54,7 @@
     "\n",
     "from evidently.metrics import ColumnSummaryMetric\n",
     "\n",
-    "from evidently.metric_preset import DataQualityPreset, TextOverviewPreset, TextEvals"
+    "from evidently.metric_preset import DataQualityPreset, TextEvals"
    ]
   },
   {
@@ -124,7 +126,7 @@
     "column_mapping = ColumnMapping(\n",
     "    datetime='start_time',\n",
     "    datetime_features=['end_time'],\n",
-    "    text_features=['question', 'response'],\n",
+    "    text_features=['question', 'response', 'target_response'],\n",
     "    categorical_features=['organization', 'model_ID', 'region', 'environment', 'feedback'],\n",
     ")"
    ]
@@ -183,6 +185,16 @@
     "report "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "204d90a4-694e-406b-949a-f7ba3b601eac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(ToxicityLLMEval().get_template().get_prompt_template())"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -223,7 +235,8 @@
    "source": [
     "report = Report(metrics=[\n",
     "    TextEvals(column_name=\"question\", descriptors=[\n",
-    "        NegativityLLMEval(include_category=True)   \n",
+    "        NegativityLLMEval(include_category=True),\n",
+    "        SemanticSimilarity(with_column=\"response\")\n",
     "    ]),\n",
     "    TextEvals(column_name=\"response\", descriptors=[\n",
     "        PIILLMEval(include_reasoning=False), \n",
@@ -237,6 +250,129 @@
     "report "
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "867af0aa-e833-49b5-9701-8f1111beb2bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = [\n",
+    "    [\"Why is the sky blue?\",\n",
+    "     \"The sky is blue because molecules in the air scatter blue light from the sun more than they scatter red light.\",\n",
+    "     \"The sky appears blue because air molecules scatter the sun’s blue light more than they scatter other colors.\"],\n",
+    "\n",
+    "    [\"How do airplanes stay in the air?\",\n",
+    "     \"Airplanes stay in the air because their wings create lift by forcing air to move faster over the top of the wing than underneath, which creates lower pressure on top.\",\n",
+    "     \"Airplanes stay airborne because the shape of their wings causes air to move faster over the top than the bottom, generating lift.\"],\n",
+    "    \n",
+    "    #irrelevant one\n",
+    "    [\"How do airplanes stay in the air?\",\n",
+    "     \"The moon changes shape, or goes through phases, because we see different portions of its illuminated half as it orbits the Earth.\",\n",
+    "     \"Airplanes stay airborne because the shape of their wings causes air to move faster over the top than the bottom, generating lift.\"],\n",
+    "\n",
+    "    [\"Why do we have seasons?\",\n",
+    "     \"We have seasons because the Earth is tilted on its axis, which causes different parts of the Earth to receive more or less sunlight throughout the year.\",\n",
+    "     \"Seasons occur because of the tilt of the Earth’s axis, leading to varying amounts of sunlight reaching different areas as the Earth orbits the sun.\"],\n",
+    "\n",
+    "    [\"How do magnets work?\",\n",
+    "     \"Magnets work because they have a magnetic field that can attract or repel certain metals, like iron, due to the alignment of their atomic particles.\",\n",
+    "     \"Magnets generate a magnetic field, which can attract metals like iron by causing the electrons in those metals to align in a particular way, creating an attractive or repulsive force.\"],\n",
+    "\n",
+    "    [\"Why does the moon change shape?\",\n",
+    "     \"The moon changes shape, or goes through phases, because we see different portions of its illuminated half as it orbits the Earth.\",\n",
+    "     \"The moon appears to change shape as it orbits Earth, which is because we see different parts of its lit-up half at different times. The sun lights up half of the moon, but as the moon moves around the Earth, we see varying portions of that lit-up side. So, the moon's shape in the sky seems to change gradually, from a thin crescent to a full circle and back to a crescent again.\"]\n",
+    "]\n",
+    "\n",
+    "columns = [\"question\", \"context\", \"target_context\"]\n",
+    "\n",
+    "eval_data = pd.DataFrame(data, columns=columns)\n",
+    "\n",
+    "report = Report(metrics=[\n",
+    "    TextEvals(column_name=\"context\", descriptors=[\n",
+    "        ContextQualityLLMEval(question=\"question\"),\n",
+    "    ])\n",
+    "])\n",
+    "\n",
+    "report.run(reference_data=eval_data[:2], \n",
+    "           current_data=eval_data[2:])\n",
+    "report "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23514f54-737b-4c8a-a68a-c34ee8ca9803",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#that's how you can see the prompt\n",
+    "\n",
+    "print(ContextQualityLLMEval(question=\"question\").get_template().get_prompt_template())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7253dced-0c84-4e27-9c97-c4bb476ef110",
+   "metadata": {},
+   "source": [
+    "### Custom descriptor over text data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c74f5f3d-56ac-42c1-b5e1-4c81411232b2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def is_empty_string_callable(val1):\n",
+    "    return pd.Series([\"EMPTY\" if val == \"\" else \"NON EMPTY\" for val in val1], index=val1.index)\n",
+    "\n",
+    "empty_string = CustomColumnEval(\n",
+    "    func=is_empty_string_callable,\n",
+    "    feature_type=\"cat\",\n",
+    "    display_name=\"Empty response\"\n",
+    ")\n",
+    "\n",
+    "report = Report(metrics=[\n",
+    "    ColumnSummaryMetric(column_name=empty_string.on(\"response\")),\n",
+    "])\n",
+    "\n",
+    "report.run(reference_data=assistant_logs[datetime(2024, 4, 8) : datetime(2024, 4, 9)][:10], \n",
+    "           current_data=assistant_logs[datetime(2024, 4, 9) : datetime(2024, 4, 10)][:10], \n",
+    "           column_mapping=column_mapping)\n",
+    "report "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82c8c30b-095c-4aeb-a87b-4fd637295fe7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def exact_match_callable(val1, val2):\n",
+    "    return pd.Series([\"MATCH\" if val else \"MISMATCH\" for val in val1 == val2])\n",
+    "\n",
+    "exact_match =  CustomPairColumnEval(\n",
+    "    func=exact_match_callable,\n",
+    "    first_column=\"response\",\n",
+    "    second_column=\"question\",\n",
+    "    feature_type=\"cat\",\n",
+    "    display_name=\"Exact match between response and question\"\n",
+    ")\n",
+    "\n",
+    "report = Report(metrics=[\n",
+    "    ColumnSummaryMetric(column_name=exact_match.as_column())\n",
+    "])\n",
+    "\n",
+    "report.run(reference_data=assistant_logs[datetime(2024, 4, 8) : datetime(2024, 4, 9)][:10], \n",
+    "           current_data=assistant_logs[datetime(2024, 4, 9) : datetime(2024, 4, 10)][:10], \n",
+    "           column_mapping=column_mapping)\n",
+    "report "
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "3806d7d8-5acf-45cb-b16b-3b4336dea6e0",
@@ -372,14 +508,6 @@
     "           column_mapping=column_mapping)\n",
     "report "
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c63c0d6e-e5fc-44ec-a1cd-ef85c7585973",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {