From 210b0605eeb0d77462252758672812d62833aca8 Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 09:25:46 -0800 Subject: [PATCH 01/11] added quickstart notebook and updated aspects of the code to improve usability --- connect/__init__.py | 2 + connect/adapters/adapters.py | 8 +- connect/governance/credo_api_client.py | 5 +- connect/governance/governance.py | 15 +- connect/utils/__init__.py | 1 + connect/utils/version_check.py | 6 +- docs/notebooks/quickstart.ipynb | 894 +++++++++++++++++++++++++ 7 files changed, 920 insertions(+), 11 deletions(-) create mode 100644 docs/notebooks/quickstart.ipynb diff --git a/connect/__init__.py b/connect/__init__.py index ee94960..66a8c15 100644 --- a/connect/__init__.py +++ b/connect/__init__.py @@ -1,6 +1,8 @@ """ Credo AI Connect package """ +from connect.adapters import Adapter +from connect.governance import Governance from connect.utils.version_check import validate_version __version__ = "0.0.4" diff --git a/connect/adapters/adapters.py b/connect/adapters/adapters.py index b4237f9..1a6a792 100644 --- a/connect/adapters/adapters.py +++ b/connect/adapters/adapters.py @@ -38,9 +38,10 @@ def __init__( def metrics_to_governance( self, metrics: dict, + source: str, labels: dict = None, metadata: dict = None, - overwrite_governance: bool = False, + overwrite_governance: bool = True, ): """ Packages metrics as evidence and sends them to governance @@ -49,14 +50,17 @@ def metrics_to_governance( --------- metrics : dict or pd.DataFrame Dictionary of metrics. Form: {metric_type: value, ...} + source : str + Label for what generated the metrics labels : dict - Additional labels to pass to underlying evidence + Additional key/value pairs to act as labels for the evidence metadata : dict Metadata to pass to underlying evidence overwrite_governance : bool When adding evidence to a Governance object, whether to overwrite existing evidence or not, default False. """ + labels = {**(labels or {}), "source": source} evidence = self._metrics_to_evidence(metrics, labels, metadata) if overwrite_governance: diff --git a/connect/governance/credo_api_client.py b/connect/governance/credo_api_client.py index 98dfd26..36c8851 100644 --- a/connect/governance/credo_api_client.py +++ b/connect/governance/credo_api_client.py @@ -9,8 +9,7 @@ from dotenv import dotenv_values from json_api_doc import deserialize, serialize -from connect import __version__ -from connect.utils import global_logger, json_dumps +from connect.utils import get_version, global_logger, json_dumps CREDO_URL = "https://api.credo.ai" @@ -133,7 +132,7 @@ def set_access_token(self, access_token): "accept": "application/vnd.api+json", "content-type": "application/vnd.api+json", "X-Client-Name": "Credo AI Connect", - "X-Client-Version": __version__, + "X-Client-Version": get_version(), } self._session.headers.update(headers) diff --git a/connect/governance/governance.py b/connect/governance/governance.py index ba57c3c..583ecc9 100644 --- a/connect/governance/governance.py +++ b/connect/governance/governance.py @@ -10,9 +10,14 @@ from json_api_doc import deserialize, serialize -from connect import __version__ from connect.evidence import Evidence, EvidenceRequirement -from connect.utils import check_subset, global_logger, json_dumps, wrap_list +from connect.utils import ( + check_subset, + get_version, + global_logger, + json_dumps, + wrap_list, +) from .credo_api import CredoApi from .credo_api_client import CredoApiClient @@ -171,7 +176,7 @@ def get_evidence_requirements(self, tags: dict = None, verbose=False): self._print_evidence(reqs) return reqs - def get_evidence_tags(self): + def get_requirement_tags(self): """Return the unique tags used for all evidence requirements""" return self._unique_tags @@ -318,7 +323,7 @@ def set_evidence(self, evidences: List[Evidence]): def tag_model(self, model): """Interactive utility to tag a model tags from assessment plan""" - tags = self.get_evidence_tags() + tags = self.get_requirement_tags() print(f"Select tag from assessment plan to associated with model:") print("0: No tags") for number, tag in enumerate(tags): @@ -393,7 +398,7 @@ def _file_export(self, filename): f"Saving {len(self._evidences)} evidences to {filename}.. for use_case_id={self._use_case_id} policy_pack_id={self._policy_pack_id} " ) data = self._prepare_export_data() - meta = {"client": "Credo AI Connect", "version": __version__} + meta = {"client": "Credo AI Connect", "version": get_version()} data = json_dumps(serialize(data=data, meta=meta)) with open(filename, "w") as f: f.write(data) diff --git a/connect/utils/__init__.py b/connect/utils/__init__.py index 4310848..fd8bfc5 100644 --- a/connect/utils/__init__.py +++ b/connect/utils/__init__.py @@ -5,3 +5,4 @@ from .common import * from .data_scrubbing import Scrubber from .logging import * +from .version_check import get_version diff --git a/connect/utils/version_check.py b/connect/utils/version_check.py index 05fa1ec..b83b4fa 100644 --- a/connect/utils/version_check.py +++ b/connect/utils/version_check.py @@ -4,8 +4,12 @@ from connect.utils import global_logger +def get_version(): + return connect.__version__ + + def validate_version(): - current_version = connect.__version__ + current_version = get_version() package = "credoai-connect" response = requests.get(f"https://pypi.org/pypi/{package}/json") diff --git a/docs/notebooks/quickstart.ipynb b/docs/notebooks/quickstart.ipynb new file mode 100644 index 0000000..ecd7b42 --- /dev/null +++ b/docs/notebooks/quickstart.ipynb @@ -0,0 +1,894 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "cb8273f5", + "metadata": {}, + "source": [ + "\n", + "Click here to download this notebook." + ] + }, + { + "cell_type": "markdown", + "id": "858e3872-01c7-4dc5-bc70-eb9a821c3466", + "metadata": { + "tags": [] + }, + "source": [ + "\n", + "\n", + "# Credo AI Connect\n", + "\n", + "**Credo AI Connect** is the library to interact with the Credo AI Governance Platform. It is responsible for receiving \"evidence requirements\" and sending \"evidence\"\n", + "\n", + "Get started here. We will assess a payment default prediction model, assess it, and send the results to the [Credo AI Governance Platform](https://www.credo.ai/). For a tool to help you assess your AI systems, see [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html).\n", + "\n", + "**Setup**\n", + "\n", + "Connect installation instruction can be found on [readthedocs](https://credoai-connect.readthedocs.io/en/stable/setup.html)\n", + "\n", + "**Find the code**" + ] + }, + { + "cell_type": "raw", + "id": "e5793e90-e1aa-4e5a-a723-7e67b908aa18", + "metadata": {}, + "source": [ + "\n", + "Click here to download this notebook." + ] + }, + { + "cell_type": "markdown", + "id": "2feee674-2dce-4d74-bfbb-d8e58d84415f", + "metadata": {}, + "source": [ + "## Useful keywords\n", + "\n", + "- **Credo AI Platform**: Also referred to as simply \"Platform\". The central AI governance/reporting Platform, found at [https://app.credo.ai/](https://app.credo.ai/)\n", + "\n", + "- **credoconfig**: configuration file to be copied in the user's home folder\n", + "\n", + "- **use_case_name**: The name of your Use Case as it is registered on Credo AI Platform\n", + "\n", + "- **policy_pack**: A set of governance controls that a Use Case needs to satisfy. A Use Case can have multiple policy packs applied to it.\n", + "\n", + "- **policy_pack_key**: A unique identifier for a policy pack (where do we get this?)\n", + "\n", + "- **assessment_plan_url**: The link to the assessment plan, this is generated in the Platform and used to download the assessment plan in the Governance object. See example below.\n", + "\n", + "- **evidence**: Any evaluation of an AI system, formatted specifically to be uploaded to the platform.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ef6d3ba0-6133-474c-97cf-9be99d7bf950", + "metadata": {}, + "source": [ + "## Setup API Connection with the Platform" + ] + }, + { + "cell_type": "markdown", + "id": "4255288f-0ffa-4050-b62c-ee9dcd7a9dd3", + "metadata": {}, + "source": [ + "### Get a config file\n", + "This file contains all the necessary information to connect Lens to the Credo AI Platform. \n", + "\n", + "To generate the config file, once you logged into the platform, click on your name icon (top left) and follow: \n", + "\n", + "`My Settings -> Tokens -> Plus Sign -> Generate`\n", + "\n", + "Immediately after generating the token, you will be given the possibility to download the config file.\n", + "\n", + "The default location/file name Lens is expecting is `~/.credoconfig`, where `~` is your home folder. You can specify any other location when you are initializing the `Governance` object (see below)." + ] + }, + { + "cell_type": "markdown", + "id": "b3bcfb4b-3249-499d-86d0-400d82823802", + "metadata": {}, + "source": [ + "### Get your ML environment ready\n", + " In this tutorial we will emulate the modeling phase by running a quick script. This script loads a dataset, splits it into training and testing, and fits a model. You can see the full script [here](https://github.com/credo-ai/credoai_lens/blob/release/1.0.0/docs/notebooks/training_script.py)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "47e05093-f918-4e8a-8636-47eac6f0897c", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "import connect as ct" + ] + }, + { + "cell_type": "markdown", + "id": "b592bab4-843a-461d-af9f-5a38ee895ed3", + "metadata": {}, + "source": [ + "## Do some AI stuff!" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fd155239-7dc8-4576-91e6-cc09e4463204", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "import sklearn.metrics as sk_metrics\n", + "from sklearn.model_selection import train_test_split\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "33445116-3ad6-48f1-970d-e9e436d9d7ea", + "metadata": {}, + "source": [ + "### Get data and train model\n", + "\n", + "We'll make up some data and fit a simple model to get us started." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "62531aa3-11bd-489d-a3cf-a805b327fe46", + "metadata": {}, + "outputs": [], + "source": [ + "model = LinearRegression()\n", + "r = np.random.RandomState(42)\n", + "\n", + "# hallucinate data\n", + "N = 10000\n", + "data = r.randn(N, 2)\n", + "y = (data[:,0] + 3*data[:,1] + r.randn(N)*.2)\n", + "# split\n", + "X_train, X_test, y_train, y_test = train_test_split(data, y)\n", + "\n", + "# train model\n", + "model.fit(X_train, y_train)\n", + "pred_test = model.predict(X_test)" + ] + }, + { + "cell_type": "markdown", + "id": "69153277-5e82-4c76-9b92-9dba5f254323", + "metadata": {}, + "source": [ + "### Assess the AI System\n", + "\n", + "The most likely thing you'll want to send to the Credo AI Platform are _assessments_ of the AI system. These are often _metrics_ that are calculated to summarize aspects of the system's behavior (e.g., performance, fairness) but can be other things too, like descriptive statistics of the dataset.\n", + "\n", + "We'll assume we have a suite of assessments we plan to use for this use-case." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a255a36d-c26b-463e-80e5-699961c789ef", + "metadata": {}, + "outputs": [], + "source": [ + "required_assessments = [sk_metrics.r2_score, sk_metrics.mean_squared_error]\n", + "\n", + "assessments = {m.__name__: m(y_test, pred_test) for m in required_assessments}" + ] + }, + { + "cell_type": "markdown", + "id": "4fb1c630-4f10-460c-97fc-85330360f070", + "metadata": { + "tags": [] + }, + "source": [ + "## Platform integration in 5 Minutes\n", + "\n", + "To send evidence to the platform takes three steps:\n", + "\n", + "1. Connect to the platform via the Governance class\n", + "2. Use the Adapter class to send the assessments to the Governance class\n", + "3. Export the evidence to the platform.\n", + "\n", + "First we will see all the code together, and then break it down." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ec32fc49-cbf1-4674-836d-91dfc190a416", + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "ename": "HTTPError", + "evalue": "404 Client Error: Not Found for url: https://dev-api.dev-uswe2.credoai.net/api/v2/credoai/your%20assessment%20url", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/w8/gwbk059971v4hy3x2q4vrsy00000gn/T/ipykernel_6322/4162864874.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mgov\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGovernance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Specify config_path if your config file is not the default one: ~/.credoconfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'your assessment url'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mgov\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massessment_plan_url\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;31m# set up adapter and send metrics to governance class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Credo/credoai_connect/connect/governance/governance.py\u001b[0m in \u001b[0;36mregister\u001b[0;34m(self, assessment_plan_url, use_case_name, policy_pack_key, assessment_plan, assessment_plan_file)\u001b[0m\n\u001b[1;32m 244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0massessment_plan_url\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m \u001b[0mplan\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_assessment_plan\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massessment_plan_url\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 248\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0massessment_plan\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Credo/credoai_connect/connect/governance/credo_api.py\u001b[0m in \u001b[0;36mget_assessment_plan\u001b[0;34m(self, url)\u001b[0m\n\u001b[1;32m 105\u001b[0m \"\"\"\n\u001b[1;32m 106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_assessment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_case_id\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Credo/credoai_connect/connect/governance/credo_api_client.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0mSend\u001b[0m \u001b[0mget\u001b[0m \u001b[0mrequest\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mretult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m \"\"\"\n\u001b[0;32m--> 165\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"get\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Credo/credoai_connect/connect/governance/credo_api_client.py\u001b[0m in \u001b[0;36m__make_request\u001b[0;34m(self, method, path, **kwargs)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mendpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/test/lib/python3.9/site-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mHTTPError\u001b[0m: 404 Client Error: Not Found for url: https://dev-api.dev-uswe2.credoai.net/api/v2/credoai/your%20assessment%20url" + ] + } + ], + "source": [ + "# connect to platform via governace\n", + "gov = ct.Governance() # Specify config_path if your config file is not the default one: ~/.credoconfig\n", + "url = 'your assessment url'\n", + "gov.register(assessment_plan_url=url)\n", + "\n", + "# set up adapter and send metrics to governance class\n", + "adapter = ct.Adapter(governance = gov, model_name='My Model')\n", + "source = f\"Quickstart_Connect-{ct.__version__}\"\n", + "adapter.metrics_to_governance(metrics=assessments, source = source)\n", + "\n", + "# export\n", + "gov.export()" + ] + }, + { + "cell_type": "markdown", + "id": "6c69d81a-83d8-4e76-afd9-c16b51f6ff68", + "metadata": {}, + "source": [ + "#### 1. Connect to the platform via the Governance Class\n", + "\n", + "The Governance class handles the connection with the Credo AI Platform. On the Platform, you can govern an AI system by specifying \"policy packs\" that specify the technical requirements the AI system must meet. This class can retrieve them, which is most useful if you are using [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html), our assessment framework. Since we are only using `Connect` in this demo, we will ignore that functionality.\n", + "\n", + "The important functionality relevant here is that the Governance object handles the API calls allowing you to _send_ evidence to the platform. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6df1b60f-48d7-4f6c-9c7d-adb3f166f734", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-12-12 09:18:57,216 - connect - INFO - Successfully registered with 8 evidence requirements\n" + ] + } + ], + "source": [ + "# Retrieve Policy Pack Assessment Plan\n", + "gov = ct.Governance() # Specify config_path if your config file is not the default one: ~/.credoconfig\n", + "url = 'your assessment url'\n", + "gov.register(assessment_plan_url=url)" + ] + }, + { + "cell_type": "markdown", + "id": "12a8ff57-099a-4e63-b2bc-6a6da6779ce2", + "metadata": {}, + "source": [ + "#### 2. Use the Adapter class to send the assessments to the Governance class\n", + "\n", + "The `Adapter` class handles structuring your assessments so that the Credo AI Platform can understand them. Connect uses `EvidenceContainers` to handle converting python objects like dictionaries into `Evidence`, the structured output that the Platform can understand. The Adapter then passes this `Evidence` to the `Governance` object for export" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "4b0fa6b7-2daf-4061-aa37-7e036750c124", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-12-12 09:19:04,413 - connect - INFO - Adding model (My Model) to governance. Model has tags: None\n" + ] + } + ], + "source": [ + "adapter = ct.Adapter(governance = gov, model_name='My Model')\n" + ] + }, + { + "cell_type": "markdown", + "id": "1dfb4be0-eee6-4a9b-8ffa-412b375d5636", + "metadata": {}, + "source": [ + "Once we initialize an adapter, we can use its functionality to send different kinds of evidence.\n", + "\n", + "For instance, metrics (which must be organized as key:value pairs of metric_name:value) are sent to governance using `metrics_to_governance`\n", + "This converts the dictionary of assessments into `Evidence` (in this case a `MetricEvidence`).\n", + "\n", + "Every time we send evidence we specify a source. This is for you to define however you would like. We suggest you make the source useful to establish provenance of your assessments. " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "deb31c08-40a2-4341-b795-5e37333edfc4", + "metadata": {}, + "outputs": [], + "source": [ + "source = f\"Quickstart_Connect-{ct.__version__}\"\n", + "adapter.metrics_to_governance(metrics=assessments, source = source)" + ] + }, + { + "cell_type": "markdown", + "id": "b90b05c4-c541-442d-aa41-79ba567167f3", + "metadata": {}, + "source": [ + "You can see the evidence in the `Governance` class we instantiated before" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "fd32329f-44d0-400d-ba61-0e090e52152f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[,\n", + " ]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gov.get_evidence()" + ] + }, + { + "cell_type": "markdown", + "id": "8dfab5cf-311b-43d6-a846-b6f4486f51c4", + "metadata": {}, + "source": [ + "If you need to send more evidence (perhaps a different set of metrics), you'll just call a function like `metrics_to_governance` again. By default, this functions overwright the `Evidence` in `Governance` so you'll have to change a keyword." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "41c399ec-d05a-4c4a-987c-bf9584a0d6ac", + "metadata": {}, + "outputs": [], + "source": [ + "additional_assessments = {'MAE': sk_metrics.mean_absolute_error(y_test, pred_test)}\n", + "\n", + "adapter.metrics_to_governance(metrics=additional_assessments, \n", + " source=source,\n", + " overwrite_governance=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "9f3061d6-4e48-423d-94cf-a599839c6add", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[,\n", + " ,\n", + " ]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gov.get_evidence()" + ] + }, + { + "cell_type": "markdown", + "id": "6e4364e8-d5da-436e-a0b3-e38a1b283f11", + "metadata": {}, + "source": [ + "#### 3. Export the evidence to the platform.\n", + "\n", + "Exporting is straight forward! You can either export directly to the Platform or to a file. At the time of export, the uploaded evidence will be checked against the governance requirements specified on Platform and let you know what's missing." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "bddb93de-6139-4f22-9df0-b97019157541", + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-12-12 09:19:49,883 - connect - INFO - Missing required evidence with label ({'evaluator': 'ModelEquity', 'metric_type': 'demographic_parity_ratio', 'sensitive_feature': 'RACE'}).\n", + "2022-12-12 09:19:49,885 - connect - INFO - Missing required evidence with label ({'evaluator': 'ModelEquity', 'metric_type': 'demographic_parity_ratio', 'sensitive_feature': 'ETHNICITY'}).\n", + "2022-12-12 09:19:49,885 - connect - INFO - Missing required evidence with label ({'evaluator': 'ModelEquity', 'metric_type': 'demographic_parity_ratio', 'sensitive_feature': 'SEX'}).\n", + "2022-12-12 09:19:49,886 - connect - INFO - Missing required evidence with label ({'metric_type': 'equal_opportunity_difference'}).\n", + "2022-12-12 09:19:49,887 - connect - INFO - Missing required evidence with label ({'metric_type': 'selection_rate'}).\n", + "2022-12-12 09:19:49,887 - connect - INFO - Missing required evidence with label ({'metric_type': 'accuracy_score'}).\n", + "2022-12-12 09:19:49,887 - connect - INFO - Missing required evidence with label ({'metric_type': 'soc'}).\n", + "2022-12-12 09:19:49,888 - connect - INFO - Missing required evidence with label ({'data_type': 'disaggregated_performance'}).\n", + "2022-12-12 09:19:49,888 - connect - INFO - Uploading 3 evidences.. for use_case_id=BRu2eLMdpwqqgYnab359mY policy_pack_id=NYCE+1\n", + "2022-12-12 09:19:51,125 - connect - INFO - 3 evidences were successfuly uploaded, took 224.652 ms\n", + "2022-12-12 09:19:51,126 - connect - INFO - Partial match of requirements.\n" + ] + }, + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# export to API\n", + "gov.export()\n", + "\n", + "# export to file\n", + "gov.export(\"assessment_export.json\")" + ] + }, + { + "cell_type": "markdown", + "id": "d6c71569-6152-42d6-aadc-da60a25eca52", + "metadata": { + "tags": [] + }, + "source": [ + "## Other ways to label your evidence" + ] + }, + { + "cell_type": "markdown", + "id": "21ea5e8a-2cb1-41ec-aafb-a10299774725", + "metadata": {}, + "source": [ + "On the Credo AI Platform, certain governance requirements may only apply to models that are tagged in a certain way. You can get the requirements tags from governance and apply it by passing a dictionary to the `model_tags` of the `Adapter`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd269b76-3fb7-4288-8b0a-bf16772b5b6a", + "metadata": {}, + "outputs": [], + "source": [ + "gov.get_requirement_tags()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "861cdace-f849-4248-bdd7-cd5a86888902", + "metadata": {}, + "outputs": [], + "source": [ + "adapter = Adapter(governance = gov, model_name='My Model', model_tags={'model_type': 'regression'})" + ] + }, + { + "cell_type": "markdown", + "id": "ee605487-1de4-4fe7-8df2-cb8947a260cd", + "metadata": {}, + "source": [ + "If you need to label your evidence with more information than just `source` you can do that too." + ] + }, + { + "cell_type": "markdown", + "id": "c0797a51", + "metadata": {}, + "source": [ + "## Lens in 5 minutes" + ] + }, + { + "cell_type": "markdown", + "id": "cce779fd", + "metadata": { + "tags": [] + }, + "source": [ + "Below is a basic example where our goal is to evaluate the above model. We'll break down this code [below](#Breaking-Down-The-Steps).\n", + "\n", + "Briefly, the code is doing four things:\n", + "\n", + "* Wrapping ML artifacts (like models and data) in Lens objects\n", + "* Initializing an instance of Lens. Lens is the main object that performs evaluations. Under the hood, it creates a `pipeline` of evaluations that are run.\n", + "* Add evaluators to Lens.\n", + "* Run Lens" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b4e6a6a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# set up model and data artifacts\n", + "credo_model = ClassificationModel(name=\"credit_default_classifier\", model_like=model)\n", + "credo_data = TabularData(\n", + " name=\"UCI-credit-default\",\n", + " X=X_test,\n", + " y=y_test,\n", + " sensitive_features=sensitive_features_test,\n", + ")\n", + "\n", + "# Initialization of the Lens object\n", + "lens = Lens(model=credo_model, assessment_data=credo_data)\n", + "\n", + "# initialize the evaluator and add it to Lens\n", + "metrics = ['precision_score', 'recall_score', 'equal_opportunity']\n", + "lens.add(ModelFairness(metrics=metrics))\n", + "lens.add(Performance(metrics=metrics))\n", + "\n", + "# run Lens\n", + "lens.run()" + ] + }, + { + "cell_type": "markdown", + "id": "c188b100-2ebb-42c8-9451-2812fe7383d6", + "metadata": {}, + "source": [ + "### Getting results within your python environment\n", + "\n", + "`lens.get_results()` provides a list where the results of the evaluators (a list of dataframes) are stored along with the evaluator metadata. In this case, there are 2 results - one for each evaluator." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7eeac730-29bd-4300-8e02-d8677e10123e", + "metadata": {}, + "outputs": [], + "source": [ + "results = lens.get_results()\n", + "print(f\"Results for {len(results)} evaluators\")" + ] + }, + { + "cell_type": "markdown", + "id": "8fa1eab0", + "metadata": {}, + "source": [ + "`lens.get_results()` has some arguments, which makes it easier for you to get a subset of results. These are the same arguments that can be passed to `lens.get_pipeline` and `lens.get_evidence`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7735dd2-5797-45aa-90f2-212faa063edd", + "metadata": {}, + "outputs": [], + "source": [ + "lens.get_results(evaluator_name='Performance')" + ] + }, + { + "cell_type": "markdown", + "id": "94ca7568", + "metadata": {}, + "source": [ + "## Using Len's pipeline argument" + ] + }, + { + "cell_type": "markdown", + "id": "498a11b3", + "metadata": {}, + "source": [ + "If we want to add multiple evaluators to our pipeline, one way of doing it could be repeating the `add` step, as shown above. Another way is to define the pipeline steps, and pass it to `Lens` at initialization time. Let's explore the latter!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f506b7d8", + "metadata": {}, + "outputs": [], + "source": [ + "pipeline = [\n", + " (ModelFairness(metrics)),\n", + " (Performance(metrics)),\n", + "]\n", + "lens = Lens(model=credo_model, assessment_data=credo_data, pipeline=pipeline)" + ] + }, + { + "cell_type": "markdown", + "id": "aa465d26", + "metadata": {}, + "source": [ + "Above, each of the `tuples` in the `list` is in the form `(instantiated_evaluator, id)`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c46bdd9e", + "metadata": {}, + "outputs": [], + "source": [ + "# notice that Lens functions can be chained together\n", + "results = lens.run().get_results()\n", + "print(f'\\nFound results for {len(results)} evaluators')" + ] + }, + { + "cell_type": "markdown", + "id": "601f5203", + "metadata": {}, + "source": [ + "Let's check that we have results for both of our evaluators." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ab0ed632", + "metadata": {}, + "outputs": [], + "source": [ + "results[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bb680a5", + "metadata": {}, + "outputs": [], + "source": [ + "results[1]" + ] + }, + { + "cell_type": "markdown", + "id": "bb18d57e", + "metadata": {}, + "source": [ + "## That's it!\n", + "\n", + "That should get you up and running. Next steps include:\n", + "\n", + "* Trying out other evaluators (they are all accessible via `credoai.evaluators`)\n", + "* Checking out our developer guide to better understand the Lens ecosystem and see how you can extend it.\n", + "* Exploring the Credo AI Governance Platform, which will connect AI assessments with customizable governance to support reporting, compliance, multi-stakeholder translation and more!" + ] + }, + { + "cell_type": "markdown", + "id": "c6c05dfe", + "metadata": { + "tags": [] + }, + "source": [ + "## Breaking Down The Steps\n", + "\n", + "### Preparing artifacts\n", + "\n", + "Lens interacts with \"AI Artifacts\" which wrap model and data objects and standardize them for use by different evaluators.\n", + "\n", + "Below we create a `ClassificationModel` artifact. This is a light wrapper for any kind of fitted classification model-like object. \n", + "\n", + "We also create a `TabularData` artifact which stores X, y and sensitive features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4f1fae7", + "metadata": {}, + "outputs": [], + "source": [ + "# set up model and data artifacts\n", + "credo_model = ClassificationModel(name=\"credit_default_classifier\", model_like=model)\n", + "\n", + "credo_data = TabularData(\n", + " name=\"UCI-credit-default\",\n", + " X=X_test,\n", + " y=y_test,\n", + " sensitive_features=sensitive_features_test,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "114bd0bb", + "metadata": { + "tags": [] + }, + "source": [ + "#### Model\n", + "\n", + "Model type objects, like `ClassificationModel` used above, serve as adapters between arbitrary models and the evaluators in Lens. Some evaluators depend on Model instantiating certain methods. For example, `ClassificationModel` can accept any generic object having `predict` and `predict_proba` methods, including fitted sklearn pipelines.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "db7fb09b", + "metadata": { + "tags": [] + }, + "source": [ + "#### Data\n", + "\n", + "_Data_ type artifact, like `TabularData` serve as adapters between datasets and the evaluators in Lens.\n", + "\n", + "When you pass data to a _Data_ artifact, the artifact performs various steps of validation, and formats them so that they can be used by evaluators. The aim of this procedure is to preempt errors down the line.\n", + "\n", + "You can pass Data to Lens as a **training dataset** or an **assessment dataset** (see lens class documentation). If the former, it will not be used to assess the model. Instead, dataset assessments will be performed on the dataset (e.g., fairness assessment). The validation dataset will be assessed in the same way, but _also_ used to assess the model, if provided.\n", + "\n", + "Similarly to _Model_ type objects, _Data_ objects can be customized, see !!insertlink!!" + ] + }, + { + "cell_type": "markdown", + "id": "e4583249", + "metadata": {}, + "source": [ + "### Evaluators \n", + "\n", + "Lens uses the above artifacts to ensure a successfull run of the evaluators. As we have seen in the sections [Lens in 5 minutes](##Lens-in-5-minutes) and [Adding multiple evaluators](##Adding-multiple-evaluators), multiple evaluators can be added to _Lens_ pipeline. Each evaluators contains information on what it needs in order to run successfully, and it executes a validation step at _add_ time.\n", + "\n", + "The result of the validation depends on what artifacts are available, their content and the type of evaluator being added to the pipeline. In case the validation process fails, the user is notified the reason why the evaluator cannot be added to the pipeline.\n", + "\n", + "See for example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "836cccd0", + "metadata": {}, + "outputs": [], + "source": [ + "from credoai.evaluators import Privacy\n", + "lens.add(Privacy())" + ] + }, + { + "cell_type": "markdown", + "id": "e76369e8", + "metadata": {}, + "source": [ + "Currently no automatic run of evaluators is supported. However, when Lens is used in combination with Credo AI Platform, it is possible to download an assessment plan which then gets converted into a set of evaluations that Lens can run programmatically. For more information see the [governance tutorial](https://credoai-lens.readthedocs.io/en/stable/notebooks/governance_integration.html)." + ] + }, + { + "cell_type": "markdown", + "id": "cdb3c5b8", + "metadata": {}, + "source": [ + "### Run Lens\n", + "\n", + "After we have initialized _Lens_ the _Model_ and _Data_ (`ClassificationModel` and `TabularData` in our example) type artifacts, we can add whichever evaluators we want to the pipeline, and finally run it!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc76430f", + "metadata": {}, + "outputs": [], + "source": [ + "lens = Lens(model=credo_model, assessment_data=credo_data)\n", + "metrics = ['precision_score', 'recall_score', 'equal_opportunity']\n", + "lens.add(ModelFairness(metrics=metrics))\n", + "lens.run()" + ] + }, + { + "cell_type": "markdown", + "id": "673dbbc9", + "metadata": {}, + "source": [ + "As you can notice, when adding _evaluators_ to lens, they need to be instantiated. If any extra arguments need to be passed to the evaluator (like metrics in this case), this is the time to do it." + ] + }, + { + "cell_type": "markdown", + "id": "2a5dcbe3", + "metadata": {}, + "source": [ + "**Getting Evaluator Results**\n", + "\n", + "Afte the pipeline is run, the results become accessible via the method `get_results()`\n", + "\n", + "`lens.get_results()` provides a dictionary where the results of the evaluators are stored as values, and the keys correspond to the ids of the evaluators. \n", + "\n", + "In the previous case we specified the id of the evaluator when we added `ModelFairness` to the pipeline, however `id` is an optional argument for the `add` method. If omitted, a random one will be generated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "031d237a", + "metadata": {}, + "outputs": [], + "source": [ + "lens.get_results()" + ] + }, + { + "cell_type": "markdown", + "id": "ac3bfe1a", + "metadata": {}, + "source": [ + "**Credo AI Governance Platform**\n", + "\n", + "For information on how to interact with the plaform, please look into: [Connecting with Governance App](https://credoai-lens.readthedocs.io/en/stable/notebooks/governance_integration.html) tutorial for directions.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "507a61bde0a0183a71b2d35939f461921273a091e2cc4517af66dd70c4baafc9" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 0b64f471c7ef28ea991ff60ef8136e3c1a6184ac Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 09:45:46 -0800 Subject: [PATCH 02/11] added table evidence --- connect/adapters/adapters.py | 91 +++++++++++++++++-- docs/notebooks/quickstart.ipynb | 149 ++++++-------------------------- 2 files changed, 111 insertions(+), 129 deletions(-) diff --git a/connect/adapters/adapters.py b/connect/adapters/adapters.py index 1a6a792..8463201 100644 --- a/connect/adapters/adapters.py +++ b/connect/adapters/adapters.py @@ -1,10 +1,11 @@ +from functools import partial from typing import Optional import pandas as pd -from connect.evidence import MetricContainer +from connect.evidence import MetricContainer, TableContainer from connect.governance import Governance -from connect.utils import ValidationError +from connect.utils import ValidationError, wrap_list class Adapter: @@ -60,9 +61,80 @@ def metrics_to_governance( When adding evidence to a Governance object, whether to overwrite existing evidence or not, default False. """ - labels = {**(labels or {}), "source": source} - evidence = self._metrics_to_evidence(metrics, labels, metadata) + self._evidence_to_governance( + self._metrics_to_evidence, + metrics, + source, + labels, + metadata, + overwrite_governance, + ) + + def table_to_governance( + self, + data: dict, + source: str, + labels: dict = None, + metadata: dict = None, + overwrite_governance: bool = True, + ): + """ + Packages metrics as evidence and sends them to governance + + Parameters + --------- + data: pd.DataFrame + Dataframe to pass to evidence_fun. The DataFrame must have a "name" attribute + source : str + Label for what generated the table + labels : dict + Additional key/value pairs to act as labels for the evidence + metadata : dict + Metadata to pass to underlying evidence + overwrite_governance : bool + When adding evidence to a Governance object, whether to overwrite existing + evidence or not, default False. + evidence_fun : callable + Function to pass data, labels and metadata. The function should return a list of + evidence. Default: self._to_evidence + """ + self._evidence_to_governance( + TableContainer, data, source, labels, metadata, overwrite_governance + ) + def _evidence_to_governance( + self, + evidence_fun, + data, + source, + labels, + metadata, + overwrite_governance=True, + ): + """ + Packages data as evidence and sends to governance + + Parameters + --------- + evidence_fun : callable or Container + Function to pass data, labels and metadata. The function should return a list of + evidence. If a Container, use self._to_evidence with the specified container + data + data to pass to evidence_fun + source : str + Label for what generated the table + labels : dict + Additional key/value pairs to act as labels for the evidence + metadata : dict + Metadata to pass to underlying evidence + overwrite_governance : bool + When adding evidence to a Governance object, whether to overwrite existing + evidence or not, default False. + """ + if evidence_fun is None: + evidence_fun = partial(self._to_evidence, container_class=evidence_fun) + labels = {**(labels or {}), "source": source} + evidence = evidence_fun(data, labels, metadata) if overwrite_governance: self.governance.set_evidence(evidence) else: @@ -93,11 +165,14 @@ def _metrics_to_evidence(self, metrics, labels=None, metadata=None): List list of Evidence """ - meta = self._get_artifact_meta() - meta.update(metadata or {}) if isinstance(metrics, dict): metrics = pd.DataFrame(metrics.items(), columns=["type", "value"]) elif not isinstance(metrics, pd.DataFrame): raise ValidationError("Metrics must be a dictionary or a dataframe") - container = MetricContainer(metrics, labels, meta) - return container.to_evidence() + return self._to_evidence(MetricContainer, metrics, labels, metadata) + + def _to_evidence(self, container_class, data, labels, metadata): + meta = self._get_artifact_meta() + meta.update(metadata or {}) + container = container_class(data, labels, meta) + return wrap_list(container.to_evidence()) diff --git a/docs/notebooks/quickstart.ipynb b/docs/notebooks/quickstart.ipynb index ecd7b42..2ecec38 100644 --- a/docs/notebooks/quickstart.ipynb +++ b/docs/notebooks/quickstart.ipynb @@ -98,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "47e05093-f918-4e8a-8636-47eac6f0897c", "metadata": {}, "outputs": [], @@ -118,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "fd155239-7dc8-4576-91e6-cc09e4463204", "metadata": {}, "outputs": [], @@ -141,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "62531aa3-11bd-489d-a3cf-a805b327fe46", "metadata": {}, "outputs": [], @@ -175,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "a255a36d-c26b-463e-80e5-699961c789ef", "metadata": {}, "outputs": [], @@ -205,33 +205,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ec32fc49-cbf1-4674-836d-91dfc190a416", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, - "outputs": [ - { - "ename": "HTTPError", - "evalue": "404 Client Error: Not Found for url: https://dev-api.dev-uswe2.credoai.net/api/v2/credoai/your%20assessment%20url", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/w8/gwbk059971v4hy3x2q4vrsy00000gn/T/ipykernel_6322/4162864874.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mgov\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGovernance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Specify config_path if your config file is not the default one: ~/.credoconfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'your assessment url'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mgov\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massessment_plan_url\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;31m# set up adapter and send metrics to governance class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Credo/credoai_connect/connect/governance/governance.py\u001b[0m in \u001b[0;36mregister\u001b[0;34m(self, assessment_plan_url, use_case_name, policy_pack_key, assessment_plan, assessment_plan_file)\u001b[0m\n\u001b[1;32m 244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0massessment_plan_url\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m \u001b[0mplan\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_assessment_plan\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massessment_plan_url\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 248\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0massessment_plan\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Credo/credoai_connect/connect/governance/credo_api.py\u001b[0m in \u001b[0;36mget_assessment_plan\u001b[0;34m(self, url)\u001b[0m\n\u001b[1;32m 105\u001b[0m \"\"\"\n\u001b[1;32m 106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcreate_assessment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_case_id\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Credo/credoai_connect/connect/governance/credo_api_client.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0mSend\u001b[0m \u001b[0mget\u001b[0m \u001b[0mrequest\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mretult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m \"\"\"\n\u001b[0;32m--> 165\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__make_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"get\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpost\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Credo/credoai_connect/connect/governance/credo_api_client.py\u001b[0m in \u001b[0;36m__make_request\u001b[0;34m(self, method, path, **kwargs)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mendpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/test/lib/python3.9/site-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mHTTPError\u001b[0m: 404 Client Error: Not Found for url: https://dev-api.dev-uswe2.credoai.net/api/v2/credoai/your%20assessment%20url" - ] - } - ], + "outputs": [], "source": [ "# connect to platform via governace\n", "gov = ct.Governance() # Specify config_path if your config file is not the default one: ~/.credoconfig\n", @@ -261,20 +240,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "6df1b60f-48d7-4f6c-9c7d-adb3f166f734", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2022-12-12 09:18:57,216 - connect - INFO - Successfully registered with 8 evidence requirements\n" - ] - } - ], + "outputs": [], "source": [ "# Retrieve Policy Pack Assessment Plan\n", "gov = ct.Governance() # Specify config_path if your config file is not the default one: ~/.credoconfig\n", @@ -294,18 +265,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "4b0fa6b7-2daf-4061-aa37-7e036750c124", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2022-12-12 09:19:04,413 - connect - INFO - Adding model (My Model) to governance. Model has tags: None\n" - ] - } - ], + "outputs": [], "source": [ "adapter = ct.Adapter(governance = gov, model_name='My Model')\n" ] @@ -325,7 +288,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "deb31c08-40a2-4341-b795-5e37333edfc4", "metadata": {}, "outputs": [], @@ -344,22 +307,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "fd32329f-44d0-400d-ba61-0e090e52152f", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[,\n", - " ]" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "gov.get_evidence()" ] @@ -369,42 +320,30 @@ "id": "8dfab5cf-311b-43d6-a846-b6f4486f51c4", "metadata": {}, "source": [ - "If you need to send more evidence (perhaps a different set of metrics), you'll just call a function like `metrics_to_governance` again. By default, this functions overwright the `Evidence` in `Governance` so you'll have to change a keyword." + "If you need to send more evidence, you'll just call a function like `metrics_to_governance` again. By default, this functions overwright the `Evidence` in `Governance` so you'll have to change an argument to allow overwriting.\n", + "\n", + "We will send a different type of evidence - a table, which must be a pandas DataFrame." ] }, { "cell_type": "code", - "execution_count": 17, - "id": "41c399ec-d05a-4c4a-987c-bf9584a0d6ac", + "execution_count": null, + "id": "a6861fda-a0b4-49b4-8f53-c8f187da0c26", "metadata": {}, "outputs": [], "source": [ - "additional_assessments = {'MAE': sk_metrics.mean_absolute_error(y_test, pred_test)}\n", - "\n", - "adapter.metrics_to_governance(metrics=additional_assessments, \n", - " source=source,\n", - " overwrite_governance=False)" + "import pandas as pd\n", + "table = pd.DataFrame({'arbitrary_data': [3,4,5]})\n", + "table.name = 'my_table'\n", + "adapter.table_to_governance(table, source=source, overwrite_governance=False)" ] }, { "cell_type": "code", - "execution_count": 18, - "id": "9f3061d6-4e48-423d-94cf-a599839c6add", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[,\n", - " ,\n", - " ]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "id": "9da37aa7-5305-4ec9-8c49-a11d32116f16", + "metadata": {}, + "outputs": [], "source": [ "gov.get_evidence()" ] @@ -421,44 +360,12 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "bddb93de-6139-4f22-9df0-b97019157541", "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true - }, "tags": [] }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2022-12-12 09:19:49,883 - connect - INFO - Missing required evidence with label ({'evaluator': 'ModelEquity', 'metric_type': 'demographic_parity_ratio', 'sensitive_feature': 'RACE'}).\n", - "2022-12-12 09:19:49,885 - connect - INFO - Missing required evidence with label ({'evaluator': 'ModelEquity', 'metric_type': 'demographic_parity_ratio', 'sensitive_feature': 'ETHNICITY'}).\n", - "2022-12-12 09:19:49,885 - connect - INFO - Missing required evidence with label ({'evaluator': 'ModelEquity', 'metric_type': 'demographic_parity_ratio', 'sensitive_feature': 'SEX'}).\n", - "2022-12-12 09:19:49,886 - connect - INFO - Missing required evidence with label ({'metric_type': 'equal_opportunity_difference'}).\n", - "2022-12-12 09:19:49,887 - connect - INFO - Missing required evidence with label ({'metric_type': 'selection_rate'}).\n", - "2022-12-12 09:19:49,887 - connect - INFO - Missing required evidence with label ({'metric_type': 'accuracy_score'}).\n", - "2022-12-12 09:19:49,887 - connect - INFO - Missing required evidence with label ({'metric_type': 'soc'}).\n", - "2022-12-12 09:19:49,888 - connect - INFO - Missing required evidence with label ({'data_type': 'disaggregated_performance'}).\n", - "2022-12-12 09:19:49,888 - connect - INFO - Uploading 3 evidences.. for use_case_id=BRu2eLMdpwqqgYnab359mY policy_pack_id=NYCE+1\n", - "2022-12-12 09:19:51,125 - connect - INFO - 3 evidences were successfuly uploaded, took 224.652 ms\n", - "2022-12-12 09:19:51,126 - connect - INFO - Partial match of requirements.\n" - ] - }, - { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# export to API\n", "gov.export()\n", From 281f4fa14eb5c227563fe3b1290fec3d2f0a9efa Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 16:15:51 -0800 Subject: [PATCH 03/11] added ilnks to get started --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 955c67d..ccca612 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,6 +4,7 @@ Home page Setup + Get Started API reference <_autosummary/connect> From 3988128ef0e675539bde31f47dcd2f0d3c6010b5 Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 16:34:04 -0800 Subject: [PATCH 04/11] added pandas to doc requirements --- docs/conf.py | 7 +------ docs/requirements.txt | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 985b48e..4aa61b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,12 +63,7 @@ nbsphinx_allow_errors = True # Continue through Jupyter errors nbsphinx_execute = "never" # do not execute jupyter notebooks -autodoc_mock_imports = [ - "dotenv", - "json_api_doc", - "numpy", - "pandas", -] +autodoc_mock_imports = ["dotenv", "json_api_doc"] # -- Options for HTML output ------------------------------------------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index c1570e2..b9ccd34 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,6 +2,7 @@ ipykernel furo==2022.3.4 nbsphinx==0.8.7 +pandas >= 1.4.1 pandoc==1.1.0 prompt-toolkit<3.0.0 pydata-sphinx-theme==0.6.3 From 2687a7777492e1275ec095a3a54e6cf19279723a Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 16:46:57 -0800 Subject: [PATCH 05/11] added readthedocs config --- .readthedocs.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..1ce82c8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-20.04 + tools: + python: "3.8" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file From 7581b55efec71f224799e0f07415a47d482814bf Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 16:52:33 -0800 Subject: [PATCH 06/11] updated doc installations --- .readthedocs.yaml | 1 + docs/conf.py | 3 --- docs/requirements.txt | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1ce82c8..c53011f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -15,4 +15,5 @@ sphinx: python: install: + - requirements: requirements.txt - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 4aa61b2..c91bc40 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,9 +63,6 @@ nbsphinx_allow_errors = True # Continue through Jupyter errors nbsphinx_execute = "never" # do not execute jupyter notebooks -autodoc_mock_imports = ["dotenv", "json_api_doc"] - - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/requirements.txt b/docs/requirements.txt index b9ccd34..c1570e2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,7 +2,6 @@ ipykernel furo==2022.3.4 nbsphinx==0.8.7 -pandas >= 1.4.1 pandoc==1.1.0 prompt-toolkit<3.0.0 pydata-sphinx-theme==0.6.3 From 5e37401c91664a62c5fbb3bdc00dad82ec655523 Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 17:01:06 -0800 Subject: [PATCH 07/11] updated version --- connect/__init__.py | 2 +- docs/setup.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/connect/__init__.py b/connect/__init__.py index 66a8c15..e156aa0 100644 --- a/connect/__init__.py +++ b/connect/__init__.py @@ -5,7 +5,7 @@ from connect.governance import Governance from connect.utils.version_check import validate_version -__version__ = "0.0.4" +__version__ = "0.0.5" __all__ = ["governance", "evidence", "utils"] diff --git a/docs/setup.rst b/docs/setup.rst index 59473fe..b2f0496 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -27,4 +27,6 @@ would like to install requirements for testing and formatting, you'll have to install the dev requirements. :: + pip install credoai-connect[dev] + From beec82de1aa889750bf7e4a843a370ba54fc61a1 Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 17:49:21 -0800 Subject: [PATCH 08/11] updated notebook --- docs/notebooks/quickstart.ipynb | 41 +++++++++++++-------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/docs/notebooks/quickstart.ipynb b/docs/notebooks/quickstart.ipynb index 2ecec38..cabda2d 100644 --- a/docs/notebooks/quickstart.ipynb +++ b/docs/notebooks/quickstart.ipynb @@ -22,7 +22,7 @@ "\n", "**Credo AI Connect** is the library to interact with the Credo AI Governance Platform. It is responsible for receiving \"evidence requirements\" and sending \"evidence\"\n", "\n", - "Get started here. We will assess a payment default prediction model, assess it, and send the results to the [Credo AI Governance Platform](https://www.credo.ai/). For a tool to help you assess your AI systems, see [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html).\n", + "In this demo we train a model on some made-up data, and send the results to the [Credo AI Governance Platform](https://www.credo.ai/). For a tool to help you assess your AI systems, see [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html).\n", "\n", "**Setup**\n", "\n", @@ -87,27 +87,6 @@ "The default location/file name Lens is expecting is `~/.credoconfig`, where `~` is your home folder. You can specify any other location when you are initializing the `Governance` object (see below)." ] }, - { - "cell_type": "markdown", - "id": "b3bcfb4b-3249-499d-86d0-400d82823802", - "metadata": {}, - "source": [ - "### Get your ML environment ready\n", - " In this tutorial we will emulate the modeling phase by running a quick script. This script loads a dataset, splits it into training and testing, and fits a model. You can see the full script [here](https://github.com/credo-ai/credoai_lens/blob/release/1.0.0/docs/notebooks/training_script.py)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47e05093-f918-4e8a-8636-47eac6f0897c", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "import connect as ct" - ] - }, { "cell_type": "markdown", "id": "b592bab4-843a-461d-af9f-5a38ee895ed3", @@ -170,7 +149,7 @@ "\n", "The most likely thing you'll want to send to the Credo AI Platform are _assessments_ of the AI system. These are often _metrics_ that are calculated to summarize aspects of the system's behavior (e.g., performance, fairness) but can be other things too, like descriptive statistics of the dataset.\n", "\n", - "We'll assume we have a suite of assessments we plan to use for this use-case." + "We'll assume we have a suite of assessments we plan to use for this use-case. The below is just an example - any assessments can be done and sent to the Platform." ] }, { @@ -203,6 +182,16 @@ "First we will see all the code together, and then break it down." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "47e05093-f918-4e8a-8636-47eac6f0897c", + "metadata": {}, + "outputs": [], + "source": [ + "import connect as ct" + ] + }, { "cell_type": "code", "execution_count": null, @@ -233,7 +222,9 @@ "source": [ "#### 1. Connect to the platform via the Governance Class\n", "\n", - "The Governance class handles the connection with the Credo AI Platform. On the Platform, you can govern an AI system by specifying \"policy packs\" that specify the technical requirements the AI system must meet. This class can retrieve them, which is most useful if you are using [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html), our assessment framework. Since we are only using `Connect` in this demo, we will ignore that functionality.\n", + "The Governance class handles the connection with the Credo AI Platform. On the Platform, you can govern an AI system by specifying \"policy packs\" that specify the technical requirements the AI system must meet. This class can retrieve them, which is most useful if you are using [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html), our assessment framework. If you aren't using Lens, the requirements are still important in directing your assessment, but they aren't _programatically_ connected to your assessments.\n", + "\n", + "Since we are only using `Connect` in this demo, we will ignore that functionality.\n", "\n", "The important functionality relevant here is that the Governance object handles the API calls allowing you to _send_ evidence to the platform. " ] @@ -409,7 +400,7 @@ "metadata": {}, "outputs": [], "source": [ - "adapter = Adapter(governance = gov, model_name='My Model', model_tags={'model_type': 'regression'})" + "adapter = ct.Adapter(governance = gov, model_name='My Model', model_tags={'model_type': 'regression'})" ] }, { From 3cfc2f3b45a68e150542bf306c7a2ee480b348bb Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 17:57:14 -0800 Subject: [PATCH 09/11] removed cells hidden to me for some reason --- docs/notebooks/quickstart.ipynb | 366 +------------------------------- 1 file changed, 3 insertions(+), 363 deletions(-) diff --git a/docs/notebooks/quickstart.ipynb b/docs/notebooks/quickstart.ipynb index cabda2d..cccd58f 100644 --- a/docs/notebooks/quickstart.ipynb +++ b/docs/notebooks/quickstart.ipynb @@ -1,14 +1,5 @@ { "cells": [ - { - "cell_type": "raw", - "id": "cb8273f5", - "metadata": {}, - "source": [ - "\n", - "Click here to download this notebook." - ] - }, { "cell_type": "markdown", "id": "858e3872-01c7-4dc5-bc70-eb9a821c3466", @@ -410,362 +401,11 @@ "source": [ "If you need to label your evidence with more information than just `source` you can do that too." ] - }, - { - "cell_type": "markdown", - "id": "c0797a51", - "metadata": {}, - "source": [ - "## Lens in 5 minutes" - ] - }, - { - "cell_type": "markdown", - "id": "cce779fd", - "metadata": { - "tags": [] - }, - "source": [ - "Below is a basic example where our goal is to evaluate the above model. We'll break down this code [below](#Breaking-Down-The-Steps).\n", - "\n", - "Briefly, the code is doing four things:\n", - "\n", - "* Wrapping ML artifacts (like models and data) in Lens objects\n", - "* Initializing an instance of Lens. Lens is the main object that performs evaluations. Under the hood, it creates a `pipeline` of evaluations that are run.\n", - "* Add evaluators to Lens.\n", - "* Run Lens" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5b4e6a6a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# set up model and data artifacts\n", - "credo_model = ClassificationModel(name=\"credit_default_classifier\", model_like=model)\n", - "credo_data = TabularData(\n", - " name=\"UCI-credit-default\",\n", - " X=X_test,\n", - " y=y_test,\n", - " sensitive_features=sensitive_features_test,\n", - ")\n", - "\n", - "# Initialization of the Lens object\n", - "lens = Lens(model=credo_model, assessment_data=credo_data)\n", - "\n", - "# initialize the evaluator and add it to Lens\n", - "metrics = ['precision_score', 'recall_score', 'equal_opportunity']\n", - "lens.add(ModelFairness(metrics=metrics))\n", - "lens.add(Performance(metrics=metrics))\n", - "\n", - "# run Lens\n", - "lens.run()" - ] - }, - { - "cell_type": "markdown", - "id": "c188b100-2ebb-42c8-9451-2812fe7383d6", - "metadata": {}, - "source": [ - "### Getting results within your python environment\n", - "\n", - "`lens.get_results()` provides a list where the results of the evaluators (a list of dataframes) are stored along with the evaluator metadata. In this case, there are 2 results - one for each evaluator." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7eeac730-29bd-4300-8e02-d8677e10123e", - "metadata": {}, - "outputs": [], - "source": [ - "results = lens.get_results()\n", - "print(f\"Results for {len(results)} evaluators\")" - ] - }, - { - "cell_type": "markdown", - "id": "8fa1eab0", - "metadata": {}, - "source": [ - "`lens.get_results()` has some arguments, which makes it easier for you to get a subset of results. These are the same arguments that can be passed to `lens.get_pipeline` and `lens.get_evidence`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f7735dd2-5797-45aa-90f2-212faa063edd", - "metadata": {}, - "outputs": [], - "source": [ - "lens.get_results(evaluator_name='Performance')" - ] - }, - { - "cell_type": "markdown", - "id": "94ca7568", - "metadata": {}, - "source": [ - "## Using Len's pipeline argument" - ] - }, - { - "cell_type": "markdown", - "id": "498a11b3", - "metadata": {}, - "source": [ - "If we want to add multiple evaluators to our pipeline, one way of doing it could be repeating the `add` step, as shown above. Another way is to define the pipeline steps, and pass it to `Lens` at initialization time. Let's explore the latter!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f506b7d8", - "metadata": {}, - "outputs": [], - "source": [ - "pipeline = [\n", - " (ModelFairness(metrics)),\n", - " (Performance(metrics)),\n", - "]\n", - "lens = Lens(model=credo_model, assessment_data=credo_data, pipeline=pipeline)" - ] - }, - { - "cell_type": "markdown", - "id": "aa465d26", - "metadata": {}, - "source": [ - "Above, each of the `tuples` in the `list` is in the form `(instantiated_evaluator, id)`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c46bdd9e", - "metadata": {}, - "outputs": [], - "source": [ - "# notice that Lens functions can be chained together\n", - "results = lens.run().get_results()\n", - "print(f'\\nFound results for {len(results)} evaluators')" - ] - }, - { - "cell_type": "markdown", - "id": "601f5203", - "metadata": {}, - "source": [ - "Let's check that we have results for both of our evaluators." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ab0ed632", - "metadata": {}, - "outputs": [], - "source": [ - "results[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5bb680a5", - "metadata": {}, - "outputs": [], - "source": [ - "results[1]" - ] - }, - { - "cell_type": "markdown", - "id": "bb18d57e", - "metadata": {}, - "source": [ - "## That's it!\n", - "\n", - "That should get you up and running. Next steps include:\n", - "\n", - "* Trying out other evaluators (they are all accessible via `credoai.evaluators`)\n", - "* Checking out our developer guide to better understand the Lens ecosystem and see how you can extend it.\n", - "* Exploring the Credo AI Governance Platform, which will connect AI assessments with customizable governance to support reporting, compliance, multi-stakeholder translation and more!" - ] - }, - { - "cell_type": "markdown", - "id": "c6c05dfe", - "metadata": { - "tags": [] - }, - "source": [ - "## Breaking Down The Steps\n", - "\n", - "### Preparing artifacts\n", - "\n", - "Lens interacts with \"AI Artifacts\" which wrap model and data objects and standardize them for use by different evaluators.\n", - "\n", - "Below we create a `ClassificationModel` artifact. This is a light wrapper for any kind of fitted classification model-like object. \n", - "\n", - "We also create a `TabularData` artifact which stores X, y and sensitive features." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a4f1fae7", - "metadata": {}, - "outputs": [], - "source": [ - "# set up model and data artifacts\n", - "credo_model = ClassificationModel(name=\"credit_default_classifier\", model_like=model)\n", - "\n", - "credo_data = TabularData(\n", - " name=\"UCI-credit-default\",\n", - " X=X_test,\n", - " y=y_test,\n", - " sensitive_features=sensitive_features_test,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "114bd0bb", - "metadata": { - "tags": [] - }, - "source": [ - "#### Model\n", - "\n", - "Model type objects, like `ClassificationModel` used above, serve as adapters between arbitrary models and the evaluators in Lens. Some evaluators depend on Model instantiating certain methods. For example, `ClassificationModel` can accept any generic object having `predict` and `predict_proba` methods, including fitted sklearn pipelines.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "db7fb09b", - "metadata": { - "tags": [] - }, - "source": [ - "#### Data\n", - "\n", - "_Data_ type artifact, like `TabularData` serve as adapters between datasets and the evaluators in Lens.\n", - "\n", - "When you pass data to a _Data_ artifact, the artifact performs various steps of validation, and formats them so that they can be used by evaluators. The aim of this procedure is to preempt errors down the line.\n", - "\n", - "You can pass Data to Lens as a **training dataset** or an **assessment dataset** (see lens class documentation). If the former, it will not be used to assess the model. Instead, dataset assessments will be performed on the dataset (e.g., fairness assessment). The validation dataset will be assessed in the same way, but _also_ used to assess the model, if provided.\n", - "\n", - "Similarly to _Model_ type objects, _Data_ objects can be customized, see !!insertlink!!" - ] - }, - { - "cell_type": "markdown", - "id": "e4583249", - "metadata": {}, - "source": [ - "### Evaluators \n", - "\n", - "Lens uses the above artifacts to ensure a successfull run of the evaluators. As we have seen in the sections [Lens in 5 minutes](##Lens-in-5-minutes) and [Adding multiple evaluators](##Adding-multiple-evaluators), multiple evaluators can be added to _Lens_ pipeline. Each evaluators contains information on what it needs in order to run successfully, and it executes a validation step at _add_ time.\n", - "\n", - "The result of the validation depends on what artifacts are available, their content and the type of evaluator being added to the pipeline. In case the validation process fails, the user is notified the reason why the evaluator cannot be added to the pipeline.\n", - "\n", - "See for example:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "836cccd0", - "metadata": {}, - "outputs": [], - "source": [ - "from credoai.evaluators import Privacy\n", - "lens.add(Privacy())" - ] - }, - { - "cell_type": "markdown", - "id": "e76369e8", - "metadata": {}, - "source": [ - "Currently no automatic run of evaluators is supported. However, when Lens is used in combination with Credo AI Platform, it is possible to download an assessment plan which then gets converted into a set of evaluations that Lens can run programmatically. For more information see the [governance tutorial](https://credoai-lens.readthedocs.io/en/stable/notebooks/governance_integration.html)." - ] - }, - { - "cell_type": "markdown", - "id": "cdb3c5b8", - "metadata": {}, - "source": [ - "### Run Lens\n", - "\n", - "After we have initialized _Lens_ the _Model_ and _Data_ (`ClassificationModel` and `TabularData` in our example) type artifacts, we can add whichever evaluators we want to the pipeline, and finally run it!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fc76430f", - "metadata": {}, - "outputs": [], - "source": [ - "lens = Lens(model=credo_model, assessment_data=credo_data)\n", - "metrics = ['precision_score', 'recall_score', 'equal_opportunity']\n", - "lens.add(ModelFairness(metrics=metrics))\n", - "lens.run()" - ] - }, - { - "cell_type": "markdown", - "id": "673dbbc9", - "metadata": {}, - "source": [ - "As you can notice, when adding _evaluators_ to lens, they need to be instantiated. If any extra arguments need to be passed to the evaluator (like metrics in this case), this is the time to do it." - ] - }, - { - "cell_type": "markdown", - "id": "2a5dcbe3", - "metadata": {}, - "source": [ - "**Getting Evaluator Results**\n", - "\n", - "Afte the pipeline is run, the results become accessible via the method `get_results()`\n", - "\n", - "`lens.get_results()` provides a dictionary where the results of the evaluators are stored as values, and the keys correspond to the ids of the evaluators. \n", - "\n", - "In the previous case we specified the id of the evaluator when we added `ModelFairness` to the pipeline, however `id` is an optional argument for the `add` method. If omitted, a random one will be generated." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "031d237a", - "metadata": {}, - "outputs": [], - "source": [ - "lens.get_results()" - ] - }, - { - "cell_type": "markdown", - "id": "ac3bfe1a", - "metadata": {}, - "source": [ - "**Credo AI Governance Platform**\n", - "\n", - "For information on how to interact with the plaform, please look into: [Connecting with Governance App](https://credoai-lens.readthedocs.io/en/stable/notebooks/governance_integration.html) tutorial for directions.\n" - ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3.9.12 ('dev')", "language": "python", "name": "python3" }, @@ -779,11 +419,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.9.12" }, "vscode": { "interpreter": { - "hash": "507a61bde0a0183a71b2d35939f461921273a091e2cc4517af66dd70c4baafc9" + "hash": "476253e59e65943f63347f07b9cde9e6998ced5a0f745ea85ba02156ab78e294" } } }, From c8be0acefd57a650d7d68f9566762aabd276367f Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 18:12:48 -0800 Subject: [PATCH 10/11] another update --- docs/notebooks/quickstart.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/notebooks/quickstart.ipynb b/docs/notebooks/quickstart.ipynb index cccd58f..029c5c6 100644 --- a/docs/notebooks/quickstart.ipynb +++ b/docs/notebooks/quickstart.ipynb @@ -211,7 +211,7 @@ "id": "6c69d81a-83d8-4e76-afd9-c16b51f6ff68", "metadata": {}, "source": [ - "#### 1. Connect to the platform via the Governance Class\n", + "### 1. Connect to the platform via the Governance Class\n", "\n", "The Governance class handles the connection with the Credo AI Platform. On the Platform, you can govern an AI system by specifying \"policy packs\" that specify the technical requirements the AI system must meet. This class can retrieve them, which is most useful if you are using [Credo AI Lens](https://credoai-lens.readthedocs.io/en/stable/setup.html), our assessment framework. If you aren't using Lens, the requirements are still important in directing your assessment, but they aren't _programatically_ connected to your assessments.\n", "\n", @@ -240,7 +240,7 @@ "id": "12a8ff57-099a-4e63-b2bc-6a6da6779ce2", "metadata": {}, "source": [ - "#### 2. Use the Adapter class to send the assessments to the Governance class\n", + "### 2. Use the Adapter class to send the assessments to the Governance class\n", "\n", "The `Adapter` class handles structuring your assessments so that the Credo AI Platform can understand them. Connect uses `EvidenceContainers` to handle converting python objects like dictionaries into `Evidence`, the structured output that the Platform can understand. The Adapter then passes this `Evidence` to the `Governance` object for export" ] @@ -335,7 +335,7 @@ "id": "6e4364e8-d5da-436e-a0b3-e38a1b283f11", "metadata": {}, "source": [ - "#### 3. Export the evidence to the platform.\n", + "### 3. Export the evidence to the platform.\n", "\n", "Exporting is straight forward! You can either export directly to the Platform or to a file. At the time of export, the uploaded evidence will be checked against the governance requirements specified on Platform and let you know what's missing." ] @@ -405,7 +405,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9.12 ('dev')", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -419,7 +419,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.9.13" }, "vscode": { "interpreter": { From 6a0efeceaaba67bad19809c93fad2d5e76fc469c Mon Sep 17 00:00:00 2001 From: Ian Eisenberg Date: Mon, 12 Dec 2022 18:40:09 -0800 Subject: [PATCH 11/11] fixed adapter bug --- connect/adapters/adapters.py | 23 +++++++++++++---------- docs/notebooks/quickstart.ipynb | 2 ++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/connect/adapters/adapters.py b/connect/adapters/adapters.py index 8463201..e610001 100644 --- a/connect/adapters/adapters.py +++ b/connect/adapters/adapters.py @@ -3,7 +3,7 @@ import pandas as pd -from connect.evidence import MetricContainer, TableContainer +from connect.evidence import EvidenceContainer, MetricContainer, TableContainer from connect.governance import Governance from connect.utils import ValidationError, wrap_list @@ -131,10 +131,13 @@ def _evidence_to_governance( When adding evidence to a Governance object, whether to overwrite existing evidence or not, default False. """ - if evidence_fun is None: - evidence_fun = partial(self._to_evidence, container_class=evidence_fun) + try: + if issubclass(evidence_fun, EvidenceContainer): + evidence_fun = partial(self._to_evidence, container_class=evidence_fun) + except TypeError: + pass labels = {**(labels or {}), "source": source} - evidence = evidence_fun(data, labels, metadata) + evidence = evidence_fun(data=data, labels=labels, metadata=metadata) if overwrite_governance: self.governance.set_evidence(evidence) else: @@ -148,12 +151,12 @@ def _get_artifact_meta(self): del model["tags"] return model or {} - def _metrics_to_evidence(self, metrics, labels=None, metadata=None): + def _metrics_to_evidence(self, data, labels=None, metadata=None): """Converts a dictionary of metrics to evidence Parameters ---------- - metrics : dict or pd.DataFrame + data : dict or pd.DataFrame Dictionary of metrics. Form: {metric_type: value, ...} labels : dict Additional labels to pass to underlying evidence @@ -165,11 +168,11 @@ def _metrics_to_evidence(self, metrics, labels=None, metadata=None): List list of Evidence """ - if isinstance(metrics, dict): - metrics = pd.DataFrame(metrics.items(), columns=["type", "value"]) - elif not isinstance(metrics, pd.DataFrame): + if isinstance(data, dict): + data = pd.DataFrame(data.items(), columns=["type", "value"]) + elif not isinstance(data, pd.DataFrame): raise ValidationError("Metrics must be a dictionary or a dataframe") - return self._to_evidence(MetricContainer, metrics, labels, metadata) + return self._to_evidence(MetricContainer, data, labels, metadata) def _to_evidence(self, container_class, data, labels, metadata): meta = self._get_artifact_meta() diff --git a/docs/notebooks/quickstart.ipynb b/docs/notebooks/quickstart.ipynb index 029c5c6..0b4e4ad 100644 --- a/docs/notebooks/quickstart.ipynb +++ b/docs/notebooks/quickstart.ipynb @@ -93,6 +93,8 @@ "metadata": {}, "outputs": [], "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", "from sklearn.linear_model import LinearRegression\n", "import sklearn.metrics as sk_metrics\n", "from sklearn.model_selection import train_test_split\n",