Merge pull request #12 from credo-ai/demo

Demo

.readthedocs.yaml

@@ -0,0 +1,19 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.8"
+
+sphinx:
+   configuration: docs/conf.py
+
+python:
+   install:
+   - requirements: requirements.txt
+   - requirements: docs/requirements.txt

connect/__init__.py

@@ -1,9 +1,11 @@
 """
 Credo AI Connect package
 """
+from connect.adapters import Adapter
+from connect.governance import Governance
 from connect.utils.version_check import validate_version
 
-__version__ = "0.0.4"
+__version__ = "0.0.5"
 
 __all__ = ["governance", "evidence", "utils"]
 

connect/adapters/adapters.py

@@ -1,10 +1,11 @@
+from functools import partial
 from typing import Optional
 
 import pandas as pd
 
-from connect.evidence import MetricContainer
+from connect.evidence import EvidenceContainer, MetricContainer, TableContainer
 from connect.governance import Governance
-from connect.utils import ValidationError
+from connect.utils import ValidationError, wrap_list
 
 
 class Adapter:
@@ -38,9 +39,10 @@ def __init__(
     def metrics_to_governance(
         self,
         metrics: dict,
+        source: str,
         labels: dict = None,
         metadata: dict = None,
-        overwrite_governance: bool = False,
+        overwrite_governance: bool = True,
     ):
         """
         Packages metrics as evidence and sends them to governance
@@ -49,16 +51,93 @@ def metrics_to_governance(
         ---------
         metrics : dict or pd.DataFrame
             Dictionary of metrics. Form: {metric_type: value, ...}
+        source : str
+            Label for what generated the metrics
         labels : dict
-            Additional labels to pass to underlying evidence
+            Additional key/value pairs to act as labels for the evidence
+        metadata : dict
+            Metadata to pass to underlying evidence
+        overwrite_governance : bool
+            When adding evidence to a Governance object, whether to overwrite existing
+            evidence or not, default False.
+        """
+        self._evidence_to_governance(
+            self._metrics_to_evidence,
+            metrics,
+            source,
+            labels,
+            metadata,
+            overwrite_governance,
+        )
+
+    def table_to_governance(
+        self,
+        data: dict,
+        source: str,
+        labels: dict = None,
+        metadata: dict = None,
+        overwrite_governance: bool = True,
+    ):
+        """
+        Packages metrics as evidence and sends them to governance
+
+        Parameters
+        ---------
+        data: pd.DataFrame
+            Dataframe to pass to evidence_fun. The DataFrame must have a "name" attribute
+        source : str
+            Label for what generated the table
+        labels : dict
+            Additional key/value pairs to act as labels for the evidence
         metadata : dict
             Metadata to pass to underlying evidence
         overwrite_governance : bool
             When adding evidence to a Governance object, whether to overwrite existing
             evidence or not, default False.
+        evidence_fun : callable
+            Function to pass data, labels and metadata. The function should return a list of
+            evidence. Default: self._to_evidence
         """
-        evidence = self._metrics_to_evidence(metrics, labels, metadata)
+        self._evidence_to_governance(
+            TableContainer, data, source, labels, metadata, overwrite_governance
+        )
 
+    def _evidence_to_governance(
+        self,
+        evidence_fun,
+        data,
+        source,
+        labels,
+        metadata,
+        overwrite_governance=True,
+    ):
+        """
+        Packages data as evidence and sends to governance
+
+        Parameters
+        ---------
+        evidence_fun : callable or Container
+            Function to pass data, labels and metadata. The function should return a list of
+            evidence. If a Container, use self._to_evidence with the specified container
+        data
+            data to pass to evidence_fun
+        source : str
+            Label for what generated the table
+        labels : dict
+            Additional key/value pairs to act as labels for the evidence
+        metadata : dict
+            Metadata to pass to underlying evidence
+        overwrite_governance : bool
+            When adding evidence to a Governance object, whether to overwrite existing
+            evidence or not, default False.
+        """
+        try:
+            if issubclass(evidence_fun, EvidenceContainer):
+                evidence_fun = partial(self._to_evidence, container_class=evidence_fun)
+        except TypeError:
+            pass
+        labels = {**(labels or {}), "source": source}
+        evidence = evidence_fun(data=data, labels=labels, metadata=metadata)
         if overwrite_governance:
             self.governance.set_evidence(evidence)
         else:
@@ -72,12 +151,12 @@ def _get_artifact_meta(self):
             del model["tags"]
         return model or {}
 
-    def _metrics_to_evidence(self, metrics, labels=None, metadata=None):
+    def _metrics_to_evidence(self, data, labels=None, metadata=None):
         """Converts a dictionary of metrics to evidence
 
         Parameters
         ----------
-        metrics : dict or pd.DataFrame
+        data : dict or pd.DataFrame
             Dictionary of metrics. Form: {metric_type: value, ...}
         labels : dict
             Additional labels to pass to underlying evidence
@@ -89,11 +168,14 @@ def _metrics_to_evidence(self, metrics, labels=None, metadata=None):
         List
             list of Evidence
         """
+        if isinstance(data, dict):
+            data = pd.DataFrame(data.items(), columns=["type", "value"])
+        elif not isinstance(data, pd.DataFrame):
+            raise ValidationError("Metrics must be a dictionary or a dataframe")
+        return self._to_evidence(MetricContainer, data, labels, metadata)
+
+    def _to_evidence(self, container_class, data, labels, metadata):
         meta = self._get_artifact_meta()
         meta.update(metadata or {})
-        if isinstance(metrics, dict):
-            metrics = pd.DataFrame(metrics.items(), columns=["type", "value"])
-        elif not isinstance(metrics, pd.DataFrame):
-            raise ValidationError("Metrics must be a dictionary or a dataframe")
-        container = MetricContainer(metrics, labels, meta)
-        return container.to_evidence()
+        container = container_class(data, labels, meta)
+        return wrap_list(container.to_evidence())

connect/governance/credo_api_client.py

@@ -9,8 +9,7 @@
 from dotenv import dotenv_values
 from json_api_doc import deserialize, serialize
 
-from connect import __version__
-from connect.utils import global_logger, json_dumps
+from connect.utils import get_version, global_logger, json_dumps
 
 CREDO_URL = "https://api.credo.ai"
 
@@ -133,7 +132,7 @@ def set_access_token(self, access_token):
             "accept": "application/vnd.api+json",
             "content-type": "application/vnd.api+json",
             "X-Client-Name": "Credo AI Connect",
-            "X-Client-Version": __version__,
+            "X-Client-Version": get_version(),
         }
         self._session.headers.update(headers)
 

connect/governance/governance.py

@@ -10,9 +10,14 @@
 
 from json_api_doc import deserialize, serialize
 
-from connect import __version__
 from connect.evidence import Evidence, EvidenceRequirement
-from connect.utils import check_subset, global_logger, json_dumps, wrap_list
+from connect.utils import (
+    check_subset,
+    get_version,
+    global_logger,
+    json_dumps,
+    wrap_list,
+)
 
 from .credo_api import CredoApi
 from .credo_api_client import CredoApiClient
@@ -171,7 +176,7 @@ def get_evidence_requirements(self, tags: dict = None, verbose=False):
             self._print_evidence(reqs)
         return reqs
 
-    def get_evidence_tags(self):
+    def get_requirement_tags(self):
         """Return the unique tags used for all evidence requirements"""
         return self._unique_tags
 
@@ -318,7 +323,7 @@ def set_evidence(self, evidences: List[Evidence]):
 
     def tag_model(self, model):
         """Interactive utility to tag a model tags from assessment plan"""
-        tags = self.get_evidence_tags()
+        tags = self.get_requirement_tags()
         print(f"Select tag from assessment plan to associated with model:")
         print("0: No tags")
         for number, tag in enumerate(tags):
@@ -393,7 +398,7 @@ def _file_export(self, filename):
             f"Saving {len(self._evidences)} evidences to {filename}.. for use_case_id={self._use_case_id} policy_pack_id={self._policy_pack_id} "
         )
         data = self._prepare_export_data()
-        meta = {"client": "Credo AI Connect", "version": __version__}
+        meta = {"client": "Credo AI Connect", "version": get_version()}
         data = json_dumps(serialize(data=data, meta=meta))
         with open(filename, "w") as f:
             f.write(data)

connect/utils/__init__.py

@@ -5,3 +5,4 @@
 from .common import *
 from .data_scrubbing import Scrubber
 from .logging import *
+from .version_check import get_version

connect/utils/version_check.py

@@ -4,8 +4,12 @@
 from connect.utils import global_logger
 
 
+def get_version():
+    return connect.__version__
+
+
 def validate_version():
-    current_version = connect.__version__
+    current_version = get_version()
 
     package = "credoai-connect"
     response = requests.get(f"https://pypi.org/pypi/{package}/json")

docs/conf.py

@@ -63,14 +63,6 @@
 nbsphinx_allow_errors = True  # Continue through Jupyter errors
 nbsphinx_execute = "never"  # do not execute jupyter notebooks
 
-autodoc_mock_imports = [
-    "dotenv",
-    "json_api_doc",
-    "numpy",
-    "pandas",
-]
-
-
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for

docs/index.rst

@@ -4,6 +4,7 @@
 
    Home page <self>
    Setup <setup>
+   Get Started <notebooks/quickstart>
    API reference <_autosummary/connect>