Merge pull request #19 from credo-ai/release/0.0.7

Release/0.0.7

.github/workflows/release.yml

@@ -0,0 +1,24 @@
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

connect/__init__.py

@@ -5,7 +5,7 @@
 from connect.governance import Governance
 from connect.utils.version_check import validate_version
 
-__version__ = "0.0.5"
+__version__ = "0.0.6"
 
 __all__ = ["governance", "evidence", "utils"]
 

connect/adapters/adapters.py

@@ -52,7 +52,7 @@ def metrics_to_governance(
         metrics : dict or pd.DataFrame
             Dictionary of metrics. Form: {metric_type: value, ...}
         source : str
-            Label for what generated the metrics
+            Label for what generated the metrics. Added to metadata
         labels : dict
             Additional key/value pairs to act as labels for the evidence
         metadata : dict
@@ -86,7 +86,7 @@ def table_to_governance(
         data: pd.DataFrame
             Dataframe to pass to evidence_fun. The DataFrame must have a "name" attribute
         source : str
-            Label for what generated the table
+            Label for what generated the table. Added to metadata
         labels : dict
             Additional key/value pairs to act as labels for the evidence
         metadata : dict
@@ -122,7 +122,7 @@ def _evidence_to_governance(
         data
             data to pass to evidence_fun
         source : str
-            Label for what generated the table
+            Label for what generated the evidence. Added to metadata
         labels : dict
             Additional key/value pairs to act as labels for the evidence
         metadata : dict
@@ -136,7 +136,7 @@ def _evidence_to_governance(
                 evidence_fun = partial(self._to_evidence, container_class=evidence_fun)
         except TypeError:
             pass
-        labels = {**(labels or {}), "source": source}
+        metadata = {**(metadata or {}), "source": source}
         evidence = evidence_fun(data=data, labels=labels, metadata=metadata)
         if overwrite_governance:
             self.governance.set_evidence(evidence)

connect/evidence/__init__.py

@@ -4,6 +4,11 @@
 All the logic defined here is in order to standardize communication with 
 Credo AI Platform API
 """
-from .containers import EvidenceContainer, MetricContainer, TableContainer
-from .evidence import Evidence, MetricEvidence, TableEvidence
+from .containers import (
+    EvidenceContainer,
+    MetricContainer,
+    TableContainer,
+    StatisticTestContainer,
+)
+from .evidence import Evidence, MetricEvidence, TableEvidence, StatisticTestEvidence
 from .evidence_requirement import EvidenceRequirement

connect/evidence/containers.py

@@ -10,7 +10,7 @@
 
 from connect.utils import Scrubber, ValidationError
 
-from .evidence import MetricEvidence, TableEvidence
+from .evidence import MetricEvidence, StatisticTestEvidence, TableEvidence
 
 
 class EvidenceContainer(ABC):
@@ -91,6 +91,36 @@ def _validate(self, data):
             )
 
 
+class StatisticTestContainer(EvidenceContainer):
+    """Containers for all Statistic Test type evidence"""
+
+    def __init__(self, data: pd.DataFrame, labels: dict = None, metadata: dict = None):
+        super().__init__(StatisticTestEvidence, data, labels, metadata)
+
+    def to_evidence(self, **metadata):
+        evidence = []
+        for _, data in self.scrubbed_data.iterrows():
+            evidence.append(
+                self.evidence_class(
+                    additional_labels=self.labels, **data, **self.metadata, **metadata
+                )
+            )
+        return evidence
+
+    def _validate(self, data):
+        required_columns = {
+            "statistic_type",
+            "test_statistic",
+            "significance_threshold",
+            "p_value",
+        }
+        column_overlap = data.columns.intersection(required_columns)
+        if len(column_overlap) != len(required_columns):
+            raise ValidationError(
+                f"Metrics dataframe must have columns: {required_columns}"
+            )
+
+
 class TableContainer(EvidenceContainer):
     """Container for all Table type evidence"""
 

connect/evidence/evidence.py

@@ -1,13 +1,12 @@
 """
 Wrappers formatting results of evaluator runs for the Credo AI Platform
 """
-import json
 import pprint
 from abc import ABC, abstractproperty
 from datetime import datetime
 from typing import Tuple
 
-from pandas import DataFrame, Series
+from pandas import DataFrame
 
 from connect.utils import ValidationError
 
@@ -118,6 +117,60 @@ def _validate(self):
             raise ValidationError
 
 
+class StatisticTestEvidence(Evidence):
+    """
+    Evidence for Statistical Test:value result type
+
+    Parameters
+    ----------
+    statistic_type : string
+        Short identifier for statistical test.
+    value : float
+        Test calculation result
+    p_value : float
+        p_value associated to the calculation
+    significance_threshold : float
+        p_value threshold to consider test significant, e.g., 0.01
+    additional_labels: dict, opional
+        Extra info to be added to the label section.
+    metadata : dict, optional
+        Arbitrary keyword arguments to append to metric as metadata. These will be
+        displayed in the governance app
+    """
+
+    def __init__(
+        self,
+        statistic_type: str,
+        test_statistic: float,
+        significance_threshold: float,
+        p_value: float,
+        additional_labels=None,
+        **metadata
+    ):
+        self.statistic_type = statistic_type
+        self.test_statistic = test_statistic
+        self.significance_threshold = significance_threshold
+        self.p_value = p_value
+        self.significant = (
+            True if self.p_value <= self.significance_threshold else False
+        )
+        super().__init__("statisticTest", additional_labels, **metadata)
+
+    @property
+    def data(self):
+        return {
+            "test_statistic": self.test_statistic,
+            "significance_threshold": self.significance_threshold,
+            "p_value": self.p_value,
+            "significant": self.significant,
+        }
+
+    @property
+    def base_label(self):
+        label = {"statistic_type": self.statistic_type}
+        return label
+
+
 class TableEvidence(Evidence):
     """
     Evidence for tabular data

connect/evidence/evidence_requirement.py

@@ -8,9 +8,9 @@ def __init__(
         data: dict,
     ):
         self._evidence_type: str = data.get("evidence_type")
-        self._label: dict = data.get("label")
+        self._label: dict = data.get("label", {})
         self._sensitive_features: List[str] = data.get("sensitive_features", [])
-        self._tags: dict = data.get("tags")
+        self._tags: dict = data.get("tags", {})
 
     def __str__(self) -> str:
         return f"{self.evidence_type}-EvidenceRequirement.label-{self.label}"

connect/governance/governance.py

@@ -150,8 +150,15 @@ def get_evidence(self, verbose=False):
     def get_evidence_requirements(self, tags: dict = None, verbose=False):
         """
         Returns evidence requirements. Each evidence requirement can have optional tags
-        (a dictionary). Only returns requirements that have tags that match the model
-        (if provided), which are tags with the same tags as the model, or no tags.
+        (a dictionary). Returns requirements whose tags are a subset of the model's tags.
+
+        For example, imagine a model has two tags: {'risk': 'high', 'model_type': 'binary'}
+        and three requirements with the following tags:
+            req1 = {}
+            req2 = {'risk': 'high'}
+            req3 = {'risk': 'high', 'model_type': 'binary'}
+        In this case all requirements will apply. If the model risk was was 'low' only req_1 would
+        apply. If the 'model_type' was not 'binary' then only req1 and req2 would apply.
 
         Parameters
         ----------
@@ -169,9 +176,7 @@ def get_evidence_requirements(self, tags: dict = None, verbose=False):
         if tags is None:
             tags = self.get_model_tags()
 
-        reqs = [
-            e for e in self._evidence_requirements if (not e.tags or e.tags == tags)
-        ]
+        reqs = [e for e in self._evidence_requirements if check_subset(e.tags, tags)]
         if verbose:
             self._print_evidence(reqs)
         return reqs
@@ -185,7 +190,7 @@ def get_model_tags(self):
         if self._model:
             return self._model["tags"]
         else:
-            return None
+            return {}
 
     def register(
         self,

docs/notebooks/quickstart.ipynb

@@ -421,7 +421,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.13"
+   "version": "3.9.12"
   },
   "vscode": {
    "interpreter": {

tests/governance/test_governance.py

@@ -19,6 +19,16 @@
         "label": {"table_name": "disaggregated_performance"},
         "sensitive_features": ["profession", "gender"],
     },
+    {
+        "evidence_type": "metric",
+        "label": {"metric_type": "precision"},
+        "tags": {"risk": "high"},
+    },
+    {
+        "evidence_type": "metric",
+        "label": {"metric_type": "recall"},
+        "tags": {"risk": "high", "model_type": "binary"},
+    },
 ]
 ASSESSMENT_PLAN_URL = f"http://api.credo.ai/api/v2/credoai/use_cases/${USE_CASE_ID}/assessment_plans/{POLICY_PACK_ID}"
 ASSESSMENT_PLAN = {
@@ -204,3 +214,12 @@ def test_export_to_file_partial_requirements_fulfilled(self, gov):
         with tempfile.TemporaryDirectory() as tempDir:
             filename = f"{tempDir}/assessment.json"
             assert False == gov.export(filename)
+
+    def test_get_evidence_requirements(self, gov):
+        gov.register(assessment_plan=ASSESSMENT_PLAN_JSON_STR)
+        gov.set_artifacts(model="test", model_tags={"risk": "high"})
+        assert 4 == len(gov.get_evidence_requirements())
+        gov.set_artifacts(
+            model="test", model_tags={"risk": "high", "model_type": "binary"}
+        )
+        assert 5 == len(gov.get_evidence_requirements())