Use tags to populate subject areas

datahub_client/client.py

@@ -1,6 +1,5 @@
 import json
 import logging
-from importlib.resources import files
 from typing import Sequence
 
 from datahub.configuration.common import ConfigurationError
@@ -26,6 +25,7 @@
     TableEntityMapping,
 )
 from datahub_client.exceptions import ConnectivityError, EntityDoesNotExist
+from datahub_client.graphql.loader import get_graphql_query
 from datahub_client.parsers import (
     ChartParser,
     DashboardParser,
@@ -104,26 +104,10 @@ def __init__(self, jwt_token, api_url: str, graph=None):
 
         self.search_client = SearchClient(self.graph)
 
-        self.database_query = (
-            files("datahub_client.graphql")
-            .joinpath("getContainerDetails.graphql")
-            .read_text()
-        )
-        self.dataset_query = (
-            files("datahub_client.graphql")
-            .joinpath("getDatasetDetails.graphql")
-            .read_text()
-        )
-        self.chart_query = (
-            files("datahub_client.graphql")
-            .joinpath("getChartDetails.graphql")
-            .read_text()
-        )
-        self.dashboard_query = (
-            files("datahub_client.graphql")
-            .joinpath("getDashboardDetails.graphql")
-            .read_text()
-        )
+        self.database_query = get_graphql_query("getContainerDetails")
+        self.dataset_query = get_graphql_query("getDatasetDetails")
+        self.chart_query = get_graphql_query("getChartDetails")
+        self.dashboard_query = get_graphql_query("getDashboardDetails")
 
     def check_entity_exists_by_urn(self, urn: str | None):
         if urn is not None:
@@ -190,7 +174,7 @@ def search(
             sort=sort,
         )
 
-    def list_domains(
+    def list_subject_areas(
         self,
         query: str = "*",
         filters: Sequence[MultiSelectFilter] | None = None,
@@ -199,7 +183,7 @@ def list_domains(
         """
         Returns a list of DomainOption objects
         """
-        return self.search_client.list_domains(
+        return self.search_client.list_subject_areas(
             query=query, filters=filters, count=count
         )
 

datahub_client/entities.py

@@ -279,6 +279,10 @@ class TagRef(BaseModel):
     Reference to a tag
     """
 
+    @classmethod
+    def from_name(cls, name):
+        return cls(display_name=name, urn=f"urn:li:tag:{name}")
+
     display_name: str = Field(
         description="Human friendly tag name",
         examples=["PII"],
@@ -692,3 +696,36 @@ class Dashboard(Entity):
         description="URL to view the dashboard",
         examples=["https://data.justice.gov.uk"],
     )
+
+
+class SubjectArea(TagRef):
+    @property
+    def domain_urn(self):
+        return self.urn.replace(":tag:", ":domain:")
+
+
+class SubjectAreaTaxonomy:
+    TOP_LEVEL = [
+        SubjectArea.from_name("Bold"),
+        SubjectArea.from_name("Civil"),
+        SubjectArea.from_name("Courts"),
+        SubjectArea.from_name("Electronic monitoring"),
+        SubjectArea.from_name("Finance"),
+        SubjectArea.from_name("General"),
+        SubjectArea.from_name("Interventions"),
+        SubjectArea.from_name("OPG"),
+        SubjectArea.from_name("People"),
+        SubjectArea.from_name("Prison"),
+        SubjectArea.from_name("Probation"),
+        SubjectArea.from_name("Property"),
+        SubjectArea.from_name("Risk"),
+    ]
+
+    @classmethod
+    def get_top_level(cls, name):
+        matches = [i for i in cls.TOP_LEVEL if i.display_name == name]
+        return matches[0] if matches else None
+
+    @classmethod
+    def is_subject_area(cls, name):
+        return cls.get_top_level(name) is not None

datahub_client/graphql/getChartDetails.graphql

@@ -39,15 +39,7 @@ query getChartDetails($urn: String!) {
       }
     }
     tags {
-      tags {
-        tag {
-          urn
-          properties {
-            name
-            description
-          }
-        }
-      }
+      ...globalTagsFields
     }
     domain {
       domain {
@@ -80,6 +72,21 @@ query getChartDetails($urn: String!) {
   }
 }
 
+fragment globalTagsFields on GlobalTags {
+  tags {
+    tag {
+      urn
+      type
+      name
+      properties {
+        name
+        colorHex
+      }
+    }
+    associatedUrn
+  }
+}
+
 fragment ownershipFields on Ownership {
   owners {
     owner {

datahub_client/graphql/getContainerDetails.graphql

@@ -151,6 +151,7 @@ fragment globalTagsFields on GlobalTags {
     tag {
       urn
       type
+      name
       properties {
         name
         colorHex

datahub_client/graphql/getDashboardDetails.graphql

@@ -10,6 +10,9 @@ query getDashboard($urn: String!) {
     ownership {
       ...ownershipFields
     }
+    tags {
+      ...globalTagsFields
+    }
     properties {
       name
       description
@@ -55,6 +58,21 @@ query getDashboard($urn: String!) {
   }
 }
 
+fragment globalTagsFields on GlobalTags {
+  tags {
+    tag {
+      urn
+      type
+      name
+      properties {
+        name
+        colorHex
+      }
+    }
+    associatedUrn
+  }
+}
+
 fragment ownershipFields on Ownership {
   owners {
     owner {

datahub_client/graphql/getDatasetDetails.graphql

@@ -113,15 +113,7 @@ query getDatasetDetails($urn: String!) {
       description
     }
     tags {
-      tags {
-        tag {
-          urn
-          properties {
-            name
-            description
-          }
-        }
-      }
+      ...globalTagsFields
     }
     lastIngested
     domain {
@@ -173,6 +165,21 @@ query getDatasetDetails($urn: String!) {
   }
 }
 
+fragment globalTagsFields on GlobalTags {
+  tags {
+    tag {
+      urn
+      type
+      name
+      properties {
+        name
+        colorHex
+      }
+    }
+    associatedUrn
+  }
+}
+
 fragment ownershipFields on Ownership {
   owners {
     owner {

datahub_client/graphql/listSubjectAreas.graphql

@@ -0,0 +1,26 @@
+query listSubjectAreas(
+    $filters:[FacetFilterInput!]
+    $query: String!
+    $types: [EntityType!]
+) {
+  aggregateAcrossEntities(
+      input: {searchFlags: {maxAggValues:100}, query: $query, types: $types, facets: ["tags"], orFilters: [{and: $filters}]}
+  ) {
+    facets {
+      field
+      aggregations {
+        value
+        count
+        entity {
+          urn
+          ... on Tag {
+            name
+            properties {
+              name
+            }
+          }
+        }
+      }
+    }
+  }
+}

datahub_client/graphql/loader.py

@@ -0,0 +1,23 @@
+import logging
+from importlib.resources import files
+
+from ..exceptions import CatalogueError
+
+GRAPHQL_FILES_PATH = "datahub_client.graphql"
+GRAPHQL_FILE_EXTENSION = ".graphql"
+
+logger = logging.getLogger(__name__)
+
+
+def get_graphql_query(graphql_query_file_name: str) -> str:
+    query_text = (
+        files(GRAPHQL_FILES_PATH)
+        .joinpath(f"{graphql_query_file_name}{GRAPHQL_FILE_EXTENSION}")
+        .read_text()
+    )
+    if not query_text:
+        logger.error("No graphql query file found for %s", graphql_query_file_name)
+        raise CatalogueError(
+            f"No graphql query file found for {graphql_query_file_name}"
+        )
+    return query_text

datahub_client/graphql/search.graphql

@@ -64,15 +64,7 @@ query Search(
             ...ownershipFields
           }
           tags {
-            tags {
-              tag {
-                urn
-                properties {
-                  name
-                  description
-                }
-              }
-            }
+            ...globalTagsFields
           }
           glossaryTerms {
             terms {
@@ -134,15 +126,7 @@ query Search(
             }
           }
           tags {
-            tags {
-              tag {
-                urn
-                properties {
-                  name
-                  description
-                }
-              }
-            }
+            ...globalTagsFields
           }
           glossaryTerms {
             terms {
@@ -194,15 +178,7 @@ query Search(
             description
           }
           tags {
-            tags {
-              tag {
-                urn
-                properties {
-                  name
-                  description
-                }
-              }
-            }
+            ...globalTagsFields
           }
           glossaryTerms {
             terms {
@@ -254,15 +230,7 @@ query Search(
             }
           }
           tags {
-            tags {
-              tag {
-                urn
-                properties {
-                  name
-                  description
-                }
-              }
-            }
+            ...globalTagsFields
           }
           glossaryTerms {
             terms {
@@ -281,6 +249,21 @@ query Search(
   }
 }
 
+fragment globalTagsFields on GlobalTags {
+  tags {
+    tag {
+      urn
+      type
+      name
+      properties {
+        name
+        colorHex
+      }
+    }
+    associatedUrn
+  }
+}
+
 fragment ownershipFields on Ownership {
   owners {
     owner {