From 0b5c00abe9db1268653f6f785e316be1bef8367e Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 09:31:39 +0100 Subject: [PATCH 1/9] return parent container for datasets in search --- .../data_platform_catalogue/client/graphql/search.graphql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/datahub-client/data_platform_catalogue/client/graphql/search.graphql b/lib/datahub-client/data_platform_catalogue/client/graphql/search.graphql index 672cd237..40f9b470 100644 --- a/lib/datahub-client/data_platform_catalogue/client/graphql/search.graphql +++ b/lib/datahub-client/data_platform_catalogue/client/graphql/search.graphql @@ -128,6 +128,13 @@ query Search( platform { name } + container { + urn + properties { + name + qualifiedName + } + } subTypes { typeNames } From 9137f09fc9d487089e6d8c48ed6615f8922f560f Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 09:32:32 +0100 Subject: [PATCH 2/9] parse parent container metadata for dataset in search query --- .../data_platform_catalogue/client/search.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/datahub-client/data_platform_catalogue/client/search.py b/lib/datahub-client/data_platform_catalogue/client/search.py index a0029baf..5d869fc3 100644 --- a/lib/datahub-client/data_platform_catalogue/client/search.py +++ b/lib/datahub-client/data_platform_catalogue/client/search.py @@ -231,12 +231,25 @@ def _parse_result( terms = parse_glossary_terms(entity) last_modified = parse_last_modified(entity) name, display_name, qualified_name = parse_names(entity, properties) - + container = entity.get("container") domain = parse_domain(entity) metadata = { "owner": owner.display_name, "owner_email": owner.email, + "parent_container_display_name": ( + container.get("properties").get("name") + if container is not None + else None + ), + "parent_container_fqn": ( + container.get("properties").get("qualifiedName") + if container is not None + else None + ), + "parent_cotainer_urn": ( + container.get("urn") if container is not None else None + ), "total_parents": entity.get("relationships", {}).get("total", 0), "domain_name": domain.display_name, "domain_id": domain.urn, From ff5f34cc9713a5b439a9a823a9e454f8eb6315c3 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 09:33:52 +0100 Subject: [PATCH 3/9] display table name only with database as separately in search results --- templates/partial/search_result.html | 32 ++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/templates/partial/search_result.html b/templates/partial/search_result.html index 04a23420..f8bc3d11 100644 --- a/templates/partial/search_result.html +++ b/templates/partial/search_result.html @@ -7,7 +7,11 @@

{% with result_type=result.result_type.name|lower %} - {{result.fully_qualified_name}} + {% if result.display_name %} + {{result.display_name}} + {% else %} + {{result.fully_qualified_name}} + {% endif %} {% endwith %} {% if result.result_type.name == "TABLE" %} @@ -34,19 +38,29 @@

TBC
  • - Domain name: + Domain: {{result.metadata.domain_name}}
    • -
  • + {% if result.result_type.name == "TABLE" %} +
  • + Database: + {% if result.metadata.parent_container_display_name %} + {{result.metadata.parent_container_display_name}} + {% else %} + {{result.metadata.parent_container_fqn}} + {% endif %} +
    • + {% endif %} +
  • Tags: - {% if result.tags_to_display %} - {% for tag in result.tags_to_display %} - {{ tag }}{% if not forloop.last %}, {% endif %} - {% endfor %} - {% endif %} + {% if result.tags_to_display %} + {% for tag in result.tags_to_display %} + {{ tag }}{% if not forloop.last %}, {% endif %} + {% endfor %} + {% endif %} -
    • + {% if result.matches %}
  • Matched fields: From b4c606e858baaec4b8c1ac661a05381e432c13d3 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 10:06:05 +0100 Subject: [PATCH 4/9] correct typo --- lib/datahub-client/data_platform_catalogue/client/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datahub-client/data_platform_catalogue/client/search.py b/lib/datahub-client/data_platform_catalogue/client/search.py index 5d869fc3..fc632bb2 100644 --- a/lib/datahub-client/data_platform_catalogue/client/search.py +++ b/lib/datahub-client/data_platform_catalogue/client/search.py @@ -247,7 +247,7 @@ def _parse_result( if container is not None else None ), - "parent_cotainer_urn": ( + "parent_container_urn": ( container.get("urn") if container is not None else None ), "total_parents": entity.get("relationships", {}).get("total", 0), From e5223481ed57a32eb4f36e30fa36eae72804ede6 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 10:06:45 +0100 Subject: [PATCH 5/9] update client search tests --- .../tests/client/datahub/test_search.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lib/datahub-client/tests/client/datahub/test_search.py b/lib/datahub-client/tests/client/datahub/test_search.py index b09a7a26..dbbebdbf 100644 --- a/lib/datahub-client/tests/client/datahub/test_search.py +++ b/lib/datahub-client/tests/client/datahub/test_search.py @@ -79,6 +79,13 @@ def test_one_search_result(mock_graph, searcher): "type": "DATASET", "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", # noqa E501 "platform": {"name": "bigquery"}, + "container": { + "urn": "urn:li:container:abc", + "properties": { + "name": "abc", + "qualifiedName": None, + }, + }, "ownership": None, "name": "calm-pagoda-323403.jaffle_shop.customers", "properties": { @@ -123,6 +130,9 @@ def test_one_search_result(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": "abc", + "parent_container_fqn": None, + "parent_container_urn": "urn:li:container:abc", "total_parents": 0, "domain_name": "HMPPS", "domain_id": "urn:li:domain:3dc18e48-c062-4407-84a9-73e23f768023", @@ -161,6 +171,7 @@ def test_dataset_result(mock_graph, searcher): "type": "DATASET", "urn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", # noqa E501 "platform": {"name": "bigquery"}, + "container": None, "ownership": None, "name": "calm-pagoda-323403.jaffle_shop.customers", "properties": { @@ -209,6 +220,9 @@ def test_dataset_result(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "HMPPS", "domain_id": "urn:li:domain:3dc18e48-c062-4407-84a9-73e23f768023", @@ -300,6 +314,9 @@ def test_full_page(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -331,6 +348,9 @@ def test_full_page(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -360,6 +380,9 @@ def test_full_page(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -439,6 +462,9 @@ def test_query_match(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -514,6 +540,9 @@ def test_result_with_owner(mock_graph, searcher): metadata={ "owner": "Shannon Lovett", "owner_email": "shannon@longtail.com", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -884,6 +913,9 @@ def test_search_for_charts(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -1056,6 +1088,9 @@ def test_tag_to_display(tags, result): metadata={ "owner": "", "owner_email": "", + "parent_container_display_name": None, + "parent_container_fqn": None, + "parent_container_urn": None, "total_parents": 0, "parents": [], "domain_name": "", From 8afe0b00f57d50bc64fe54e7d4aeb0d69fda889b Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 11:44:17 +0100 Subject: [PATCH 6/9] remove redundant logic from template --- templates/partial/search_result.html | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/templates/partial/search_result.html b/templates/partial/search_result.html index f8bc3d11..9637ca06 100644 --- a/templates/partial/search_result.html +++ b/templates/partial/search_result.html @@ -7,11 +7,7 @@

    {% with result_type=result.result_type.name|lower %} - {% if result.display_name %} - {{result.display_name}} - {% else %} - {{result.fully_qualified_name}} - {% endif %} + {{result.name}} {% endwith %} {% if result.result_type.name == "TABLE" %} @@ -44,11 +40,7 @@

    {% if result.result_type.name == "TABLE" %}
  • Database: - {% if result.metadata.parent_container_display_name %} - {{result.metadata.parent_container_display_name}} - {% else %} - {{result.metadata.parent_container_fqn}} - {% endif %} + {{result.parent_entity.display_name}}
    • {% endif %}
  • From 0deccdf7af2f1bfe2cd5ca4ea5531f4cc8cd6744 Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 11:45:09 +0100 Subject: [PATCH 7/9] expand SearchResult to include parent entity --- lib/datahub-client/data_platform_catalogue/search_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/datahub-client/data_platform_catalogue/search_types.py b/lib/datahub-client/data_platform_catalogue/search_types.py index 3c43fcd5..9adc5cfc 100644 --- a/lib/datahub-client/data_platform_catalogue/search_types.py +++ b/lib/datahub-client/data_platform_catalogue/search_types.py @@ -5,7 +5,7 @@ from enum import Enum, auto from typing import Any -from data_platform_catalogue.entities import GlossaryTermRef, TagRef +from data_platform_catalogue.entities import EntityRef, GlossaryTermRef, TagRef class ResultType(Enum): @@ -68,6 +68,7 @@ class SearchResult: glossary_terms: list[GlossaryTermRef] = field(default_factory=list) last_modified: datetime | None = None created: datetime | None = None + parent_entity: EntityRef | None = None tags_to_display: list[str] = field(init=False) def __post_init__(self): From d68cf3ac0661a1bfcf99ca75eb905241b05d8b9b Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 11:46:02 +0100 Subject: [PATCH 8/9] return parent as EnityRef in search result --- .../data_platform_catalogue/client/search.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/datahub-client/data_platform_catalogue/client/search.py b/lib/datahub-client/data_platform_catalogue/client/search.py index fc632bb2..3ca37777 100644 --- a/lib/datahub-client/data_platform_catalogue/client/search.py +++ b/lib/datahub-client/data_platform_catalogue/client/search.py @@ -17,6 +17,7 @@ parse_properties, parse_tags, ) +from data_platform_catalogue.entities import EntityRef from data_platform_catalogue.search_types import ( FacetOption, MultiSelectFilter, @@ -232,24 +233,15 @@ def _parse_result( last_modified = parse_last_modified(entity) name, display_name, qualified_name = parse_names(entity, properties) container = entity.get("container") + if container: + container_name, container_display_name, container_qualified_name = ( + parse_names(container, container.get("properties")) + ) domain = parse_domain(entity) metadata = { "owner": owner.display_name, "owner_email": owner.email, - "parent_container_display_name": ( - container.get("properties").get("name") - if container is not None - else None - ), - "parent_container_fqn": ( - container.get("properties").get("qualifiedName") - if container is not None - else None - ), - "parent_container_urn": ( - container.get("urn") if container is not None else None - ), "total_parents": entity.get("relationships", {}).get("total", 0), "domain_name": domain.display_name, "domain_id": domain.urn, @@ -270,6 +262,11 @@ def _parse_result( name=name, display_name=display_name, fully_qualified_name=qualified_name, + parent_entity=( + EntityRef(urn=container.get("urn"), display_name=container_display_name) + if container + else None + ), description=properties.get("description", ""), metadata=metadata, tags=tags, From a0d1527d6cfd2aa56b247f07e5a0e881e278a38a Mon Sep 17 00:00:00 2001 From: LavMatt Date: Wed, 3 Jul 2024 11:46:39 +0100 Subject: [PATCH 9/9] align tests with suggested changes --- .../tests/client/datahub/test_search.py | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/lib/datahub-client/tests/client/datahub/test_search.py b/lib/datahub-client/tests/client/datahub/test_search.py index dbbebdbf..f128674b 100644 --- a/lib/datahub-client/tests/client/datahub/test_search.py +++ b/lib/datahub-client/tests/client/datahub/test_search.py @@ -7,6 +7,7 @@ from data_platform_catalogue.entities import ( AccessInformation, DataSummary, + EntityRef, FurtherInformation, TagRef, UsageRestrictions, @@ -130,9 +131,6 @@ def test_one_search_result(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": "abc", - "parent_container_fqn": None, - "parent_container_urn": "urn:li:container:abc", "total_parents": 0, "domain_name": "HMPPS", "domain_id": "urn:li:domain:3dc18e48-c062-4407-84a9-73e23f768023", @@ -150,6 +148,7 @@ def test_one_search_result(mock_graph, searcher): tags=[], last_modified=None, created=None, + parent_entity=EntityRef(urn="urn:li:container:abc", display_name="abc"), ) ], facets=SearchFacets(facets={}), @@ -220,9 +219,6 @@ def test_dataset_result(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "HMPPS", "domain_id": "urn:li:domain:3dc18e48-c062-4407-84a9-73e23f768023", @@ -314,9 +310,6 @@ def test_full_page(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -348,9 +341,6 @@ def test_full_page(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -380,9 +370,6 @@ def test_full_page(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -462,9 +449,6 @@ def test_query_match(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -540,9 +524,6 @@ def test_result_with_owner(mock_graph, searcher): metadata={ "owner": "Shannon Lovett", "owner_email": "shannon@longtail.com", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -913,9 +894,6 @@ def test_search_for_charts(mock_graph, searcher): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "domain_name": "", "domain_id": "", @@ -1088,9 +1066,6 @@ def test_tag_to_display(tags, result): metadata={ "owner": "", "owner_email": "", - "parent_container_display_name": None, - "parent_container_fqn": None, - "parent_container_urn": None, "total_parents": 0, "parents": [], "domain_name": "",