From eb1e5746d18de8838abb141702016bf1c40d5d32 Mon Sep 17 00:00:00 2001 From: aibunny Date: Sat, 10 Feb 2024 12:51:26 +0300 Subject: [PATCH 1/6] Added dbt labels as part of columns info when pushing decscriptions --- dbt_superset_lineage/push_descriptions.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/dbt_superset_lineage/push_descriptions.py b/dbt_superset_lineage/push_descriptions.py index 1890c3e..f0be349 100644 --- a/dbt_superset_lineage/push_descriptions.py +++ b/dbt_superset_lineage/push_descriptions.py @@ -63,7 +63,7 @@ def get_datasets_from_superset(superset, superset_db_id): break assert datasets, "There are no datasets in Superset!" - + return datasets @@ -152,7 +152,9 @@ def merge_columns_info(dataset, tables): sst_description = dataset['description'] dbt_description = tables.get(key, {}).get('description') - + + + sst_owners = dataset['owners'] columns_new = [] @@ -166,16 +168,31 @@ def merge_columns_info(dataset, tables): 'id': sst_column['id'] } - # add column descriptions + # add column descriptions and labels if column_name in dbt_columns \ and 'description' in dbt_columns[column_name] \ and (sst_column['expression'] is None # database columns or sst_column['expression'] == ''): + description = dbt_columns[column_name]['description'] description = convert_markdown_to_plain_text(description) + + + # Check if label exists in dbt_columns + if 'label' in dbt_columns[column_name]: + label = dbt_columns[column_name]['label'] + label = convert_markdown_to_plain_text(label) + else: + label = sst_column['verbose_name'] + + else: description = sst_column['description'] + + + column_new['description'] = description + column_new['verbose_name'] = label columns_new.append(column_new) From 6c51b4c255148d956f51f79d87b3964145a84c3c Mon Sep 17 00:00:00 2001 From: aibunny Date: Mon, 12 Feb 2024 19:28:36 +0300 Subject: [PATCH 2/6] update to get labels from meta.lable --- dbt_superset_lineage/push_descriptions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt_superset_lineage/push_descriptions.py b/dbt_superset_lineage/push_descriptions.py index f0be349..64aafce 100644 --- a/dbt_superset_lineage/push_descriptions.py +++ b/dbt_superset_lineage/push_descriptions.py @@ -178,13 +178,13 @@ def merge_columns_info(dataset, tables): description = convert_markdown_to_plain_text(description) - # Check if label exists in dbt_columns - if 'label' in dbt_columns[column_name]: - label = dbt_columns[column_name]['label'] + # Check if 'meta' field exists in dbt_columns + if 'meta' in dbt_columns[column_name] and 'label' in dbt_columns[column_name]['meta']: + label = dbt_columns[column_name]['meta']['label'] label = convert_markdown_to_plain_text(label) else: - label = sst_column['verbose_name'] - + label = sst_column['verbose_name'] + else: description = sst_column['description'] From 44b5d30ecc9e215cc5bf5b2e0658e381deafec25 Mon Sep 17 00:00:00 2001 From: aibunny Date: Tue, 13 Feb 2024 22:11:51 +0300 Subject: [PATCH 3/6] update --- dbt_superset_lineage/push_descriptions.py | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dbt_superset_lineage/push_descriptions.py b/dbt_superset_lineage/push_descriptions.py index 64aafce..74bf290 100644 --- a/dbt_superset_lineage/push_descriptions.py +++ b/dbt_superset_lineage/push_descriptions.py @@ -145,7 +145,7 @@ def convert_markdown_to_plain_text(md_string): def merge_columns_info(dataset, tables): logging.info("Merging columns info from Superset and manifest.json file.") - + key = dataset['key'] sst_columns = dataset['columns'] dbt_columns = tables.get(key, {}).get('columns', {}) @@ -176,22 +176,22 @@ def merge_columns_info(dataset, tables): description = dbt_columns[column_name]['description'] description = convert_markdown_to_plain_text(description) - - - # Check if 'meta' field exists in dbt_columns - if 'meta' in dbt_columns[column_name] and 'label' in dbt_columns[column_name]['meta']: - label = dbt_columns[column_name]['meta']['label'] - label = convert_markdown_to_plain_text(label) - else: - label = sst_column['verbose_name'] - - else: description = sst_column['description'] - - column_new['description'] = description + + # Check if 'meta' field exists in dbt_columns + if column_name in dbt_columns \ + and 'meta' in dbt_columns[column_name] \ + and 'label' in dbt_columns[column_name]['meta']: + label = dbt_columns[column_name]['meta']['label'] + label = convert_markdown_to_plain_text(label) + + else: + label = sst_column['verbose_name'] + + column_new['verbose_name'] = label columns_new.append(column_new) From eb6a61e88c95c9b2821f1fefc6b64317031192fa Mon Sep 17 00:00:00 2001 From: Bunny <98146814+aibunny@users.noreply.github.com> Date: Fri, 16 Feb 2024 09:32:25 +0300 Subject: [PATCH 4/6] Update dbt_superset_lineage/push_descriptions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michal Koláček <57877739+one-data-cookie@users.noreply.github.com> --- dbt_superset_lineage/push_descriptions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt_superset_lineage/push_descriptions.py b/dbt_superset_lineage/push_descriptions.py index 74bf290..d761895 100644 --- a/dbt_superset_lineage/push_descriptions.py +++ b/dbt_superset_lineage/push_descriptions.py @@ -63,7 +63,6 @@ def get_datasets_from_superset(superset, superset_db_id): break assert datasets, "There are no datasets in Superset!" - return datasets From ca1d7f4be2a3fdbce7821e8a3da894392f749982 Mon Sep 17 00:00:00 2001 From: aibunny Date: Sat, 17 Feb 2024 10:56:32 +0300 Subject: [PATCH 5/6] update --- dbt_superset_lineage/push_descriptions.py | 39 +++++++++++------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/dbt_superset_lineage/push_descriptions.py b/dbt_superset_lineage/push_descriptions.py index d761895..2e35bdd 100644 --- a/dbt_superset_lineage/push_descriptions.py +++ b/dbt_superset_lineage/push_descriptions.py @@ -168,30 +168,29 @@ def merge_columns_info(dataset, tables): } # add column descriptions and labels - if column_name in dbt_columns \ - and 'description' in dbt_columns[column_name] \ + if column_name in dbt_columns : + if 'description' in dbt_columns[column_name] \ and (sst_column['expression'] is None # database columns or sst_column['expression'] == ''): - description = dbt_columns[column_name]['description'] - description = convert_markdown_to_plain_text(description) - else: - description = sst_column['description'] - - column_new['description'] = description - - # Check if 'meta' field exists in dbt_columns - if column_name in dbt_columns \ - and 'meta' in dbt_columns[column_name] \ - and 'label' in dbt_columns[column_name]['meta']: - label = dbt_columns[column_name]['meta']['label'] - label = convert_markdown_to_plain_text(label) - - else: - label = sst_column['verbose_name'] - + description = dbt_columns[column_name]['description'] + description = convert_markdown_to_plain_text(description) + else: + description = sst_column['description'] - column_new['verbose_name'] = label + column_new['description'] = description + + # Check if 'meta' field exists in dbt_columns + if 'meta' in dbt_columns[column_name] \ + and 'label' in dbt_columns[column_name]['meta']: + label = dbt_columns[column_name]['meta']['label'] + label = convert_markdown_to_plain_text(label) + + else: + label = sst_column['verbose_name'] + + + column_new['verbose_name'] = label columns_new.append(column_new) From 8ef351af571f5ccdd29eb3d349d003a7e973405c Mon Sep 17 00:00:00 2001 From: aibunny Date: Sat, 17 Feb 2024 10:57:50 +0300 Subject: [PATCH 6/6] update --- dbt_superset_lineage/push_descriptions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt_superset_lineage/push_descriptions.py b/dbt_superset_lineage/push_descriptions.py index 2e35bdd..a055ed0 100644 --- a/dbt_superset_lineage/push_descriptions.py +++ b/dbt_superset_lineage/push_descriptions.py @@ -187,8 +187,7 @@ def merge_columns_info(dataset, tables): label = convert_markdown_to_plain_text(label) else: - label = sst_column['verbose_name'] - + label = sst_column['verbose_name'] column_new['verbose_name'] = label