Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DC-3717] Remove 9 functions for bq_utils.py #1851

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

[DC-3717] Remove 9 functions for bq_utils.py #1851

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
071f025
[DC-3717] Remove 9 functions for bq_utils.py
Jan 16, 2024
66f7a6d
[DC-3717] Linting
Jan 16, 2024
fb825c6
[DC-3717] Merge in develop
Jan 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 0 additions & 72 deletions data_steward/bq_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,36 +46,11 @@ def get_rdr_project_id():
return os.environ.get('RDR_PROJECT_ID')


@deprecated(reason='get_output_project_id is deprecated')
def get_output_project_id():
return os.environ.get('OUTPUT_PROJECT_ID')


@deprecated(reason='Use common.BIGQUERY_DATASET_ID instead')
def get_dataset_id():
return os.environ.get('BIGQUERY_DATASET_ID')


@deprecated(reason='Use common.UNIONED_DATASET_ID instead')
def get_unioned_dataset_id():
return os.environ.get('UNIONED_DATASET_ID')


@deprecated(reason='Use common.RDR_DATASET_ID instead')
def get_rdr_dataset_id():
return os.environ.get('RDR_DATASET_ID')


@deprecated(reason='get_combined_snapshot_dataset_id is deprecated')
def get_combined_snapshot_dataset_id():
return os.environ.get('COMBINED_SNAPSHOT')


@deprecated(reason='Use common.COMBINED_DATASET_ID instead')
def get_combined_dataset_id():
return os.environ.get('COMBINED_DATASET_ID')


@deprecated(reason='get_combined_deid_clean_dataset_id is deprecated')
def get_combined_deid_clean_dataset_id():
return os.environ.get('COMBINED_DEID_CLEAN_DATASET_ID')
Expand All @@ -97,11 +72,6 @@ def get_retraction_sandbox_dataset_id():
return os.environ.get('RETRACTION_SANDBOX_DATASET_ID')


@deprecated(reason='get_fitbit_dataset_id is deprecated')
def get_fitbit_dataset_id():
return os.environ.get('FITBIT_DATASET_ID')


def get_retraction_dataset_ids_table():
"""
BigQuery table containing dataset ids from which to retract, on separate rows.
Expand All @@ -128,29 +98,6 @@ def get_retraction_submission_folder():
return os.environ.get('RETRACTION_SUBMISSION_FOLDER')


@deprecated(reason='get_combined_deid_dataset_id is deprecated')
def get_combined_deid_dataset_id():
return os.environ.get('COMBINED_DEID_DATASET_ID')


@deprecated(reason='get_validation_results_dataset_id is deprecated')
def get_validation_results_dataset_id():
"""
Get the Validation dataset id.

If the environment variable has not been set, default to the defined name,
set the environment variable, and return the dataset name.

:return: A name for the validation dataset id.
"""
dataset_id = os.environ.get(bq_consts.MATCH_DATASET, bq_consts.BLANK)
if dataset_id == bq_consts.BLANK:
date_string = datetime.now().strftime(bq_consts.DATE_FORMAT)
dataset_id = bq_consts.VALIDATION_DATASET_FORMAT.format(date_string)
os.environ[bq_consts.MATCH_DATASET] = dataset_id
return dataset_id


@deprecated(
reason='Discovery client is being replaced by gcloud.gcs.BigQueryClient()')
def create_service():
Expand Down Expand Up @@ -957,25 +904,6 @@ def get_hpo_info():
return hpo_list


@deprecated(reason='Use gcloud.bq.BigQueryClient.get_hpo_bucket_info() instead')
def get_hpo_bucket_info():
hpo_list = []
project_id = app_identity.get_application_id()
hpo_table_query = bq_consts.GET_HPO_CONTENTS_QUERY.format(
project_id=project_id,
TABLES_DATASET_ID=bq_consts.LOOKUP_TABLES_DATASET_ID,
HPO_SITE_TABLE=bq_consts.HPO_ID_BUCKET_NAME_TABLE_ID)
hpo_response = query(hpo_table_query)
hpo_table_contents = response2rows(hpo_response)
for hpo_table_row in hpo_table_contents:
hpo_id = hpo_table_row[bq_consts.HPO_ID.lower()].lower()
hpo_bucket = hpo_table_row[bq_consts.BUCKET_NAME].lower()
if hpo_id:
hpo_dict = {"hpo_id": hpo_id, "bucket_name": hpo_bucket}
hpo_list.append(hpo_dict)
return hpo_list


@deprecated(
reason='Use gcloud.bq.BigQueryClient.get_hpo_site_state_info() instead')
def get_hpo_site_state_info():
Expand Down