From cec2a89870141b8c26c1c92c89b3f71a83a4195c Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 15 Aug 2024 10:01:37 -0700 Subject: [PATCH 1/6] update code pylint and flake8 --- podaac/merger/preprocess_worker.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/podaac/merger/preprocess_worker.py b/podaac/merger/preprocess_worker.py index 4bedba33..979448bb 100644 --- a/podaac/merger/preprocess_worker.py +++ b/podaac/merger/preprocess_worker.py @@ -236,14 +236,10 @@ def _run_multi_core(file_list: list[Path], if var_info is None: var_info = result['var_info'] elif var_info != result['var_info']: - if set(var_info.keys()).difference(result['var_info']): - # If not all variables match, only compare variables that intersect - intersecting_vars = set(var_info).intersection(result['var_info']) - if list( - map(var_info.get, intersecting_vars) - ) != list(map(result['var_info'].get, intersecting_vars)): - raise RuntimeError('Variable schemas are inconsistent between granules') - var_info.update(result['var_info']) + intersecting_vars = set(var_info).intersection(result['var_info']) + if any(var_info[var] != result['var_info'][var] for var in intersecting_vars): + raise RuntimeError('Variable schemas are inconsistent between granules') + var_info.update(result['var_info']) # The following data requires accumulation methods merge_max_dims(max_dims, result['max_dims']) @@ -259,6 +255,8 @@ def _run_multi_core(file_list: list[Path], default=str ) + print(var_info) + return { 'group_list': group_list, 'max_dims': max_dims, From ab0a0c60ede4a8ff3f8e21f904d2a386f4ce02af Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 15 Aug 2024 10:07:32 -0700 Subject: [PATCH 2/6] remove print --- podaac/merger/preprocess_worker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/podaac/merger/preprocess_worker.py b/podaac/merger/preprocess_worker.py index 979448bb..7dd98022 100644 --- a/podaac/merger/preprocess_worker.py +++ b/podaac/merger/preprocess_worker.py @@ -255,8 +255,6 @@ def _run_multi_core(file_list: list[Path], default=str ) - print(var_info) - return { 'group_list': group_list, 'max_dims': max_dims, From d25012043d3481bd8b7c9f083a784b392c789dca Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 15 Aug 2024 10:15:42 -0700 Subject: [PATCH 3/6] add harmony deployment --- .github/workflows/build-pipeline.yml | 13 ++++++ deployment/harmony_deploy.py | 67 ++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 deployment/harmony_deploy.py diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml index c28f1a0e..62e6b393 100644 --- a/.github/workflows/build-pipeline.yml +++ b/.github/workflows/build-pipeline.yml @@ -243,6 +243,19 @@ jobs: pull: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Deploy Harmony + env: + ENV: ${{ env.venue }} + CMR_USER: ${{ secrets.CMR_USER }} + CMR_PASS: ${{ secrets.CMR_PASS }} + if: | + github.ref == 'refs/heads/main' || + startsWith(github.ref, 'refs/heads/release') + working-directory: deployment + run: + poetry run python harmony_deploy.py --tag ${{ env.software_version }} + # As of 2023/01/23 these steps below for scanning the Docker image with Snyk are failing. I've tried both the official Snyk # action https://github.com/snyk/actions/tree/master/docker and this method below of manually calling the CLI. # The error when using the official Snyk action is diff --git a/deployment/harmony_deploy.py b/deployment/harmony_deploy.py new file mode 100644 index 00000000..57e46f97 --- /dev/null +++ b/deployment/harmony_deploy.py @@ -0,0 +1,67 @@ +import os +import requests +import json +import logging +import argparse +from requests.auth import HTTPBasicAuth + +# Environment variables +ENV = os.getenv('ENV') +CMR_USER = os.getenv('CMR_USER') +CMR_PASS = os.getenv('CMR_PASS') + +def bearer_token() -> str: + tokens = [] + headers = {'Accept': 'application/json'} + url = f"https://{'uat.' if ENV == 'uat' else ''}urs.earthdata.nasa.gov/api/users" + + # First just try to get a token that already exists + try: + resp = requests.get(url + "/tokens", headers=headers, auth=HTTPBasicAuth(CMR_USER, CMR_PASS)) + response_content = json.loads(resp.content) + + for x in response_content: + tokens.append(x['access_token']) + + except Exception: # noqa E722 + logging.warning("Error getting the token - check user name and password", exc_info=True) + + # No tokens exist, try to create one + if not tokens: + try: + resp = requests.post(url + "/token", headers=headers, auth=HTTPBasicAuth(CMR_USER, CMR_PASS)) + response_content = json.loads(resp.content) + tokens.append(response_content['access_token']) + except Exception: # noqa E722 + logging.warning("Error getting the token - check user name and password", exc_info=True) + + # If still no token, then we can't do anything + if not tokens: + raise RuntimeError("Unable to get bearer token from EDL") + + return next(iter(tokens)) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="Update the service image tag.") + parser.add_argument("--tag", help="The new tag version to update.", required=True) + args = parser.parse_args() + + url = f"https://harmony.{'uat.' if ENV == 'uat' else ''}earthdata.nasa.gov/service-image-tag/podaac-l2-subsetter" + token = bearer_token() + + headers = { + "Authorization": f"Bearer {token}", + "Content-type": "application/json" + } + data = { + "tag": args.tag + } + + response = requests.put(url, headers=headers, json=data) + + print(response.status_code) + try: + print(response.json()) + except json.JSONDecodeError: + print("Response content is not in JSON format") From ae850b55a49a7c28e85c0346cc688188c205a047 Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 15 Aug 2024 10:26:13 -0700 Subject: [PATCH 4/6] update harmony deployment url --- deployment/harmony_deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/harmony_deploy.py b/deployment/harmony_deploy.py index 57e46f97..38035bef 100644 --- a/deployment/harmony_deploy.py +++ b/deployment/harmony_deploy.py @@ -47,7 +47,7 @@ def bearer_token() -> str: parser.add_argument("--tag", help="The new tag version to update.", required=True) args = parser.parse_args() - url = f"https://harmony.{'uat.' if ENV == 'uat' else ''}earthdata.nasa.gov/service-image-tag/podaac-l2-subsetter" + url = f"https://harmony.{'uat.' if ENV == 'uat' else ''}earthdata.nasa.gov/service-image-tag/podaac-concise" token = bearer_token() headers = { From 113bd04d48d32c8809a198ccab6f2a03bdc00289 Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 15 Aug 2024 10:59:07 -0700 Subject: [PATCH 5/6] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b676846a..d59662bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Changed -### Changed ### Deprecated ### Removed ### Fixed + - Variable Merging + - Fixed way we merge variables when granules in a collection have varying variables. ## [0.9.0] From dee190888e021872b8803d8bb3c86b2ca03bded1 Mon Sep 17 00:00:00 2001 From: Simon Liu Date: Thu, 15 Aug 2024 10:59:43 -0700 Subject: [PATCH 6/6] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d59662bf..69746444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added + - Update Github Actions + - Added harmony deployment into github actions. ### Changed ### Deprecated ### Removed