From d583cdd31b9ac248c04a4322af99c8a5adf29796 Mon Sep 17 00:00:00 2001 From: HaneenT Date: Tue, 7 May 2024 15:03:46 -0400 Subject: [PATCH 01/15] KPMP-5199: Add build action --- .github/workflows/build-python-project.yml | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/build-python-project.yml diff --git a/.github/workflows/build-python-project.yml b/.github/workflows/build-python-project.yml new file mode 100644 index 0000000..731f720 --- /dev/null +++ b/.github/workflows/build-python-project.yml @@ -0,0 +1,39 @@ +name: Build atlas-file-service docker image + +on: + push: + +jobs: + build-python-project: + env: + IMAGE_TAG: 3.4 + runs-on: ubuntu-latest + steps: + - name: Get branch names + id: branch-names + uses: tj-actions/branch-names@v8 + + - name: Get current branch name + if: steps.branch-names.outputs.is_default == 'false' + run: | + echo "Running on branch: ${{ steps.branch-names.outputs.current_branch }}" + + - name: Login to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + username: ${{ secrets.ENV_DOCKER_USER }} + password: ${{ secrets.ENV_DOCKER_PASS }} + + - name: Build and push to Docker Hub if branch is develop + if: steps.branch-names.outputs.current_branch == 'develop' + uses: docker/build-push-action@v5 + with: + push: true + tags: kingstonduo/atlas-file-service:$IMAGE_TAG + + - name: Build and push to Docker Hub if branch is not develop + if: steps.branch-names.outputs.current_branch != 'develop' + uses: docker/build-push-action@v5 + with: + push: true + tags: kingstonduo/atlas-file-service:${{ steps.branch-names.outputs.current_branch }} \ No newline at end of file From 4573e743de2242faae671f9ba8b0843ffdd469cd Mon Sep 17 00:00:00 2001 From: HaneenT Date: Tue, 7 May 2024 15:07:42 -0400 Subject: [PATCH 02/15] rename file --- .../workflows/{build-python-project.yml => build_project.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{build-python-project.yml => build_project.yml} (98%) diff --git a/.github/workflows/build-python-project.yml b/.github/workflows/build_project.yml similarity index 98% rename from .github/workflows/build-python-project.yml rename to .github/workflows/build_project.yml index 731f720..0adc1ea 100644 --- a/.github/workflows/build-python-project.yml +++ b/.github/workflows/build_project.yml @@ -4,7 +4,7 @@ on: push: jobs: - build-python-project: + build-project: env: IMAGE_TAG: 3.4 runs-on: ubuntu-latest From d0ccc19bdb2f539a4e3db9fadf22a8490229f6ea Mon Sep 17 00:00:00 2001 From: HaneenT Date: Tue, 7 May 2024 15:09:00 -0400 Subject: [PATCH 03/15] small fix --- .github/workflows/{build_project.yml => build-project.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{build_project.yml => build-project.yml} (100%) diff --git a/.github/workflows/build_project.yml b/.github/workflows/build-project.yml similarity index 100% rename from .github/workflows/build_project.yml rename to .github/workflows/build-project.yml From f4f5b3d5a6e9ccb0abed854666fd1d93a2d68e12 Mon Sep 17 00:00:00 2001 From: HaneenT Date: Tue, 7 May 2024 15:53:28 -0400 Subject: [PATCH 04/15] KPMP-5199: fix env variable syntax --- .github/workflows/build-project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-project.yml b/.github/workflows/build-project.yml index 0adc1ea..2f80c17 100644 --- a/.github/workflows/build-project.yml +++ b/.github/workflows/build-project.yml @@ -29,11 +29,11 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: kingstonduo/atlas-file-service:$IMAGE_TAG + tags: kingstonduo/atlas-file-service:${{ steps.branch-names.outputs.current_branch }} - name: Build and push to Docker Hub if branch is not develop if: steps.branch-names.outputs.current_branch != 'develop' uses: docker/build-push-action@v5 with: push: true - tags: kingstonduo/atlas-file-service:${{ steps.branch-names.outputs.current_branch }} \ No newline at end of file + tags: kingstonduo/atlas-file-service:${{ env.IMAGE_TAG }} \ No newline at end of file From dddbc08297ebe0111572f00b8260fd9f0baa4239 Mon Sep 17 00:00:00 2001 From: HaneenT Date: Tue, 7 May 2024 15:55:01 -0400 Subject: [PATCH 05/15] fix env variable --- .github/workflows/build-project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-project.yml b/.github/workflows/build-project.yml index 2f80c17..40e5fe5 100644 --- a/.github/workflows/build-project.yml +++ b/.github/workflows/build-project.yml @@ -29,11 +29,11 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: kingstonduo/atlas-file-service:${{ steps.branch-names.outputs.current_branch }} + tags: kingstonduo/atlas-file-service:${{ env.IMAGE_TAG }} - name: Build and push to Docker Hub if branch is not develop if: steps.branch-names.outputs.current_branch != 'develop' uses: docker/build-push-action@v5 with: push: true - tags: kingstonduo/atlas-file-service:${{ env.IMAGE_TAG }} \ No newline at end of file + tags: kingstonduo/atlas-file-service:${{ steps.branch-names.outputs.current_branch }} \ No newline at end of file From 48c149711da6f1a6c94d7ebf83e91bf75180b186 Mon Sep 17 00:00:00 2001 From: rlreamy <34109594+rlreamy@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:54:45 -0400 Subject: [PATCH 06/15] Create changelog.md --- changelog.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog.md diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..51bb258 --- /dev/null +++ b/changelog.md @@ -0,0 +1,14 @@ +# Changelog + +## Release 1.4 [unreleased] +Brief summary of what's in this release: + + +### Breaking changes + +Breaking changes include any database updates needed, if we need to edit any files on system (like .env or certs, etc). Things that are outside of the code itself that need changed for the system to work. + + +### Non-breaking changes + +Just a place to keep track of things that have changed in the code that we may want to pay special attention to when smoke testing, etc. From 1bd391f995be48daaffc32a01f858bee7eae48cc Mon Sep 17 00:00:00 2001 From: dert1129 Date: Tue, 30 Jul 2024 12:57:08 -0400 Subject: [PATCH 07/15] use measurement protocol to send events to google analytics --- app.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app.py b/app.py index 3597f1d..67c10bf 100644 --- a/app.py +++ b/app.py @@ -7,6 +7,8 @@ from flask_cors import CORS import mysql.connector import logging +import requests +import json app = Flask(__name__) CORS(app) @@ -15,6 +17,9 @@ minioSecretKey = os.environ.get('MINIO_SECRET_KEY') s3Bucket = os.environ.get('BUCKET_NAME') minioUrl = os.environ.get('MINIO_URL') +apiSecret = os.environ.get("API_SECRET") +ga4Id = os.environ.get("GA4_ID") +url = "https://www.google-analytics.com/mp/collect?measurement_id=" + ga4Id + "&api_secret=" + apiSecret minioClient = Minio(minioUrl, access_key=minioAccessKey, secret_key=minioSecretKey, secure=False) s3_client = boto3.client( 's3', @@ -103,6 +108,18 @@ def downloadFile(packageId, objectName): try: objectNameFull = packageId + '/' + objectName object = minioClient.get_object(s3Bucket, objectNameFull, request_headers=None) + payload = { + "client_id": "XXXXXXXXXX.YYYYYYYYYY", + "events": [ + { + "name": "file_download", + "params": { + "file_name": objectName + } + } + ] + } + requests.post(url, json.dumps(payload), verify=True) return send_file(object, as_attachment=True, download_name=objectName) except S3Error as err: logger.error(err) From 85d0559cdac7c88653f14228e5c39a160f0f08e5 Mon Sep 17 00:00:00 2001 From: dert1129 Date: Tue, 30 Jul 2024 13:15:06 -0400 Subject: [PATCH 08/15] try json keyword --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 67c10bf..2780e31 100644 --- a/app.py +++ b/app.py @@ -119,7 +119,7 @@ def downloadFile(packageId, objectName): } ] } - requests.post(url, json.dumps(payload), verify=True) + requests.post(url, json=payload, verify=True) return send_file(object, as_attachment=True, download_name=objectName) except S3Error as err: logger.error(err) From 17e0e6af83a87439fb9d36bf111155880fe1f3f1 Mon Sep 17 00:00:00 2001 From: dert1129 Date: Tue, 30 Jul 2024 13:43:33 -0400 Subject: [PATCH 09/15] specify headers for post request --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 2780e31..0e1c745 100644 --- a/app.py +++ b/app.py @@ -119,7 +119,7 @@ def downloadFile(packageId, objectName): } ] } - requests.post(url, json=payload, verify=True) + requests.post(url, json=payload, headers={'content-type': 'application/json'}, verify=True) return send_file(object, as_attachment=True, download_name=objectName) except S3Error as err: logger.error(err) From 7391914f9e0bcc6160c947eda3755ceaeea5945a Mon Sep 17 00:00:00 2001 From: dert1129 Date: Tue, 30 Jul 2024 14:06:12 -0400 Subject: [PATCH 10/15] use file_name from database instead of object name in S3 --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 0e1c745..5cb4d7f 100644 --- a/app.py +++ b/app.py @@ -114,7 +114,7 @@ def downloadFile(packageId, objectName): { "name": "file_download", "params": { - "file_name": objectName + "file_name": result[0]["file_name"] } } ] From 47d66fa334cf75e253b0f79d623c13f559715aa6 Mon Sep 17 00:00:00 2001 From: dert1129 Date: Tue, 30 Jul 2024 14:22:56 -0400 Subject: [PATCH 11/15] trim the first 59 chars for file_name param --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 5cb4d7f..08098b8 100644 --- a/app.py +++ b/app.py @@ -99,7 +99,9 @@ def get_file_info_by_file_name(file_name): "SELECT * FROM repo_file_v WHERE file_name = %s", (file_name,), ) +# /api/v1/file/download/46669edb-0e3e-4470-8834-e9ca30a50bc3/ +# 4a45bcae-3203-4319-b455-b28965b1587a_20240604_OpenAccessClinicalData.csv' @app.route('/v1/file/download//', methods=['GET']) def downloadFile(packageId, objectName): @@ -114,7 +116,7 @@ def downloadFile(packageId, objectName): { "name": "file_download", "params": { - "file_name": result[0]["file_name"] + "file_name": result[0]["file_name"][:59] } } ] From d2d95a6fd64741fcae84ca427fd6923c65e57c33 Mon Sep 17 00:00:00 2001 From: dert1129 Date: Tue, 30 Jul 2024 15:09:56 -0400 Subject: [PATCH 12/15] fix file_name value --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 08098b8..44be0a1 100644 --- a/app.py +++ b/app.py @@ -116,7 +116,7 @@ def downloadFile(packageId, objectName): { "name": "file_download", "params": { - "file_name": result[0]["file_name"][:59] + "file_name": objectName[59:] } } ] From e632a33514dd3fe74e195f13938ce515a6a24a6b Mon Sep 17 00:00:00 2001 From: dert1129 Date: Wed, 31 Jul 2024 13:26:57 -0400 Subject: [PATCH 13/15] implement measurement protocol for google analytics --- .dockerignore | 1 + Dockerfile | 5 +++-- app.py | 24 ++++++++++-------------- requirements.txt | 1 + 4 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9abb766 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e1ef8b9..f0c7f55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ FROM python:3.7.12-alpine3.14 WORKDIR /code -ENV FLASK_APP app.py -ENV FLASK_RUN_HOST 0.0.0.0 +ENV FLASK_APP=app.py +ENV FLASK_RUN_HOST=0.0.0.0 RUN apk add --no-cache gcc musl-dev linux-headers tzdata libffi-dev libc-dev +COPY app.py app.py COPY requirements.txt requirements.txt RUN pip3 install -r requirements.txt RUN pip3 install -U flask-cors diff --git a/app.py b/app.py index 44be0a1..8f6a948 100644 --- a/app.py +++ b/app.py @@ -33,7 +33,7 @@ class MYSQLConnection: def __init__(self): - logger.debug( + logger.info( "Start: MYSQLConnection().__init__(), trying to load environment variables in docker" ) self.host = None @@ -99,11 +99,8 @@ def get_file_info_by_file_name(file_name): "SELECT * FROM repo_file_v WHERE file_name = %s", (file_name,), ) -# /api/v1/file/download/46669edb-0e3e-4470-8834-e9ca30a50bc3/ -# 4a45bcae-3203-4319-b455-b28965b1587a_20240604_OpenAccessClinicalData.csv' - -@app.route('/v1/file/download//', methods=['GET']) +@app.route('/v1/file/download//', methods=['POST', 'GET']) def downloadFile(packageId, objectName): result = get_file_info_by_file_name(objectName) if result[0]["access"] == "open": @@ -113,15 +110,14 @@ def downloadFile(packageId, objectName): payload = { "client_id": "XXXXXXXXXX.YYYYYYYYYY", "events": [ - { - "name": "file_download", - "params": { - "file_name": objectName[59:] - } + { + "name": "file_download", + "params": { + "file_name": objectName } - ] + }] } - requests.post(url, json=payload, headers={'content-type': 'application/json'}, verify=True) + requests.post(url, json=payload, headers={"Content-Type": "application/json"}) return send_file(object, as_attachment=True, download_name=objectName) except S3Error as err: logger.error(err) @@ -138,7 +134,7 @@ def downloadDerivedFileS3PS(packageId, objectName): Params={'Bucket': s3Bucket, 'Key': objectNameFull}, ExpiresIn=3600) except botocore.exceptions.ClientError as error: - logger.error(err) + logger.error(error) except botocore.exceptions.ParamValidationError as error: - logger.error(err) + logger.error(error) diff --git a/requirements.txt b/requirements.txt index 9950f47..c4796ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ minio boto3 flask-cors mysql-connector-python +requests \ No newline at end of file From 2e423e5adcb44f57e35b419ced1b29d50c82eaba Mon Sep 17 00:00:00 2001 From: dert1129 Date: Fri, 2 Aug 2024 13:22:06 -0400 Subject: [PATCH 14/15] add gunicorn --- Dockerfile | 2 +- requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f0c7f55..d3b7c18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,4 @@ COPY app.py app.py COPY requirements.txt requirements.txt RUN pip3 install -r requirements.txt RUN pip3 install -U flask-cors -CMD ["flask", "run"] \ No newline at end of file +CMD ["gunicorn", "-b", ":5000", "app:app"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c4796ff..1ed6d31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ minio boto3 flask-cors mysql-connector-python -requests \ No newline at end of file +requests +gunicorn \ No newline at end of file From 33466476245336822899f4c60d7499810fc10ee9 Mon Sep 17 00:00:00 2001 From: dert1129 Date: Mon, 5 Aug 2024 11:09:00 -0400 Subject: [PATCH 15/15] fix event and params --- app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 8f6a948..116f1b5 100644 --- a/app.py +++ b/app.py @@ -111,9 +111,11 @@ def downloadFile(packageId, objectName): "client_id": "XXXXXXXXXX.YYYYYYYYYY", "events": [ { - "name": "file_download", + "name": "AtlasRepositoryDownload", "params": { - "file_name": objectName + "event_category": "Repository", + "event_action": "Download", + "label": objectName } }] }