Merge pull request #488 from ddps-lab/gcp-collector

Refactoring GCP Collector
ddps-lab · Jan 20, 2025 · 111d9a0 · 111d9a0
2 parents 1afe275 + edf2d12
commit 111d9a0
Show file tree

Hide file tree

Showing 4 changed files with 482 additions and 331 deletions.
diff --git a/.github/workflows/gcp-lambda-sync.yml b/.github/workflows/gcp-lambda-sync.yml
@@ -2,16 +2,16 @@ name: deploy gcp files to lambda
 on:
   push:
     branches:
-      - 'main'
+      - "main"
     paths:
-      - 'collector/spot-dataset/gcp/lambda/**'
-      - 'utility/slack_msg_sender.py'
-      - 'const_config.py'
+      - "collector/spot-dataset/gcp/lambda/**"
+      - "utility/slack_msg_sender.py"
+      - "const_config.py"
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.SPOTRANK_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.SPOTRANK_SECRET_ACCESS_KEY }}
-  AWS_DEFAULT_REGION: 'us-west-2'
+  AWS_DEFAULT_REGION: "us-west-2"
 
 jobs:
   deploy_source:
@@ -20,10 +20,13 @@ jobs:
     steps:
       - name: checkout source code
         uses: actions/checkout@v1
+      - name: Create JSON file from environment variable
+        run: echo "${{ secrets.GCP_KEY_JSON }}" > ${{ secrets.GCP_JSON_FILENAME }}
       - name: Zip lambda_function code
         run: |
           zip -r gcp_lambda.zip ./utility/slack_msg_sender.py
           zip -j gcp_lambda.zip ./collector/spot-dataset/gcp/lambda/* ./const_config.py
+          zip -j gcp_lambda.zip ${{ secrets.GCP_JSON_FILENAME }}
       - name: Deploy to lambda
         run: |
           aws lambda update-function-code --function-name gcp-collector --zip-file fileb://gcp_lambda.zip
diff --git a/collector/spot-dataset/gcp/lambda/get_metadata.py b/collector/spot-dataset/gcp/lambda/get_metadata.py