Add Lithops GCP to CI matrix

cubed-dev · Mar 11, 2024 · 9af9c97 · 9af9c97
1 parent a0d0b23
commit 9af9c97
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 4 deletions.
diff --git a/.github/workflows/.lithops_config_gcp.template b/.github/workflows/.lithops_config_gcp.template
@@ -0,0 +1,12 @@
+lithops:
+  backend: gcp_functions 
+  storage: gcp_storage
+  log_level: WARN
+
+gcp:
+  region: europe-west2
+  credentials_path:
+
+gcp_storage:
+  region: europe-west2
+  storage_bucket: tom-lithops-data
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -25,29 +25,51 @@ jobs:
         python-version: ["3.11"]
         extra-env: [""]
         cubed-config: [tests/configs/local_single-threaded.yaml]
+        lithops-config: [""]
         name_prefix: [tests]
         include:
           - os: "ubuntu-latest"
             python-version: "3.11"
             extra-env: ci/environment-lithops-aws.yml
             cubed-config: tests/configs/lithops_aws.yaml
+            lithops-config: .github/workflows/.lithops_config_aws
+          - os: "ubuntu-latest"
+            python-version: "3.11"
+            extra-env: ci/environment-lithops-gcp.yml
+            cubed-config: tests/configs/lithops_gcp.yaml
+            lithops-config: .github/workflows/.lithops_config_gcp
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Copy lithops configuration template
+      - name: Copy lithops configuration templates
         run: |
           cp $GITHUB_WORKSPACE/.github/workflows/.lithops_config_aws.template $GITHUB_WORKSPACE/.github/workflows/.lithops_config_aws
+          cp $GITHUB_WORKSPACE/.github/workflows/.lithops_config_gcp.template $GITHUB_WORKSPACE/.github/workflows/.lithops_config_gcp
+
+      - name: Google auth
+        id: 'auth'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+          create_credentials_file: true
 
-      - name: Configure lithops
+      - name: Configure lithops AWS
         uses: microsoft/variable-substitution@v1
         with:
           files: ${{ github.workspace }}/.github/workflows/.lithops_config_aws
         env:
           aws.access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws.secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+      - name: Configure lithops GCP
+        uses: microsoft/variable-substitution@v1
+        with:
+          files: ${{ github.workspace }}/.github/workflows/.lithops_config_gcp
+        env:
+          gcp.credentials_path: ${{ steps.auth.outputs.credentials_file_path }}
+
       - name: Set up environment
         uses: conda-incubator/setup-miniconda@v2
         with:
@@ -79,7 +101,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           CUBED_CONFIG: ${{ matrix.cubed-config }}
           DB_NAME: ${{ matrix.name_prefix }}-${{ matrix.os }}-py${{ matrix.python_version }}.db
-          LITHOPS_CONFIG_FILE: ${{ github.workspace }}/.github/workflows/.lithops_config_aws
+          LITHOPS_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.lithops-config }}
         run: |
           pytest --benchmark --basetemp=pytest-temp
 

diff --git a/tests/configs/lithops_gcp.yaml b/tests/configs/lithops_gcp.yaml
@@ -1,5 +1,5 @@
 spec:
-  work_dir: "gs://cubed-$USER-temp"
+  work_dir: "gs://cubed-tom-temp"
   allowed_mem: "2GB"
   executor_name: "lithops"
   executor_options: