Download GIN Data for Pipeline Tests

.github/workflows/daily_tests.yml

@@ -0,0 +1,37 @@
+name: Daily Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 14 * * *" # Daily at 10am EST
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  DevTests:
+    uses: ./.github/workflows/testing_dev.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  LiveServices:
+    uses: ./.github/workflows/testing_dev_with_live_services.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      DANDI_STAGING_API_KEY: ${{ secrets.DANDI_STAGING_API_KEY }}
+
+  BuildTests:
+    uses: ./.github/workflows/testing_flask_build_and_dist.yml
+
+  ExampleDataCache:
+    uses: ./.github/workflows/example_data_cache.yml
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}
+
+  ExampleDataTests:
+    needs: ExampleDataCache
+    uses: ./.github/workflows/testing_pipelines.yml

.github/workflows/deploy_tests_on_pull_request.yml

@@ -0,0 +1,35 @@
+name: Deploy
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  DevTests:
+    uses: ./.github/workflows/testing_dev.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  LiveServices:
+    uses: ./.github/workflows/testing_dev_with_live_services.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      DANDI_STAGING_API_KEY: ${{ secrets.DANDI_STAGING_API_KEY }}
+
+  BuildTests:
+    uses: ./.github/workflows/testing_flask_build_and_dist.yml
+
+  ExampleDataCache:
+    uses: ./.github/workflows/example_data_cache.yml
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }}
+
+  ExampleDataTests:
+    needs: ExampleDataCache
+    uses: ./.github/workflows/testing_pipelines.yml

.github/workflows/example_data_cache.yml

@@ -0,0 +1,74 @@
+name: Example data cache
+on:
+  workflow_call:
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      S3_GIN_BUCKET:
+        required: true
+
+jobs:
+
+  run:
+    # Will read on PR dashboard as 'Deploy / ExampleDataCache / {os}'
+    # Action dashboard identified by 'Deploy'
+    # Requirement settings identified as 'ExampleDataCache / {os}'
+    name: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+        os: [ubuntu-latest, macos-latest, macos-13] #, windows-latest]
+
+    steps:
+
+      - name: Get ephy_testing_data current head hash
+        id: ephys
+        run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)"
+      - name: Get cached ephys example data - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }}
+        uses: actions/cache@v4
+        id: cache-ephys-datasets
+        with:
+          path: ./ephy_testing_data
+          key: ephys-datasets-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }}
+      - name: Get ophys_testing_data current head hash
+        id: ophys
+        run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)"
+      - name: Get cached ophys example data - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
+        uses: actions/cache@v4
+        id: cache-ophys-datasets
+        with:
+          path: ./ophys_testing_data
+          key: ophys-datasets-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
+      - name: Get behavior_testing_data current head hash
+        id: behavior
+        run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)"
+      - name: Get cached behavior example data - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }}
+        uses: actions/cache@v4
+        id: cache-behavior-datasets
+        with:
+          path: ./behavior_testing_data
+          key: behavior-datasets-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }}
+
+      - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true' || steps.cache-ophys-datasets.outputs.cache-hit != 'true' || steps.cache-behavior-datasets.outputs.cache-hit != 'true'
+        name: Install and configure AWS CLI
+        run: |
+          pip install awscli
+          aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - if: steps.cache-ephys-datasets.outputs.cache-hit != 'true'
+        name: Download ephys dataset from S3
+        run: |
+          aws s3 cp --region=us-east-2 ${{ secrets.S3_GIN_BUCKET }}/ephy_testing_data ./ephy_testing_data --recursive
+      - if: steps.cache-ophys-datasets.outputs.cache-hit != 'true'
+        name: Download ophys dataset from S3
+        run: |
+          aws s3 cp --region=us-east-2 ${{ secrets.S3_GIN_BUCKET }}/ophys_testing_data ./ophys_testing_data --recursive
+      - if: steps.cache-behavior-datasets.outputs.cache-hit != 'true'
+        name: Download behavior dataset from S3
+        run: |
+          aws s3 cp --region=us-east-2 ${{ secrets.S3_GIN_BUCKET }}/behavior_testing_data ./behavior_testing_data --recursive

.github/workflows/testing.yml → .github/workflows/testing_dev.yml

@@ -1,28 +1,24 @@
-name: Dev Tests
+name: Dev tests
 on:
-  schedule:
-    - cron: "0 16 * * *"  # Daily at noon EST
-  pull_request:
-
-concurrency:  # Cancel previous workflows on the same pull request
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  CACHE_NUMBER: 2  # increase to reset cache manually
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: true
 
 jobs:
-  testing:
-    name: ${{ matrix.os }}  # Will read on the dashboard as 'Dev Tests / {os}'
+
+  run:
+    # Will read on PR dashboard as 'Deploy / DevTests / {os}'
+    # Action dashboard identified by 'Deploy'
+    # Requirement settings identified as 'DevTests / {os}'
+    name: ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash -l {0}
-
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
         include:
           - os: ubuntu-latest
             label: environments/environment-Linux.yml
@@ -33,9 +29,8 @@ jobs:
           - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
 
-          - os: windows-latest
-            label: environments/environment-Windows.yml
-
+#          - os: windows-latest
+#            label: environments/environment-Windows.yml
 
     steps:
       - uses: actions/checkout@v4
@@ -89,7 +84,7 @@ jobs:
           name: test-screenshots-${{ matrix.os }}
           path: docs/assets/tutorials
           retention-days: 1
-
+          overwrite: true
 
 
       - name: Upload coverage reports to Codecov

.github/workflows/testing-live-services.yml → .github/workflows/testing_dev_with_live_services.yml

@@ -1,28 +1,26 @@
-name: Dev Tests (Live)
+name: Dev tests with live services
 on:
-  schedule:
-    - cron: "0 16 * * *"  # Daily at noon EST
-  pull_request:
-
-concurrency:  # Cancel previous workflows on the same pull request
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  CACHE_NUMBER: 2  # increase to reset cache manually
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: true
+      DANDI_STAGING_API_KEY:
+        required: true
 
 jobs:
-  testing:
-    name: Services on ${{ matrix.os }}  # Will read on the dashboard as 'Dev Tests (Live) / Services on {os}'
+
+  run:
+    # Will read on PR dashboard as 'Deploy / LiveServices / {os}'
+    # Action dashboard identified by 'Deploy'
+    # Requirement settings identified as 'LiveServices / {os}'
+    name: ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash -l {0}
-
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
         include:
           - os: ubuntu-latest
             label: environments/environment-Linux.yml
@@ -33,9 +31,8 @@ jobs:
           - os: macos-13  # Mac x64 runner
             label: environments/environment-MAC-intel.yml
 
-          - os: windows-latest
-            label: environments/environment-Windows.yml
-
+#          - os: windows-latest
+#            label: environments/environment-Windows.yml
 
     steps:
       - uses: actions/checkout@v4
@@ -87,7 +84,6 @@ jobs:
         name: Run tests with xvfb
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:app
 
-
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
         env: