Update GitHub Actions workflows

ohsu-comp-bio · Sep 30, 2024 · e7d8cbf · e7d8cbf
1 parent 1a1ae94
commit e7d8cbf
Show file tree

Hide file tree

Showing 6 changed files with 222 additions and 62 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,44 @@
+name: Funnel Build and Cache
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.21
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Cache Funnel binary
+      uses: actions/cache@v3
+      with:
+        path: ./funnel
+        key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }}
+        restore-keys: |
+          ${{ runner.os }}-funnel-bin-${{ github.ref }}
+          ${{ runner.os }}-funnel-bin-
+
+    - name: Build Funnel (if cache doesn't exist)
+      run: |
+        if [ ! -f ./funnel ]; then
+          make build
+        fi
+    
+    - name: Cache Funnel binary (after build)
+      uses: actions/cache@v3
+      with:
+        path: ./funnel
+        key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }}
+
+    - name: Upload Funnel binary as artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: funnel
+        path: funnel
diff --git a/.github/workflows/compliance.yaml b/.github/workflows/compliance.yaml
@@ -0,0 +1,85 @@
+# Workflow for running the TES compliance suite against Funnel
+#
+# This includes the following steps:
+#  1. Build Funnel and store the resulting binary artifact
+#  2. Install tes-compliance-suite and run against every version of TES simultaneously
+#  3. start-report-deployment: Send a dispatch to the funnel-compliance repository to generate and publish
+#     the tes-compliance-suite report to https://ohsu-comp-bio.github.io/funnel-compliance/
+#
+# Optionally debug via SSH
+# Ref: https://fleetdm.com/engineering/tips-for-github-actions-usability
+#
+# To use this step uncomment and place anywhere in the build steps. The build will pause on this step and
+# output a ssh address associated with the Github action worker. Helpful for debugging build steps and
+# and intermediary files/artifacts.
+#
+# - name: "Debug: Package dependancies for tmate (CentOS)"
+#   run: |
+#     yum install -y xz
+#     ln -s /bin/true /bin/apt-get
+#
+# - name: Setup tmate session
+#   uses: mxschmitt/action-tmate@v3
+
+name: Compliance Test
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+
+  compliance:
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [1.0.0, 1.1.0]
+        db: ["boltdb", "mongodb"]
+        compute: ["local", "kubernetes"]
+        storage: ["local", "s3"]
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      # Required to access the 'tests/mongo.config.yml' file
+      # Perhaps uploading it as an artifact would be more efficient?
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: funnel
+
+      - name: Start Funnel server
+        run: |
+          touch config.yml
+
+          if [ ${{ matrix.db }} = "mongodb" ]; then
+            make start-mongodb
+            cat `pwd`/tests/mongo.config.yml >> config.yml
+          fi
+
+          if [ ${{ matrix.storage }} = "s3" ]; then
+            docker run -d -p 9000:9000 --name minio \
+                       -e "MINIO_ROOT_USER=minioadmin" \
+                       -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                       -v /tmp/data:/data \
+                       -v /tmp/config:/root/.minio \
+                       minio/minio server /data
+            cat `pwd`/tests/s3.config.yml >> config.yml
+          fi
+
+          chmod +x funnel
+          ./funnel server run --config `pwd`/config.yml &> funnel.logs &
+
+      - name: Run OpenAPI Test Runner
+        run: |
+          git clone https://github.com/elixir-cloud-aai/openapi-test-runner
+          cd openapi-test-runner
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          python setup.py install  
+          openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+
diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml
@@ -59,11 +59,6 @@ jobs:
             --baseURL "${{ steps.pages.outputs.base_url }}/" \
             --source website \
             --destination public
-
-          # Run pagefind to add search functionality
-          # Reference: https://pagefind.app/docs/
-          npx -y pagefind --site public
-
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:

diff --git a/.github/workflows/nextflow.yaml b/.github/workflows/nextflow.yaml
@@ -2,49 +2,31 @@ name: Nextflow Test
 
 on:
   push:
+  pull_request:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v5
-      with:
-        go-version: 1.21
-
-    - name: Check out code
-      uses: actions/checkout@v2
+    uses: ./.github/workflows/build.yml
 
-    - name: Build Funnel (if cache does not exist)
-      run: make build
-
-    - name: Store Funnel
-      uses: actions/upload-artifact@v4
-      with:
-        name: funnelBin
-        path: funnel
-
   nextflow:
-    runs-on: ubuntu-latest
     needs: build
+    runs-on: ubuntu-latest
     steps:
     - name: Download Funnel
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
-        name: funnelBin
-        path: funnel
+        name: funnel
 
     - name: Start Funnel
-      working-directory: 
       run: |
-        cd funnel/
         chmod +x ./funnel
         ./funnel server --LocalStorage.AllowedDirs $HOME run &
 
     - name: Install Nextflow
       run: |
-        curl -s https://get.nextflow.io | bash
-        chmod +x nextflow
+        git clone https://github.com/nextflow-io/nextflow
+        cd nextflow
+        make compile
 
     - name: Install nf-canary and GA4GH-TES plugin
       run: |

diff --git a/.github/workflows/s3.yaml b/.github/workflows/s3.yaml
@@ -0,0 +1,52 @@
+# Credit: rhnvrm
+# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/
+
+name: S3 Integration Test
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+
+  s3Test:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup minio
+        run: |
+          docker run -d -p 9000:9000 --name minio \
+                     -e "MINIO_ROOT_USER=minioadmin" \
+                     -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                     -v /tmp/data:/data \
+                     -v /tmp/config:/root/.minio \
+                     minio/minio server /data
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: funnel
+
+      - name: Start Funnel server
+        run: |
+          cat <<EOF > config.yml
+            LocalStorage:
+              Disabled: true
+            AmazonS3:
+              Disabled: true
+            GoogleStorage:
+              Disabled: true
+            HTTPStorage:
+              Disabled: true
+            FTPStorage:
+              Disabled: true
+            GenericS3:
+              - Disabled: false
+                Endpoint: "localhost:9000"
+                Key: "minioadmin"
+                Secret: "minioadmin"
+          EOF
+          chmod +x funnel
+          ./funnel server run --config `pwd`/config.yml &> funnel.logs &
+          ./funnel task run examples/s3-test.yml
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -2,6 +2,7 @@ name: Go Tests
 
 on:
   push:
+  pull_request:
 
 jobs:
   lint:
@@ -11,33 +12,31 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: 1.21
+
       - uses: actions/checkout@v3
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: latest
-          # TODO: Re-enable all linters
-          args: --timeout 3m --verbose -D unused -D errcheck -D staticcheck -D govet -D gosimple
-
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v5
-      with:
-        go-version: 1.21
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Build
-      run: make build
-
-    - name: Store funnel
-      uses: actions/upload-artifact@v4
-      with:
-        name: funnelBin
-        path: funnel
+          # Matches the "primary" golangci-lint command in the Makefile
+          args: |
+              --timeout 3m --disable-all --enable=govet --enable=gofmt --enable=goimports --enable=misspell \
+              --skip-dirs "vendor" \
+              --skip-dirs "webdash" \
+              --skip-dirs "cmd/webdash" \
+              --skip-dirs "funnel-work-dir" \
+              -e '.*bundle.go' -e ".*pb.go" -e ".*pb.gw.go" \
+              ./...
+      
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: latest
+          # Matches the "termdash" golangci-lint command in the Makefile
+          args: |
+              --timeout 3m --disable-all --enable=vet --enable=gofmt --enable=goimports --enable=misspell \
+              ./cmd/termdash/...
 
   unitTest:
     runs-on: ubuntu-latest
@@ -52,6 +51,9 @@ jobs:
     - name: Unit Tests
       run: make test-verbose
 
+  build:
+    uses: ./.github/workflows/build.yml
+
   mongoTest:
     runs-on: ubuntu-latest
     needs: build
@@ -65,9 +67,9 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Download funnel bin
-      uses: actions/download-artifact@v4.1.7
+      uses: actions/download-artifact@v4
       with:
-        name: funnelBin
+        name: funnel
 
     - name: MongoTest
       run: |
@@ -88,9 +90,9 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Download funnel bin
-      uses: actions/download-artifact@v4.1.7
+      uses: actions/download-artifact@v4
       with:
-        name: funnelBin
+        name: funnel
     - name: Badger Test
       run: |
         chmod +x funnel
@@ -108,9 +110,9 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Download funnel bin
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
-        name: funnelBin
+        name: funnel
 
     - name: Slurm Test
       run: |
@@ -129,14 +131,14 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Download funnel bin
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
-        name: funnelBin
+        name: funnel
 
     - name: S3 Test
       run: |
         chmod +x funnel
         make start-generic-s3
         sleep 10
         make test-generic-s3
-          
+