Update GitHub Actions

.github/workflows/build.yml

@@ -0,0 +1,44 @@
+name: Funnel Build and Cache
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.21
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Cache Funnel binary
+      uses: actions/cache@v3
+      with:
+        path: ./funnel
+        key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }}
+        restore-keys: |
+          ${{ runner.os }}-funnel-bin-${{ github.ref }}
+          ${{ runner.os }}-funnel-bin-
+
+    - name: Build Funnel (if cache doesn't exist)
+      run: |
+        if [ ! -f ./funnel ]; then
+          make build
+        fi
+    
+    - name: Cache Funnel binary (after build)
+      uses: actions/cache@v3
+      with:
+        path: ./funnel
+        key: ${{ runner.os }}-funnel-bin-${{ hashFiles('**/go.sum') }}-${{ github.ref }}
+
+    - name: Upload Funnel binary as artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: funnel
+        path: funnel

.github/workflows/compliance-test.yaml → .github/workflows/compliance.yaml

@@ -13,42 +13,32 @@
 # output a ssh address associated with the Github action worker. Helpful for debugging build steps and
 # and intermediary files/artifacts.
 #
+# - name: "Debug: Package dependancies for tmate (CentOS)"
+#   run: |
+#     yum install -y xz
+#     ln -s /bin/true /bin/apt-get
+#
 # - name: Setup tmate session
 #   uses: mxschmitt/action-tmate@v3
 
 name: Compliance Test
 
 on:
   push:
+  pull_request:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v5
-      with:
-        go-version: 1.21
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Build
-      run: make build
-
-    - name: Store funnel
-      uses: actions/upload-artifact@v4
-      with:
-        name: funnelBin
-        path: funnel
+    uses: ./.github/workflows/build.yml
 
   compliance:
     strategy:
       fail-fast: false
       matrix:
         version: [1.0.0, 1.1.0]
         db: ["boltdb", "mongodb"]
-        compute: ["local"]
+        compute: ["local", "kubernetes"]
+        storage: ["local", "s3"]
     needs: build
     runs-on: ubuntu-latest
     steps:
@@ -57,17 +47,29 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: funnelBin
+          name: funnel
 
       - name: Start Funnel server
         run: |
           touch config.yml
+
           if [ ${{ matrix.db }} = "mongodb" ]; then
             make start-mongodb
             cat `pwd`/tests/mongo.config.yml >> config.yml
           fi
+
+          if [ ${{ matrix.storage }} = "s3" ]; then
+            docker run -d -p 9000:9000 --name minio \
+                       -e "MINIO_ROOT_USER=minioadmin" \
+                       -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                       -v /tmp/data:/data \
+                       -v /tmp/config:/root/.minio \
+                       minio/minio server /data
+            cat `pwd`/tests/s3.config.yml >> config.yml
+          fi
+
           chmod +x funnel
           ./funnel server run --config `pwd`/config.yml &> funnel.logs &
 

.github/workflows/nextflow.yaml

@@ -2,41 +2,23 @@ name: Nextflow Test
 
 on:
   push:
+  pull_request:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v5
-      with:
-        go-version: 1.21
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Build Funnel (if cache does not exist)
-      run: make build
+    uses: ./.github/workflows/build.yml
 
-    - name: Store Funnel
-      uses: actions/upload-artifact@v4
-      with:
-        name: funnelBin
-        path: funnel
-
   nextflow:
-    runs-on: ubuntu-latest
     needs: build
+    runs-on: ubuntu-latest
     steps:
     - name: Download Funnel
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
-        name: funnelBin
-        path: funnel
+        name: funnel
 
     - name: Start Funnel
       run: |
-        cd funnel/
         chmod +x ./funnel
         ./funnel server --LocalStorage.AllowedDirs $HOME run &
 
@@ -61,4 +43,3 @@ jobs:
     - name: Run nf-canary tests
       run: |
         ./nextflow/nextflow run nf-canary/main.nf
-

.github/workflows/s3.yaml

@@ -0,0 +1,52 @@
+# Credit: rhnvrm
+# Adapted from: https://rohanverma.net/blog/2021/02/09/minio-github-actions/
+
+name: S3 Integration Test
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+
+  s3Test:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup minio
+        run: |
+          docker run -d -p 9000:9000 --name minio \
+                     -e "MINIO_ROOT_USER=minioadmin" \
+                     -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                     -v /tmp/data:/data \
+                     -v /tmp/config:/root/.minio \
+                     minio/minio server /data
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: funnelBin
+
+      - name: Start Funnel server
+        run: |
+          cat <<EOF > config.yml
+            LocalStorage:
+              Disabled: true
+            AmazonS3:
+              Disabled: true
+            GoogleStorage:
+              Disabled: true
+            HTTPStorage:
+              Disabled: true
+            FTPStorage:
+              Disabled: true
+            GenericS3:
+              - Disabled: false
+                Endpoint: "localhost:9000"
+                Key: "minioadmin"
+                Secret: "minioadmin"
+          EOF
+          chmod +x funnel
+          ./funnel server run --config `pwd`/config.yml &> funnel.logs &
+          ./funnel task run examples/s3-test.yml