Merge pull request #1 from CannonLock/layout-updates

Layout updates

.github/workflows/codeql-analysis.yml

@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main, 'v[0-9].[0-9].*' ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [ main, 'v[0-9].[0-9].*' ]
   schedule:
     - cron: '36 11 * * 6'
 

.github/workflows/golangci-lint.yml → .github/workflows/pre-commit-linter.yml

@@ -1,19 +1,23 @@
-name: golangci-lint
+name: Linter
 on:
   push:
     tags:
-      - '*'
+      - "*"
     branches:
       - master
       - main
   pull_request:
 jobs:
-  golangci:
-    name: lint
+  linter:
+    name: linter
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      # We embed the contents of src/out/* into the resulting binaries
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+          cache: false
+      # We embed the contents of web_ui/frontend/out/* into the resulting binaries
       # That particular directory should contain outputs generated by the
       # npm build.  However, to keep the runtime of the linter as fast as possible,
       # instead of running `npm` here, we simply create a dummy empty file.
@@ -23,9 +27,18 @@ jobs:
       - name: Generate placeholder files
         id: generate-placeholder
         run: |
-          mkdir -p origin_ui/src/out
-          touch origin_ui/src/out/placeholder
-      - name: golangci-lint
+          go generate ./...
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      # We still run this so that we can get the nice hint of gofmt issues inline
+      - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: latest
+
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]

.github/workflows/publish-container.yml

@@ -1,10 +1,12 @@
-
 name: Release, Build, and Push
 
 on:
+  pull_request:
   push:
     tags:
-      - v[0-9]+.[0-9]+.[0-9]+
+      # only build and publish container on v7.0.0 and up
+      - v[7-9]\.[0-9]+\.[0-9]+ # match v7.x.x to v9.x.x
+      - v[1-9][0-9]+\.[0-9]+\.[0-9]+ # match any version higher
     branches:
       - main
   repository_dispatch:
@@ -75,20 +77,23 @@ jobs:
           IFS=,
           echo "::set-output name=taglist::${tag_list[*]}"
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Log in to OSG Harbor
         uses: docker/login-action@v2
+        if: github.event_name != 'pull_request'
         with:
           registry: hub.opensciencegrid.org
           username: ${{ secrets.PELICAN_HARBOR_ROBOT_USER }}
           password: ${{ secrets.PELICAN_HARBOR_ROBOT_PASSWORD }}
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
       - name: Build and push Docker images
         uses: docker/build-push-action@v4
         with:
           context: .
           file: ./images/Dockerfile
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}
           tags: "${{ steps.generate-tag-list.outputs.taglist }}"
+          build-args: |
+            IS_PR_BUILD=${{ github.event_name == 'pull_request' }}

.github/workflows/publish-dev-container.yml

@@ -0,0 +1,52 @@
+name: Release, Build, and Push Dev Image
+
+on:
+  pull_request:
+  push:
+    tags:
+      # only build and publish container on v7.0.0 and up
+      - v[7-9]\.[0-9]+\.[0-9]+-**
+      - v[1-9][0-9]+\.[0-9]+\.[0-9]+-**
+    branches:
+      - main
+
+jobs:
+  build-dev-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: hub.opensciencegrid.org/pelican_platform/pelican-dev
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest-itb
+            type=ref,enable=true,prefix=itb-,suffix=-{{date 'YYYYMMDDHHmmss'}},event=tag
+            type=raw,value=sha-{{sha}}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to OSG Harbor
+        uses: docker/login-action@v2
+        if: github.event_name != 'pull_request'
+        with:
+          registry: hub.opensciencegrid.org
+          username: ${{ secrets.PELICAN_HARBOR_ROBOT_USER }}
+          password: ${{ secrets.PELICAN_HARBOR_ROBOT_PASSWORD }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/arm64,linux/amd64
+          file: ./images/dev.Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/release.yml

@@ -4,7 +4,9 @@ name: goreleaser
 on:
   push:
     tags:
-      - '*'
+      # only run release on v7.0.0 and up
+      - v[7-9]\.[0-9]+\.[0-9]+
+      - v[1-9][0-9]+\.[0-9]+\.[0-9]+
 
 permissions:
   contents: write
@@ -13,18 +15,30 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      -
-        name: Set up Go
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Update npm version
+        run: |
+          cd web_ui/frontend
+
+          # Get the current tag and set the package.json version to it
+          npm version ${GITHUB_REF_NAME:1}
+          
+          # Add some verbosity 
+          echo "NPM version is now $(npm version)"
+      - name: Build the website
+        run: |
+          make web-build
+      - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20
-      -
-        name: Run GoReleaser
+          go-version: "1.20"
+      - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:
           # either 'goreleaser' (default) or 'goreleaser-pro'
@@ -35,4 +49,3 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' distribution
           # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
-

.github/workflows/test.yml

@@ -1,34 +1,144 @@
 on: [push, pull_request]
 name: Test
+permissions:
+  pull-requests: write
 jobs:
   test:
     strategy:
       matrix:
         go-version: [1.20.x]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+      # Do fetch depth 0 here because otherwise goreleaser might not work properly:
+      # https://goreleaser.com/ci/actions/?h=tag#workflow
+        fetch-depth: 0
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20
+    - name: Cache Next.js
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.npm
+          ${{ github.workspace }}/.next/cache
+        # Generate a new cache whenever packages or source files change.
+        key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx', '!**/node_modules/**') }}
+        # If source files changed but packages didn't, rebuild from a prior cache.
+        restore-keys: |
+          ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
     - name: Install Go
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
+    - name: Install Mac OS X Dependencies
+      run: ./github_scripts/osx_install.sh
+      if: runner.os == 'macOS'
+    - name: Test OS X
+      if: runner.os == 'macOS'
+      run: |
+        make web-build
+        # Explicitly set XRootD to IPv4-only on Mac OS X to avoid a subtle config bug.
+        export PELICAN_XROOTD_IPV4ONLY=true
+        go test -v -coverpkg=./... -coverprofile=coverage.out -covermode=count ./...
+    - name: Test
+      if: runner.os != 'macOS'
+      run: |
+        make web-build
+        go test ./...
+    - name: Run GoReleaser for Non-Ubuntu
+      uses: goreleaser/goreleaser-action@v4
+      with:
+        # either 'goreleaser' (default) or 'goreleaser-pro'
+        distribution: goreleaser
+        version: latest
+        args: build --single-target --clean --snapshot
+  test-ubuntu:
+    runs-on: ubuntu-latest
+    container:
+      image: hub.opensciencegrid.org/pelican_platform/pelican-dev:latest-itb
+    steps:
     - name: Checkout code
       uses: actions/checkout@v3
+      with:
+        # See above for why fetch depth is 0 here
+        fetch-depth: 0
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 20
+      # Fetch the tags is essential so that goreleaser can build the correct version. Workaround found here:
+      # https://github.com/actions/checkout/issues/290
+    - name: Fetch tags
+      run: |
+        git config --global --add safe.directory /__w/pelican/pelican
+        git fetch --force --tags
+    - name: Cache Next.js
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.npm
+          ${{ github.workspace }}/.next/cache
+        # Generate a new cache whenever packages or source files change.
+        key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx') }}
+        # If source files changed but packages didn't, rebuild from a prior cache.
+        restore-keys: |
+          ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-
     - name: Test
       run: |
         make web-build
-        go test ./...
-    - name: Run GoReleaser
+        go test -coverpkg=./... -coverprofile=coverage.out -covermode=count ./...
+    - name: Get total code coverage
+      if: github.event_name == 'pull_request'
+      id: cc
+      run: |
+        set -x
+        cc_total=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'`
+        echo "cc_total=$cc_total" >> $GITHUB_OUTPUT
+    - name: Restore base test coverage
+      id: base-coverage
+      if: github.event.pull_request.base.sha != ''
+      uses: actions/cache@v3
+      with:
+        path: |
+          unit-base.txt
+        # Use base sha for PR or new commit hash for master/main push in test result key.
+        key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
+    - name: Run test for base code
+      if: steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != ''
+      run: |
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
+        git fetch origin main ${{ github.event.pull_request.base.sha }}
+        HEAD=$(git rev-parse HEAD)
+        git reset --hard ${{ github.event.pull_request.base.sha }}
+        make web-build
+        go generate ./...
+        go test -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./...
+        go tool cover -func=base_coverage.out > unit-base.txt
+        git reset --hard $HEAD
+    - name: Get base code coverage value
+      if: github.event_name == 'pull_request'
+      id: cc_b
+      run: |
+        set -x
+        cc_base_total=`grep total ./unit-base.txt | grep -Eo '[0-9]+\.[0-9]+'`
+        echo "cc_base_total=$cc_base_total" >> $GITHUB_OUTPUT
+    - name: Add coverage information to action summary
+      if: github.event_name == 'pull_request'
+      run: echo 'Code coverage ' ${{steps.cc.outputs.cc_total}}'% Prev ' ${{steps.cc_b.outputs.cc_base_total}}'%' >> $GITHUB_STEP_SUMMARY
+    - name: Run GoReleaser for Ubuntu
       uses: goreleaser/goreleaser-action@v4
       with:
         # either 'goreleaser' (default) or 'goreleaser-pro'
         distribution: goreleaser
         version: latest
-        args: build --rm-dist --snapshot
+        args: --clean --snapshot
     - name: Copy files (Ubuntu)
-      if: matrix.os == 'ubuntu-latest'
       run: |
         cp dist/pelican_linux_amd64_v1/pelican ./
     - name: Run Integration Tests
-      if: matrix.os == 'ubuntu-latest'
-      run: ./tests/citests.sh
+      run: ./github_scripts/citests.sh
+    - name: Run End-to-End Tests
+      run: ./github_scripts/get_put_test.sh

.gitignore

@@ -1,2 +1,4 @@
 dist/
 README.dev.md
+docs/parameters.json
+local

.golangci.yaml

@@ -2,7 +2,7 @@
 run:
   # Pelican has gotten large enough that the GitHub Action sometimes
   # times out on a cold cache
-  timeout: 2m
+  timeout: 3m
 
 linters:
   enable: