add megalinter

.github/workflows/build.yaml

@@ -14,7 +14,7 @@ jobs:
 
     strategy:
       matrix:
-        release: [edge]  # TODO 3.12, 3.11, etc
+        release: [edge] # TODO 3.12, 3.11, etc
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/megalinter.yml

@@ -0,0 +1,95 @@
+---
+# MegaLinter GitHub Action configuration file
+# More info at https://megalinter.io
+name: MegaLinter
+
+on:
+  # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main
+  push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
+  pull_request:
+    branches: [master, main]
+
+env: # Comment env block if you don't want to apply fixes
+  # Apply linter fixes configuration
+  APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
+  APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all)
+  APPLY_FIXES_MODE: commit # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request)
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  megalinter:
+    name: MegaLinter
+    runs-on: ubuntu-latest
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push, comment issues & post new PR
+      # Remove the ones you do not need
+      contents: write
+      issues: write
+      pull-requests: write
+    steps:
+      # Git Checkout
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
+          # fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances
+
+      # MegaLinter
+      - name: MegaLinter
+        id: ml
+        # You can override MegaLinter flavor used to have faster performances
+        # More info at https://megalinter.io/flavors/
+        uses: oxsecurity/megalinter@v7
+        env:
+          # All available variables are described in documentation
+          # https://megalinter.io/configuration/
+          VALIDATE_ALL_CODEBASE: true # ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Validates all source when push on main, else just the git diff with main. Override with true if you always want to lint all sources
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY
+          DISABLE: COPYPASTE,SPELL # Uncomment to disable copy-paste and spell checks
+          DISABLE_ERRORS_LINTERS: REPOSITORY_CHECKOV,ACTION_ACTIONLINT,REPOSITORY_KICS,REPOSITORY_DEVSKIM,PYTHON_PYLINT,PYTHON_PYRIGHT
+          BASH_SHELLCHECK_ARGUMENTS: "-s bash" # silence complaints about posix sh
+          DOCKERFILE_HADOLINT_ARGUMENTS: "--ignore DL3059" # DL3059 = Multiple consecutive `RUN` instructions - it's intentional and desired
+          PYTHON_BANDIT_ARGUMENTS: "--skip B113" # B113 = request_without_timeout
+
+      # Upload MegaLinter artifacts
+      - name: Archive production artifacts
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: MegaLinter reports
+          path: |
+            megalinter-reports
+            mega-linter.log
+
+      # Create pull request if applicable (for now works only on PR from same repository, not from forks)
+      - name: Create Pull Request with applied fixes
+        id: cpr
+        if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'pull_request' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
+          commit-message: "[MegaLinter] Apply linters automatic fixes"
+          title: "[MegaLinter] Apply linters automatic fixes"
+          labels: bot
+      - name: Create PR output
+        if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'pull_request' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
+
+      # Push new commit if applicable (for now works only on PR from same repository, not from forks)
+      - name: Prepare commit
+        if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'commit' && github.ref != 'refs/heads/main' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
+        run: sudo chown -Rc $UID .git/
+      - name: Commit and push applied linter fixes
+        if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'commit' && github.ref != 'refs/heads/main' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix')
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          branch: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref }}
+          commit_message: "[MegaLinter] Apply linters fixes"
+          commit_user_name: iggy
+          commit_user_email: [email protected]

.gitignore

@@ -10,3 +10,13 @@ src/
 tmp/
 packages/
 pkg/
+.bash_history
+**/__pycache__
+
+# something was making git status/fish take a lot of cpu/time
+.cache
+.cargo
+.config
+.local
+.npm
+go

.trivyignore

@@ -0,0 +1,2 @@
+DS025
+DS026

Dockerfile

@@ -1,6 +1,6 @@
 # Build a docker image that's ready to build packages
 
-# some dependencies are only in edge, TODO build for stable releases later too
+# some dependencies are only in edge, should build for stable releases later too
 FROM alpine:edge
 
 LABEL org.opencontainers.image.source "https://github.com/atlascloud/aports"
@@ -10,7 +10,7 @@ LABEL org.opencontainers.image.source "https://github.com/atlascloud/aports"
 # This is a container for building other software, it doesn't need pinned packages/etc
 # Also, the edge docker image gets pretty dated at times when ncopa is prep'ing a new release
 # hadolint ignore=DL3017,DL3018,DL3019
-RUN apk add bash alpine-conf alpine-sdk ccache cmake coreutils m4 sudo
+RUN apk add bash alpine-conf alpine-sdk ccache cmake coreutils m4 sudo fish
 # hadolint ignore=DL3017,DL3018,DL3019
 RUN apk upgrade
 

Makefile

@@ -5,7 +5,7 @@ baseimage:
 	docker build --pull --tag ghcr.io/atlascloud/aports-builder:edge .
 	docker push ghcr.io/atlascloud/aports-builder:edge
 
-## Build the actual ceph packages
+## Build the actual packages
 build:
 	# docker pull atlascloud/ceph-builder
 	docker run --rm \

Taskfile.yml

@@ -7,65 +7,69 @@ dotenv: [".env"]
 #   FOO: bar
 
 # env
+vars:
+  CNTRCTL:
+    # sh: echo nerdctl
+    sh: which docker || which nerdctl
 
 tasks:
   docker:build:
     desc: build docker builder image
     cmds:
-      - docker build --pull --tag ghcr.io/atlascloud/aports-builder:edge .
+      - |
+        {{ .CNTRCTL }} pull alpine:edge
+        {{ .CNTRCTL }} build --tag ghcr.io/atlascloud/aports-builder:edge .
 
   docker:push:
     desc: push builder image to github packages
     cmds:
-      - docker push ghcr.io/atlascloud/aports-builder:edge
+      - |
+        {{- .CNTRCTL }} push ghcr.io/atlascloud/aports-builder:edge
 
   docker:run:
     desc: run the builder image
     cmds:
       - mkdir -p apkcache/ distfiles/ packages/ .ccache/ .abuild/
-      - docker run -it --rm
-        --env-file .env
-        -v ${PWD}:/home/build
-        -v ${PWD}/apkcache:/etc/apk/cache
-        -v ${PWD}/distfiles:/var/cache/distfiles
+      - |
+        {{- .CNTRCTL }} run -it --rm \
+        --env-file .env \
+        -v ${PWD}:/home/build \
+        -v ${PWD}/apkcache:/etc/apk/cache \
+        -v ${PWD}/distfiles:/var/cache/distfiles \
         --name aports-builder ghcr.io/atlascloud/aports-builder:edge
 
   docker:runshell:
     desc: run a shell in the builder image (useful for debugging)
     cmds:
       - mkdir -p apkcache/ distfiles/ packages/ .ccache/ .abuild/
-      - docker run -it --rm
-        --env-file .env
-        -v ${PWD}:/home/build
-        -v ${PWD}/apkcache:/etc/apk/cache
-        -v ${PWD}/distfiles:/var/cache/distfiles
-        --entrypoint /bin/sh
+      - |
+        {{- .CNTRCTL }} run -it --rm \
+        --env-file .env \
+        -v ${PWD}:/home/build \
+        -v ${PWD}/apkcache:/etc/apk/cache \
+        -v ${PWD}/distfiles:/var/cache/distfiles \
+        --entrypoint /bin/sh \
+        --network host \
         --name aports-builder ghcr.io/atlascloud/aports-builder:edge
 
   docker:checksum:
     desc: generate checksums for a package, must specify 'PKG'
+    requires:
+      vars: [PKG]
     cmds:
-      - docker run --rm
-        -v ${PWD}:/aports
-        -v ${PWD}/apkcache:/etc/apk/cache
-        -v ${PWD}/distfiles:/var/cache/distfiles
-        --workdir /aports/{{ .PKG }}
+      - |
+        {{- .CNTRCTL }} run --rm \
+        -v ${PWD}:/aports \
+        -v ${PWD}/apkcache:/etc/apk/cache \
+        -v ${PWD}/distfiles:/var/cache/distfiles \
+        --workdir /aports/{{ .PKG }} \
         alpine:edge sh -c "apk add abuild ; abuild -F checksum"
 
-  docker:super-linter:
-    desc: run super linter - should switch to megalinter
-    cmds:
-      - docker run
-        -e RUN_LOCAL=true
-        -e ACTIONS_RUNNER_DEBUG=false
-        -e FILTER_REGEX_EXCLUDE='/tmp/lint/(.abuild|.cacche|.git|apkcache|distfiles|src|tmp)'
-        -v ${PWD}:/tmp/lint
-        github/super-linter
-
   docker:test:
     desc: run tests in docker
     cmds:
-      - docker run
+      - |
+        {{- .CNTRCTL }} run
         --workdir /src
         -v ${PWD}:/src
         alpine:edge
@@ -85,7 +89,7 @@ tasks:
             apkbuild-lint APKBUILD
             cd /src
           {{end}}
-        ignore_errors: true
+        ignore_error: true
 
   upload-package:
     desc: upload a package to packages server
@@ -94,19 +98,37 @@ tasks:
     cmds:
       - |
         {{ $SL := splitList "/" .PKG -}}
-        {{ $repo := slice $SL 2 3 | first -}}
         {{ $vers := slice $SL 1 2 | first -}}
+        {{ $repo := slice $SL 2 3 | first -}}
+        {{ $arch := slice $SL 3 4 | first -}}
         curl -v \
           -H "Authorization: Bearer $PKGS_TOKEN" \
-          -F "architecture=x86_64;type=text/plain" \
           -F "package=@{{ .PKG }}" \
-          https://packages.atlascloud.xyz/api/o/atlascloud/r/{{ $repo }}/v/{{ $vers }}
+          https://packages.atlascloud.xyz/api/atlascloud/alpine/{{ $vers }}/{{ $repo }}/{{ $arch }}/pkgs
+
+  # we used to do this automatically on the package server, but it can take a very long time and it was doing it
+  # on every package upload
+  sign-package-index:
+    desc: instruct the packages server to generate and sign the index with the packages that we've uploaded so far
+    # vars:
+    #   PKG: should be set on command line - we aren't doing anything with the pkg, just extracting info from the path
+    cmds:
+      - |
+        {{ $SL := splitList "/" .PKG -}}
+        {{ $vers := slice $SL 1 2 | first -}}
+        {{ $repo := slice $SL 2 3 | first -}}
+        {{ $arch := slice $SL 3 4 | first -}}
+        {{ $arch := slice $SL 3 4 | first -}}
+        curl \
+          -X POST \
+          -H "Authorization: Bearer $PKGS_TOKEN" \
+          https://packages.atlascloud.xyz/api/atlascloud/alpine/{{ $vers }}/{{ $repo }}/{{ $arch }}/index
 
   check-pkg-version:
     desc: run checker scripts
     cmds:
       - |
-        pkg={{ .PKG | splitList "/" | last }}
+        pkg={{ .PKG | dir | base }}
         echo "checking $pkg"
         [ -f "./checkers/$pkg" ] || exit 0 # if we don't have a checker, skip
 
@@ -116,14 +138,12 @@ tasks:
         sed -i "s:pkgrel=.*:pkgrel=0:" main/$pkg/APKBUILD
 
   check-pkg-versions:
+    desc: run checker scripts for every APKBUILD
     vars:
       ABS:
         sh: find . -name APKBUILD
     cmds:
-      # loop over all of the apkbuild files and run check_pkg_version
-      # for that pkg name
-      # {{range $i, $APKBUILD := .ABS | splitLines -}}
-      # - task: check-pkg-version
-      #   vars:
-      #     PKG: "{{$APKBUILD | dir | base}}"
-      # {{end}}
+      - for: { var: ABS }
+        task: check-pkg-version
+        vars:
+          PKG: "{{ .ITEM }}"