From 22e542f0aae33f4e8d6d3105dc9e8f7621d3fffa Mon Sep 17 00:00:00 2001 From: Caleb Woodbine Date: Mon, 10 Jul 2023 15:48:31 +1200 Subject: [PATCH] feat: use GH Actions and separate Dockerfiles --- .github/workflows/build.yml | 118 +++++++++++++++++++++++++++++ Dockerfile.tmplate | 15 ++-- cmd/fdsn-quake-consumer/Dockerfile | 18 +---- etc/scripts/initdb.sh | 10 +-- 4 files changed, 132 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..92fcfc66 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,118 @@ +name: fdsn +on: + push: {} + release: + types: [published] + workflow_dispatch: {} +permissions: + packages: write + contents: write + pull-requests: write + id-token: write +env: + FOLDER: ./cmd + # doesn't have an ECR by that name; EXCLUDE is regex and is '|' separated (e.g: a|b|c) + EXCLUDE: s3-notify +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + git-rev: ${{ steps.git-rev.outputs.git-rev }} + matrix: ${{ steps.set.outputs.matrix }} + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: GeoNet/yq@bbe305500687a5fe8498d74883c17f0f06431ac4 # master + - id: git-rev + env: + GIT_SHA: ${{ github.sha }} + run: | + echo "git-rev=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - id: set + run: | + echo "matrix=$(find $FOLDER -mindepth 1 -maxdepth 1 -type d | grep -Ewv "$EXCLUDE" - | xargs -n 1 basename | xargs | yq 'split(" ")|.[]|{"target":.,"folder":env(FOLDER)+"/"+.}' -ojson | jq -rcM -s '{"include":.}')" >> $GITHUB_OUTPUT + - name: check output + run: | + jq . <<< '${{ steps.set.outputs.matrix }}' + build: + needs: prepare + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + uses: GeoNet/Actions/.github/workflows/reusable-docker-build.yml@main + with: + setup: | + # this is an anti-pattern + mkdir -p "${{ fromJSON(toJSON(matrix)).folder }}/assets" + DOCKERFILE="${{ fromJSON(toJSON(matrix)).folder }}/${{ fromJSON(toJSON(matrix)).target }}.Dockerfile" + if [ -f "${{ fromJSON(toJSON(matrix)).folder }}/Dockerfile" ]; then + echo "using existing" + cp "${{ fromJSON(toJSON(matrix)).folder }}/Dockerfile" "$DOCKERFILE" + else + echo "copy-editing template" + cp ./Dockerfile.tmplate "$DOCKERFILE" + cat << EOF >> "$DOCKERFILE" + CMD ["${{ fromJSON(toJSON(matrix)).target }}"] + EOF + fi + context: . + buildArgs: | + BUILD=${{ fromJSON(toJSON(matrix)).target }} + VERSION=git-${{ needs.prepare.outputs.git-rev }} + ASSET_DIR=${{ fromJSON(toJSON(matrix)).folder }}/assets + dockerfile: ${{ fromJSON(toJSON(matrix)).folder }}/${{ fromJSON(toJSON(matrix)).target }}.Dockerfile + imageName: ${{ fromJSON(toJSON(matrix)).target }} + platforms: linux/amd64 + push: ${{ github.ref == 'refs/heads/main' }} + tags: latest,git-${{ needs.prepare.outputs.git-rev }} + registryOverride: 862640294325.dkr.ecr.ap-southeast-2.amazonaws.com + aws-region: ap-southeast-2 + aws-role-arn-to-assume: arn:aws:iam::862640294325:role/github-actions-geonet-ecr-push + aws-role-duration-seconds: "3600" + go-build: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-go-build-smoke-test.yml@main + with: + paths: ${{ inputs.paths }} + gofmt: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-gofmt.yml@main + golangci-lint: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-golangci-lint.yml@main + go-vet: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-go-vet.yml@main + go-test: + runs-on: ubuntu-latest + env: + AWS_REGION: ap-southeast-2 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + check-latest: true + - name: setup + run: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + docker \ + run -d \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_USER=fdsn_w \ + -e POSTGRES_DB=fdsn \ + --name postgres \ + docker.io/postgis/postgis:15-3.3-alpine + echo "Waiting until Postgres is ready..." + until nc -zv -w 1 127.0.0.1 5432; do + sleep 1s + done + sleep 5s + docker logs postgres + echo "Postgres is ready" + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl + - name: test + run: | + ./all.sh diff --git a/Dockerfile.tmplate b/Dockerfile.tmplate index 65de7336..aeba540b 100644 --- a/Dockerfile.tmplate +++ b/Dockerfile.tmplate @@ -1,11 +1,11 @@ -ARG BUILDER_IMAGE=quay.io/geonet/golang:1.16-alpine -ARG RUNNER_IMAGE=quay.io/geonet/go-scratch:latest +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 +ARG RUNNER_IMAGE=ghcr.io/geonet/base-images/static:latest ARG RUN_USER=nobody # Only support image based on AlpineLinux FROM ${BUILDER_IMAGE} as builder # Obtain ca-cert and tzdata, which we will add to the container -RUN apk add --update ca-certificates tzdata gcc make musl-dev +RUN apk add --no-cache --update gcc make musl-dev # Project to build ARG BUILD @@ -13,9 +13,11 @@ ARG BUILD # Git commit SHA ARG GIT_COMMIT_SHA -COPY ./ /repo - WORKDIR /repo +COPY go.* /repo/ +COPY internal /repo/internal +COPY vendor /repo/vendor +COPY cmd/$BUILD /repo/cmd/$BUILD # Set a bunch of go env flags ENV GOBIN /repo/gobin @@ -23,8 +25,6 @@ ENV GOPATH /usr/src/go ENV GOFLAGS -mod=vendor ENV GOOS linux ENV GOARCH amd64 - -RUN echo 'nobody:x:65534:65534:Nobody:/:\' > /passwd RUN CGO_ENABLED=0 go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA} -extldflags -static" /repo/cmd/${BUILD} FROM ${RUNNER_IMAGE} @@ -34,7 +34,6 @@ EXPOSE $EXPOSE_PORT # Add common resource for ssl and timezones from the build container # Create a nobody user -COPY --from=builder /passwd /etc/passwd # Same ARG as before ARG BUILD # Need to make this an env for it to be interpolated by the shell diff --git a/cmd/fdsn-quake-consumer/Dockerfile b/cmd/fdsn-quake-consumer/Dockerfile index 10b23e8e..7e4b5bc1 100644 --- a/cmd/fdsn-quake-consumer/Dockerfile +++ b/cmd/fdsn-quake-consumer/Dockerfile @@ -1,15 +1,11 @@ -ARG BUILDER_IMAGE=quay.io/geonet/golang:1.16-alpine +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 FROM ${BUILDER_IMAGE} as builder # Obtain ca-cert and tzdata, which we will add to the container -RUN apk add --update ca-certificates tzdata - # Git commit SHA ARG GIT_COMMIT_SHA ARG BUILD - ADD ./ /repo - WORKDIR /repo # Set a bunch of go env flags @@ -19,21 +15,11 @@ ENV GOFLAGS -mod=vendor ENV GOOS linux ENV GOARCH amd64 ENV CGO_ENABLED 0 - -RUN echo 'nobody:x:65534:65534:Nobody:/:\' > /passwd RUN go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA}" /repo/cmd/fdsn-quake-consumer - -FROM quay.io/geonet/alpine:3.10 -RUN apk add --no-cache libxslt - -COPY --from=builder /passwd /etc/passwd -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +FROM ghcr.io/geonet/base-images/alpine-xslt:3.18 COPY --from=builder /repo/gobin/fdsn-quake-consumer /fdsn-quake-consumer - ARG ASSET_DIR COPY ${ASSET_DIR} /assets - WORKDIR / USER nobody EXPOSE 8080 diff --git a/etc/scripts/initdb.sh b/etc/scripts/initdb.sh index da8bfdfa..bda3aecc 100755 --- a/etc/scripts/initdb.sh +++ b/etc/scripts/initdb.sh @@ -28,13 +28,13 @@ export PGPASSWORD=$2 # Restart postgres. # dropdb --host=127.0.0.1 --username=$db_user fdsn -psql --host=127.0.0.1 -d postgres --username=$db_user --file=${ddl_dir}/create-users.ddl -psql --host=127.0.0.1 -d postgres --username=$db_user --file=${ddl_dir}/create-db.ddl +psql "postgresql://$db_user:$PGPASSWORD@127.0.0.1/postgres" --file=${ddl_dir}/create-users.ddl +psql "postgresql://$db_user:$PGPASSWORD@127.0.0.1/postgres" --file=${ddl_dir}/create-db.ddl # Function security means adding postgis has to be done as a superuser - here that is the postgres user. # On AWS RDS the created functions have to be transfered to the rds_superuser. # http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.PostGIS -psql --host=127.0.0.1 -d fdsn --username=$db_user -c 'create extension postgis;' -psql --host=127.0.0.1 --quiet --username=$db_user --dbname=fdsn --file=${ddl_dir}/drop-create.ddl -psql --host=127.0.0.1 --quiet --username=$db_user fdsn -f ${ddl_dir}/user-permissions.ddl +psql "postgresql://$db_user:$PGPASSWORD@127.0.0.1/fdsn" -c 'create extension if not exists postgis;' +psql "postgresql://$db_user:$PGPASSWORD@127.0.0.1/fdsn" --file=${ddl_dir}/drop-create.ddl +psql "postgresql://$db_user:$PGPASSWORD@127.0.0.1/fdsn" -f ${ddl_dir}/user-permissions.ddl