From 936a09c7575b308bb2594ecf78914bec7cafa81b Mon Sep 17 00:00:00 2001 From: Caleb Woodbine Date: Mon, 10 Jul 2023 15:48:31 +1200 Subject: [PATCH] feat: use GH Actions and separate Dockerfiles --- .github/workflows/build.yml | 110 +++++++++++++++++++++++++++++ Dockerfile.tmplate | 12 ++-- cmd/fdsn-quake-consumer/Dockerfile | 40 ----------- etc/scripts/initdb.sh | 9 ++- 4 files changed, 121 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 cmd/fdsn-quake-consumer/Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..20c6da13 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,110 @@ +name: fdsn +on: + push: {} + release: + types: [published] + workflow_dispatch: {} +permissions: + packages: write + contents: write + pull-requests: write + id-token: write +env: + FOLDER: ./cmd + # doesn't have an ECR by that name; EXCLUDE is regex and is '|' separated (e.g: a|b|c) + EXCLUDE: s3-notify +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + git-rev: ${{ steps.git-rev.outputs.git-rev }} + matrix: ${{ steps.set.outputs.matrix }} + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: GeoNet/yq@bbe305500687a5fe8498d74883c17f0f06431ac4 # master + - id: git-rev + env: + GIT_SHA: ${{ github.sha }} + run: | + echo "git-rev=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - id: set + run: | + echo "matrix=$(find $FOLDER -mindepth 1 -maxdepth 1 -type d | grep -Ewv "$EXCLUDE" - | xargs -n 1 basename | xargs | yq 'split(" ")|.[]|{"target":.,"folder":env(FOLDER)+"/"+.}' -ojson | jq -rcM -s '{"include":.}')" >> $GITHUB_OUTPUT + - name: check output + run: | + jq . <<< '${{ steps.set.outputs.matrix }}' + build: + needs: prepare + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + uses: GeoNet/Actions/.github/workflows/reusable-docker-build.yml@main + with: + setup: | + # this is an anti-pattern + DOCKERFILE="${{ fromJSON(toJSON(matrix)).folder }}/${{ fromJSON(toJSON(matrix)).target }}.Dockerfile" + cp ./Dockerfile.tmplate "$DOCKERFILE" + cat << EOF >> "$DOCKERFILE" + CMD ["${{ fromJSON(toJSON(matrix)).target }}"] + EOF + context: . + dockerfile: ${{ fromJSON(toJSON(matrix)).folder }}/${{ fromJSON(toJSON(matrix)).target }}.Dockerfile + imageName: ${{ fromJSON(toJSON(matrix)).target }} + platforms: linux/amd64 + push: ${{ github.ref == 'refs/heads/main' }} + tags: latest,git-${{ needs.prepare.outputs.git-rev }} + registryOverride: 862640294325.dkr.ecr.ap-southeast-2.amazonaws.com + aws-region: ap-southeast-2 + aws-role-arn-to-assume: arn:aws:iam::862640294325:role/github-actions-geonet-ecr-push + aws-role-duration-seconds: "3600" + go-build: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-go-build-smoke-test.yml@main + with: + paths: ${{ inputs.paths }} + gofmt: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-gofmt.yml@main + golangci-lint: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-golangci-lint.yml@main + go-vet: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-go-vet.yml@main + go-test: + runs-on: ubuntu-latest + env: + AWS_REGION: ap-southeast-2 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + check-latest: true + - name: setup + run: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + docker \ + run -d \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_USER=fdsn_w \ + -e POSTGRES_DB=fdsn \ + --name postgres \ + docker.io/postgis/postgis:15-3.3-alpine + echo "Waiting until Postgres is ready..." + until nc -zv -w 1 127.0.0.1 5432; do + sleep 1s + done + sleep 5s + docker logs postgres + echo "Postgres is ready" + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl + - name: test + run: | + ./all.sh + - name: integration test + run: | + ./all-integration.sh diff --git a/Dockerfile.tmplate b/Dockerfile.tmplate index 65de7336..0bb5cde0 100644 --- a/Dockerfile.tmplate +++ b/Dockerfile.tmplate @@ -1,5 +1,5 @@ -ARG BUILDER_IMAGE=quay.io/geonet/golang:1.16-alpine -ARG RUNNER_IMAGE=quay.io/geonet/go-scratch:latest +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 +ARG RUNNER_IMAGE=ghcr.io/geonet/base-images/static:latest ARG RUN_USER=nobody # Only support image based on AlpineLinux FROM ${BUILDER_IMAGE} as builder @@ -13,9 +13,12 @@ ARG BUILD # Git commit SHA ARG GIT_COMMIT_SHA -COPY ./ /repo - WORKDIR /repo +COPY go.* *.go /repo +RUN go mod download +COPY internal /repo/internal +COPY vendor /repo/vendor +COPY cmd/$BUILD /repo/cmd/$BUILD # Set a bunch of go env flags ENV GOBIN /repo/gobin @@ -34,7 +37,6 @@ EXPOSE $EXPOSE_PORT # Add common resource for ssl and timezones from the build container # Create a nobody user -COPY --from=builder /passwd /etc/passwd # Same ARG as before ARG BUILD # Need to make this an env for it to be interpolated by the shell diff --git a/cmd/fdsn-quake-consumer/Dockerfile b/cmd/fdsn-quake-consumer/Dockerfile deleted file mode 100644 index 10b23e8e..00000000 --- a/cmd/fdsn-quake-consumer/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -ARG BUILDER_IMAGE=quay.io/geonet/golang:1.16-alpine - -FROM ${BUILDER_IMAGE} as builder -# Obtain ca-cert and tzdata, which we will add to the container -RUN apk add --update ca-certificates tzdata - -# Git commit SHA -ARG GIT_COMMIT_SHA -ARG BUILD - -ADD ./ /repo - -WORKDIR /repo - -# Set a bunch of go env flags -ENV GOBIN /repo/gobin -ENV GOPATH /usr/src/go -ENV GOFLAGS -mod=vendor -ENV GOOS linux -ENV GOARCH amd64 -ENV CGO_ENABLED 0 - -RUN echo 'nobody:x:65534:65534:Nobody:/:\' > /passwd -RUN go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA}" /repo/cmd/fdsn-quake-consumer - -FROM quay.io/geonet/alpine:3.10 -RUN apk add --no-cache libxslt - -COPY --from=builder /passwd /etc/passwd -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo -COPY --from=builder /repo/gobin/fdsn-quake-consumer /fdsn-quake-consumer - -ARG ASSET_DIR -COPY ${ASSET_DIR} /assets - -WORKDIR / -USER nobody -EXPOSE 8080 -CMD ["/fdsn-quake-consumer"] diff --git a/etc/scripts/initdb.sh b/etc/scripts/initdb.sh index da8bfdfa..fca100db 100755 --- a/etc/scripts/initdb.sh +++ b/etc/scripts/initdb.sh @@ -28,13 +28,12 @@ export PGPASSWORD=$2 # Restart postgres. # dropdb --host=127.0.0.1 --username=$db_user fdsn -psql --host=127.0.0.1 -d postgres --username=$db_user --file=${ddl_dir}/create-users.ddl -psql --host=127.0.0.1 -d postgres --username=$db_user --file=${ddl_dir}/create-db.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres --file=${ddl_dir}/create-users.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres --file=${ddl_dir}/create-db.ddl # Function security means adding postgis has to be done as a superuser - here that is the postgres user. # On AWS RDS the created functions have to be transfered to the rds_superuser. # http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.PostGIS -psql --host=127.0.0.1 -d fdsn --username=$db_user -c 'create extension postgis;' -psql --host=127.0.0.1 --quiet --username=$db_user --dbname=fdsn --file=${ddl_dir}/drop-create.ddl -psql --host=127.0.0.1 --quiet --username=$db_user fdsn -f ${ddl_dir}/user-permissions.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres --file=${ddl_dir}/drop-create.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres -f ${ddl_dir}/user-permissions.ddl