From 42b33269004d6586942512fdbf5bd7f8e9b0a0f3 Mon Sep 17 00:00:00 2001 From: Caleb Woodbine Date: Mon, 10 Jul 2023 15:48:31 +1200 Subject: [PATCH] feat: use GH Actions and separate Dockerfiles --- .github/workflows/build.yml | 101 ++++++++++++++++++++++++++ cmd/fdsn-holdings-consumer/Dockerfile | 40 ++++++++++ cmd/fdsn-quake-consumer/Dockerfile | 2 +- cmd/fdsn-slink-db/Dockerfile | 40 ++++++++++ cmd/fdsn-ws-nrt/Dockerfile | 40 ++++++++++ cmd/fdsn-ws/Dockerfile | 40 ++++++++++ etc/scripts/initdb.sh | 9 +-- 7 files changed, 266 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 cmd/fdsn-holdings-consumer/Dockerfile create mode 100644 cmd/fdsn-slink-db/Dockerfile create mode 100644 cmd/fdsn-ws-nrt/Dockerfile create mode 100644 cmd/fdsn-ws/Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..4343696e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,101 @@ +name: build +on: + push: {} + release: + types: [published] + workflow_dispatch: {} +permissions: + packages: write + contents: write + pull-requests: write + id-token: write +env: + FOLDER: ./cmd + # doesn't have an ECR by that name; EXCLUDE is regex and is '|' separated (e.g: a|b|c) + EXCLUDE: s3-notify +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + git-rev: ${{ steps.git-rev.outputs.git-rev }} + matrix: ${{ steps.set.outputs.matrix }} + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: GeoNet/yq@bbe305500687a5fe8498d74883c17f0f06431ac4 # master + - id: git-rev + env: + GIT_SHA: ${{ github.sha }} + run: | + echo "git-rev=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - id: set + run: | + echo "matrix=$(find $FOLDER -mindepth 1 -maxdepth 1 -type d | grep -Ewv "$EXCLUDE" - | xargs -n 1 basename | xargs | yq 'split(" ")|.[]|{"target":.,"folder":env(FOLDER)+"/"+.}' -ojson | jq -rcM -s '{"include":.}')" >> $GITHUB_OUTPUT + - name: check output + run: | + jq . <<< '${{ steps.set.outputs.matrix }}' + build: + needs: prepare + strategy: + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + uses: GeoNet/Actions/.github/workflows/reusable-docker-build.yml@main + with: + context: . + dockerfile: ${{ fromJSON(toJSON(matrix)).folder }}/Dockerfile + imageName: ${{ fromJSON(toJSON(matrix)).target }} + platforms: linux/amd64 + push: ${{ github.ref == 'refs/heads/main' }} + tags: latest,git-${{ needs.prepare.outputs.git-rev }} + registryOverride: 862640294325.dkr.ecr.ap-southeast-2.amazonaws.com + aws-region: ap-southeast-2 + aws-role-arn-to-assume: arn:aws:iam::862640294325:role/github-actions-geonet-ecr-push + aws-role-duration-seconds: "3600" + go-build: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-go-build-smoke-test.yml@main + with: + paths: ${{ inputs.paths }} + gofmt: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-gofmt.yml@main + golangci-lint: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-golangci-lint.yml@main + go-vet: + if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} + uses: GeoNet/Actions/.github/workflows/reusable-go-vet.yml@main + go-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + check-latest: true + - name: setup + run: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + docker \ + run -d \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_USER=fdsn_w \ + -e POSTGRES_DB=fdsn \ + --name postgres \ + docker.io/postgis/postgis:15-3.3-alpine + echo "Waiting until Postgres is ready..." + until nc -zv -w 1 127.0.0.1 5432; do + sleep 1s + done + sleep 5s + docker logs postgres + echo "Postgres is ready" + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl + - name: test + id: test + env: + AWS_REGION: ap-southeast-2 + run: | + ./all.sh diff --git a/cmd/fdsn-holdings-consumer/Dockerfile b/cmd/fdsn-holdings-consumer/Dockerfile new file mode 100644 index 00000000..a8e070b3 --- /dev/null +++ b/cmd/fdsn-holdings-consumer/Dockerfile @@ -0,0 +1,40 @@ +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 +ARG RUNNER_IMAGE=ghcr.io/geonet/base-images/static:latest +ARG RUN_USER=nobody +# Only support image based on AlpineLinux +FROM ${BUILDER_IMAGE} as builder + +# Project to build +ARG BUILD=fdsn-holdings-consumer + +# Git commit SHA +ARG GIT_COMMIT_SHA + +WORKDIR /repo +COPY go.* *.go /repo +RUN go mod download +COPY internal /repo/internal +COPY vendor /repo/vendor +COPY cmd/$BUILD /repo/cmd/$BUILD + +# Set a bunch of go env flags +ENV GOBIN /repo/gobin +ENV GOFLAGS -mod=vendor +ENV CGO_ENABLED=0 +RUN go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA} -extldflags -static" /repo/cmd/${BUILD} + +FROM ${RUNNER_IMAGE} +# Export a port, default to 8080 +ARG EXPOSE_PORT=8080 +EXPOSE $EXPOSE_PORT +ARG BUILD +# Need to make this an env for it to be interpolated by the shell +ENV TZ Pacific/Auckland +ENV BUILD_BIN=${BUILD} +# We have to make our binary have a fixed name, otherwise, we cannot run it without a shell +COPY --from=builder /repo/gobin/${BUILD} /${BUILD} +# Copy the assets +ARG ASSET_DIR +COPY ${ASSET_DIR} /assets +USER nobody +CMD ["/fdsn-holdings-consumer"] diff --git a/cmd/fdsn-quake-consumer/Dockerfile b/cmd/fdsn-quake-consumer/Dockerfile index 10b23e8e..405079d9 100644 --- a/cmd/fdsn-quake-consumer/Dockerfile +++ b/cmd/fdsn-quake-consumer/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILDER_IMAGE=quay.io/geonet/golang:1.16-alpine +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 FROM ${BUILDER_IMAGE} as builder # Obtain ca-cert and tzdata, which we will add to the container diff --git a/cmd/fdsn-slink-db/Dockerfile b/cmd/fdsn-slink-db/Dockerfile new file mode 100644 index 00000000..643cee58 --- /dev/null +++ b/cmd/fdsn-slink-db/Dockerfile @@ -0,0 +1,40 @@ +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 +ARG RUNNER_IMAGE=ghcr.io/geonet/base-images/static:latest +ARG RUN_USER=nobody +# Only support image based on AlpineLinux +FROM ${BUILDER_IMAGE} as builder + +# Project to build +ARG BUILD=fdsn-slink-db + +# Git commit SHA +ARG GIT_COMMIT_SHA + +WORKDIR /repo +COPY go.* *.go /repo +RUN go mod download +COPY internal /repo/internal +COPY vendor /repo/vendor +COPY cmd/$BUILD /repo/cmd/$BUILD + +# Set a bunch of go env flags +ENV GOBIN /repo/gobin +ENV GOFLAGS -mod=vendor +ENV CGO_ENABLED=0 +RUN go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA} -extldflags -static" /repo/cmd/${BUILD} + +FROM ${RUNNER_IMAGE} +# Export a port, default to 8080 +ARG EXPOSE_PORT=8080 +EXPOSE $EXPOSE_PORT +ARG BUILD +# Need to make this an env for it to be interpolated by the shell +ENV TZ Pacific/Auckland +ENV BUILD_BIN=${BUILD} +# We have to make our binary have a fixed name, otherwise, we cannot run it without a shell +COPY --from=builder /repo/gobin/${BUILD} /${BUILD} +# Copy the assets +ARG ASSET_DIR +COPY ${ASSET_DIR} /assets +USER nobody +CMD ["/fdsn-slink-db"] diff --git a/cmd/fdsn-ws-nrt/Dockerfile b/cmd/fdsn-ws-nrt/Dockerfile new file mode 100644 index 00000000..89fe01b8 --- /dev/null +++ b/cmd/fdsn-ws-nrt/Dockerfile @@ -0,0 +1,40 @@ +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 +ARG RUNNER_IMAGE=ghcr.io/geonet/base-images/static:latest +ARG RUN_USER=nobody +# Only support image based on AlpineLinux +FROM ${BUILDER_IMAGE} as builder + +# Project to build +ARG BUILD=fdsn-ws-nrt + +# Git commit SHA +ARG GIT_COMMIT_SHA + +WORKDIR /repo +COPY go.* *.go /repo +RUN go mod download +COPY internal /repo/internal +COPY vendor /repo/vendor +COPY cmd/$BUILD /repo/cmd/$BUILD + +# Set a bunch of go env flags +ENV GOBIN /repo/gobin +ENV GOFLAGS -mod=vendor +ENV CGO_ENABLED=0 +RUN go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA} -extldflags -static" /repo/cmd/${BUILD} + +FROM ${RUNNER_IMAGE} +# Export a port, default to 8080 +ARG EXPOSE_PORT=8080 +EXPOSE $EXPOSE_PORT +ARG BUILD +# Need to make this an env for it to be interpolated by the shell +ENV TZ Pacific/Auckland +ENV BUILD_BIN=${BUILD} +# We have to make our binary have a fixed name, otherwise, we cannot run it without a shell +COPY --from=builder /repo/gobin/${BUILD} /${BUILD} +# Copy the assets +ARG ASSET_DIR +COPY ${ASSET_DIR} /assets +USER nobody +CMD ["/fdsn-ws-nrt"] diff --git a/cmd/fdsn-ws/Dockerfile b/cmd/fdsn-ws/Dockerfile new file mode 100644 index 00000000..97250c60 --- /dev/null +++ b/cmd/fdsn-ws/Dockerfile @@ -0,0 +1,40 @@ +ARG BUILDER_IMAGE=ghcr.io/geonet/base-images/go:1.16 +ARG RUNNER_IMAGE=ghcr.io/geonet/base-images/static:latest +ARG RUN_USER=nobody +# Only support image based on AlpineLinux +FROM ${BUILDER_IMAGE} as builder + +# Project to build +ARG BUILD=fdsn-ws + +# Git commit SHA +ARG GIT_COMMIT_SHA + +WORKDIR /repo +COPY go.* *.go /repo +RUN go mod download +COPY internal /repo/internal +COPY vendor /repo/vendor +COPY cmd/$BUILD /repo/cmd/$BUILD + +# Set a bunch of go env flags +ENV GOBIN /repo/gobin +ENV GOFLAGS -mod=vendor +ENV CGO_ENABLED=0 +RUN go install -a -ldflags "-X main.Prefix=${BUILD}/${GIT_COMMIT_SHA} -extldflags -static" /repo/cmd/${BUILD} + +FROM ${RUNNER_IMAGE} +# Export a port, default to 8080 +ARG EXPOSE_PORT=8080 +EXPOSE $EXPOSE_PORT +ARG BUILD +# Need to make this an env for it to be interpolated by the shell +ENV TZ Pacific/Auckland +ENV BUILD_BIN=${BUILD} +# We have to make our binary have a fixed name, otherwise, we cannot run it without a shell +COPY --from=builder /repo/gobin/${BUILD} /${BUILD} +# Copy the assets +ARG ASSET_DIR +COPY ${ASSET_DIR} /assets +USER nobody +CMD ["/fdsn-ws"] diff --git a/etc/scripts/initdb.sh b/etc/scripts/initdb.sh index da8bfdfa..fca100db 100755 --- a/etc/scripts/initdb.sh +++ b/etc/scripts/initdb.sh @@ -28,13 +28,12 @@ export PGPASSWORD=$2 # Restart postgres. # dropdb --host=127.0.0.1 --username=$db_user fdsn -psql --host=127.0.0.1 -d postgres --username=$db_user --file=${ddl_dir}/create-users.ddl -psql --host=127.0.0.1 -d postgres --username=$db_user --file=${ddl_dir}/create-db.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres --file=${ddl_dir}/create-users.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres --file=${ddl_dir}/create-db.ddl # Function security means adding postgis has to be done as a superuser - here that is the postgres user. # On AWS RDS the created functions have to be transfered to the rds_superuser. # http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.PostGIS -psql --host=127.0.0.1 -d fdsn --username=$db_user -c 'create extension postgis;' -psql --host=127.0.0.1 --quiet --username=$db_user --dbname=fdsn --file=${ddl_dir}/drop-create.ddl -psql --host=127.0.0.1 --quiet --username=$db_user fdsn -f ${ddl_dir}/user-permissions.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres --file=${ddl_dir}/drop-create.ddl +psql postgresql://postgres:postgres@127.0.0.1/postgres -f ${ddl_dir}/user-permissions.ddl