merge main

Makefile

@@ -1,10 +1,30 @@
-include .bingo/Variables.mk
+PROJECT := enduro
+MAKEDIR := hack/make
+SHELL   := /bin/bash
+
+.DEFAULT_GOAL := help
+.PHONY: *
+
+DBG_MAKEFILE ?=
+ifeq ($(DBG_MAKEFILE),1)
+    $(warning ***** starting Makefile for goal(s) "$(MAKECMDGOALS)")
+    $(warning ***** $(shell date))
+else
+    # If we're not debugging the Makefile, don't echo recipes.
+    MAKEFLAGS += -s
+endif
+
+include hack/make/bootstrap.mk
+include hack/make/dep_goa.mk
+include hack/make/dep_golangci_lint.mk
+include hack/make/dep_gomajor.mk
+include hack/make/dep_goreleaser.mk
+include hack/make/dep_gotestsum.mk
+include hack/make/dep_hugo.mk
+include hack/make/dep_jq.mk
+include hack/make/dep_mockgen.mk
+include hack/make/dep_temporal_cli.mk
 
-SHELL=/bin/bash
-BUILD_TIME=$(shell date -u +%Y-%m-%dT%T%z)
-GIT_COMMIT=$(shell git rev-parse --short HEAD)
-LD_FLAGS= '-X "main.buildTime=$(BUILD_TIME)" -X main.gitCommit=$(GIT_COMMIT)'
-GO_FLAGS= -ldflags=$(LD_FLAGS)
 
 define NEWLINE
 
@@ -38,82 +58,68 @@ IGNORED_PACKAGES := \
 PACKAGES		:= $(shell go list ./...)
 TEST_PACKAGES	:= $(filter-out $(IGNORED_PACKAGES),$(PACKAGES))
 
-export PATH:=$(GOBIN):$(PATH)
-
-.DEFAULT_GOAL := run
-
-$(GOBIN)/bingo:
-	$(GO) install github.com/bwplotka/bingo@latest
-
-bingo: $(GOBIN)/bingo
-
-tools: bingo
-	bingo get
-	bingo list
-
-run: enduro-dev
+run: build # @HELP Builds and run the enduro binary.
 	./build/enduro
 
-enduro-dev:
+build: # @HELP Builds the enduro binary.
+build: GO         ?= $(shell which go)
+build: BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%T%z)
+build: GIT_COMMIT ?= $(shell git rev-parse --short HEAD)
+build: LD_FLAGS   ?= '-X "main.buildTime=$(BUILD_TIME)" -X main.gitCommit=$(GIT_COMMIT)'
+build: GO_FLAGS   ?= -ldflags=$(LD_FLAGS)
+build:
 	mkdir -p ./build
 	$(GO) build -trimpath -o build/enduro $(GO_FLAGS) -v
 
-test:
-	@$(GOTESTSUM) $(TEST_PACKAGES)
+deps: $(GOMAJOR) # @HELP Lists available module dependency updates.
+	gomajor list
 
-test-race:
-	@$(GOTESTSUM) $(TEST_PACKAGES) -- -race
+test: $(GOTESTSUM) # @HELP Tests using gotestsum.
+	gotestsum $(TEST_PACKAGES)
 
-ignored:
+test-race: $(GOTESTSUM) # @HELP Tests using gotestsum and the race detector.
+	gotestsum $(TEST_PACKAGES) -- -race
+
+ignored: # @HELP Prints ignored packages.
 	$(foreach PACKAGE,$(IGNORED_PACKAGES),@echo $(PACKAGE)$(NEWLINE))
 
-lint:
-	$(GOLANGCI_LINT) run -v --timeout=5m --fix
+lint: $(GOLANGCI_LINT) # @HELP Lints the code using golangci-lint.
+	golangci-lint run -v --timeout=5m --fix
+
+gen-goa: $(GOA) # @HELP Generates Goa assets.
+	goa gen github.com/artefactual-labs/enduro/internal/api/design -o internal/api
+	@$(MAKE) gen-goa-json-pretty
 
-goagen:
-	$(GOA) gen github.com/artefactual-labs/enduro/internal/api/design -o internal/api
+gen-goa-json-pretty: goa_http_dir = "internal/api/gen/http"
+gen-goa-json-pretty: json_files = $(shell find $(goa_http_dir) -type f -name "*.json" | sort -u)
+gen-goa-json-pretty: $(JQ)
+	@for f in $(json_files); \
+		do (cat "$$f" | jq -S '.' >> "$$f".sorted && mv "$$f".sorted "$$f") \
+			&& echo "Formatting $$f with jq" || exit 1; \
+	done
 
-clean:
+clean: # @HELP Cleans temporary files.
 	rm -rf ./build ./dist
 
-release-test-config:
-	$(GORELEASER) --snapshot --skip-publish --rm-dist
+release-test-config: $(GORELEASER) # @HELP Tests the goreleaser config.
+	goreleaser --snapshot --skip-publish --rm-dist
 
-release-test:
-	$(GORELEASER) --skip-publish
+release-test: $(GORELEASER) # @HELP Tests the release with goreleaser.
+	goreleaser --skip-publish
 
-PROJECT := enduro
-UNAME_OS := $(shell uname -s)
-UNAME_ARCH := $(shell uname -m)
-CACHE_BASE := $(HOME)/.cache/$(PROJECT)
-CACHE := $(CACHE_BASE)/$(UNAME_OS)/$(UNAME_ARCH)
-CACHE_BIN := $(CACHE)/bin
-export PATH := $(abspath $(CACHE_BIN)):$(PATH)
-CACHE_VERSIONS := $(CACHE)/versions
-HUGO_VERSION := 0.113.0
-HUGO := $(CACHE_VERSIONS)/hugo/$(HUGO_VERSION)
-$(HUGO):
-	@rm -f $(CACHE_BIN)/hugo
-	@mkdir -p $(CACHE_BIN)
-	curl -sSL "https://github.com/gohugoio/hugo/releases/download/v$(HUGO_VERSION)/hugo_extended_$(HUGO_VERSION)_Linux-64bit.tar.gz" | tar xzf - -C "$(CACHE_BIN)"
-	chmod +x "$(CACHE_BIN)/hugo"
-	@rm -rf $(dir $(HUGO))
-	@mkdir -p $(dir $(HUGO))
-	@touch $(HUGO)
-
-website: $(HUGO)
+website: $(HUGO) # @HELP Serves the website for development.
 	hugo serve --source=website/
 
-ui:
+ui: # @HELP Builds the UI.
 	npm --prefix=ui install
 	npm --prefix=ui run build
 
-ui-dev:
+ui-dev: # @HELP Serves the UI for development.
 	npm --prefix=ui run dev
 
-ui-client:
-	@rm -rf $(CURDIR)/ui/src/client
-	@docker container run --rm --user $(shell id -u):$(shell id -g) --volume $(CURDIR):/local openapitools/openapi-generator-cli:v4.2.3 \
+ui-client: # @HELP Generates the UI client using openapi-generator-cli.
+	rm -rf $(CURDIR)/ui/src/client
+	docker container run --rm --user $(shell id -u):$(shell id -g) --volume $(CURDIR):/local openapitools/openapi-generator-cli:v4.2.3 \
 		generate \
 			--input-spec /local/internal/api/gen/http/openapi.json \
 			--generator-name typescript-fetch \
@@ -122,26 +128,39 @@ ui-client:
 			-p "generateAliasAsModel=true" \
 			-p "typescriptThreePlus=true" \
 			-p "withInterfaces=true"
-	@echo "@@@@ Please, review all warnings generated by openapi-generator-cli above!"
-	@echo "@@@@ We're using \`--skip-validate-spec\` to deal with Goa spec generation issues."
+	echo "@@@@ Please, review all warnings generated by openapi-generator-cli above!"
+	echo "@@@@ We're using \`--skip-validate-spec\` to deal with Goa spec generation issues."
 
-db:
+db: # @HELP Opens the MySQL CLI.
 	docker compose exec --user=root mysql mysql -hlocalhost -uroot -proot123
 
-flush:
+flush: # @HELP Flushes the enduro database.
 	docker compose exec --user=root mysql mysql -hlocalhost -uroot -proot123 -e "drop database enduro"
 	docker compose exec --user=root mysql mysql -hlocalhost -uroot -proot123 -e "create database enduro"
 
-gen-mock:
-	$(MOCKGEN) -destination=./internal/batch/fake/mock_batch.go -package=fake github.com/artefactual-labs/enduro/internal/batch Service
-	$(MOCKGEN) -destination=./internal/collection/fake/mock_collection.go -package=fake github.com/artefactual-labs/enduro/internal/collection Service
-	$(MOCKGEN) -destination=./internal/pipeline/fake/mock_pipeline.go -package=fake github.com/artefactual-labs/enduro/internal/pipeline Service
-	$(MOCKGEN) -destination=./internal/watcher/fake/mock_watcher.go -package=fake github.com/artefactual-labs/enduro/internal/watcher Service
-	$(MOCKGEN) -destination=./internal/amclient/fake/mock_ingest.go -package=fake github.com/artefactual-labs/enduro/internal/amclient IngestService
-	$(MOCKGEN) -destination=./internal/amclient/fake/mock_processing_config.go -package=fake github.com/artefactual-labs/enduro/internal/amclient ProcessingConfigService
-	$(MOCKGEN) -destination=./internal/amclient/fake/mock_transfer.go -package=fake github.com/artefactual-labs/enduro/internal/amclient TransferService
-	$(MOCKGEN) -destination=./internal/amclient/fake/mock_v2_jobs.go -package=fake github.com/artefactual-labs/enduro/internal/amclient JobsService
-	$(MOCKGEN) -destination=./internal/amclient/fake/mock_v2_package.go -package=fake github.com/artefactual-labs/enduro/internal/amclient PackageService
-	$(MOCKGEN) -destination=./internal/amclient/fake/mock_v2_task.go -package=fake github.com/artefactual-labs/enduro/internal/amclient TaskService
-
-.PHONY: *
+gen-mock: # @HELP Generates mocks with mockgen.
+gen-mock: $(MOCKGEN)
+	mockgen -destination=./internal/batch/fake/mock_batch.go -package=fake github.com/artefactual-labs/enduro/internal/batch Service
+	mockgen -destination=./internal/collection/fake/mock_collection.go -package=fake github.com/artefactual-labs/enduro/internal/collection Service
+	mockgen -destination=./internal/pipeline/fake/mock_pipeline.go -package=fake github.com/artefactual-labs/enduro/internal/pipeline Service
+	mockgen -destination=./internal/watcher/fake/mock_watcher.go -package=fake github.com/artefactual-labs/enduro/internal/watcher Service
+	mockgen -destination=./internal/amclient/fake/mock_ingest.go -package=fake github.com/artefactual-labs/enduro/internal/amclient IngestService
+	mockgen -destination=./internal/amclient/fake/mock_processing_config.go -package=fake github.com/artefactual-labs/enduro/internal/amclient ProcessingConfigService
+	mockgen -destination=./internal/amclient/fake/mock_transfer.go -package=fake github.com/artefactual-labs/enduro/internal/amclient TransferService
+	mockgen -destination=./internal/amclient/fake/mock_v2_jobs.go -package=fake github.com/artefactual-labs/enduro/internal/amclient JobsService
+	mockgen -destination=./internal/amclient/fake/mock_v2_package.go -package=fake github.com/artefactual-labs/enduro/internal/amclient PackageService
+	mockgen -destination=./internal/amclient/fake/mock_v2_task.go -package=fake github.com/artefactual-labs/enduro/internal/amclient TaskService
+
+temporal: # @HELP Runs a development instance of Temporal.
+temporal: PORT := 55555
+temporal: LOG_LEVEL := warn
+temporal: $(TEMPORAL_CLI)
+	temporal server start-dev --namespace=default --port=$(PORT) --headless --log-format=pretty --log-level=$(LOG_LEVEL)
+
+help: # @HELP Prints this message.
+	echo "TARGETS:"
+	grep -E '^.*: *# *@HELP' Makefile             \
+	    | awk '                                   \
+	        BEGIN {FS = ": *# *@HELP"};           \
+	        { printf "  %-30s %s\n", $$1, $$2 };  \
+	    '

SECURITY.md

@@ -0,0 +1,65 @@
+# Security Policy
+
+This document outlines security procedures and general policies for the Enduro
+project.
+
+**Contents**
+
+* [Reporting a security vulnerability](#reporting-a-security-vulnerability)
+* [Disclosure policy](#disclosure-policy)
+* [Supported versions](#supported-versions)
+* [Reporting general bugs](#reporting-general-bugs)
+
+## Reporting a security vulnerability
+
+The Enduro development team takes security seriously and will investigate all
+reported vulnerabilities.
+
+If you would like to report a vulnerability or have a security concern regarding
+Enduro, **please do not file a public issue in our GitHub repository or post
+about the issue on the user forum.** It is critical to the safety of other users
+that security issues are reported in a secure manner. Instead, please email a
+report to:
+
+* [[email protected]](mailto:[email protected])
+
+We will be better able to evaluate and respond to your report if it includes
+all the details needed for us to reproduce the issue locally. Please include
+the following information in your email:
+
+* The version of Enduro you are using.
+* Basic information about your installation environment, including operating
+  system and dependency versions.
+* Steps to reproduce the issue.
+* The resulting error or vulnerability.
+* If there are any error logs related to the issue, please include the
+  relevant parts as well.
+
+Your report will be acknowledged within 2 business days, and we’ll follow up
+with a more detailed response indicating the next steps we intend to take
+within 1 week.
+
+If you haven’t received a reply to your submission after 5 business days of
+the original report, please email Artefactual's info address:
+[[email protected]]([email protected]).
+
+Any information you share with the Enduro development team as a part of
+this process will be kept confidential within the team. If we determine that the
+vulnerability is located upstream in one of the libraries or dependencies that
+Enduro uses, we may need to share some information about the report with the
+dependency’s core team - in this case, we will notify you before proceeding.
+
+If the vulnerability is first reported by you, we will credit you with the
+discovery in the public disclosure, unless you tell us you would prefer to
+remain anonymous.
+
+## Disclosure policy
+
+When the Enduro development team receives a security bug report, we will assign
+it to a primary handler. This person will coordinate the fix and release
+process, involving the following steps:
+
+* Confirm the problem and determine the affected versions.
+* Audit code to find any similar potential problems.
+* Prepare fixes for all releases still under maintenance. These fixes will be
+  released as fast as possible.

enduro.toml

@@ -1,5 +1,6 @@
 # This is the configuration file valid for the development environment.
 
+verbosity = 2
 debug = true
 debugListen = "127.0.0.1:9001"
 

go.mod

@@ -3,8 +3,8 @@ module github.com/artefactual-labs/enduro
 go 1.20
 
 require (
-	github.com/alicebob/miniredis/v2 v2.30.3
-	github.com/aws/aws-sdk-go v1.44.290
+	github.com/alicebob/miniredis/v2 v2.30.4
+	github.com/aws/aws-sdk-go v1.44.299
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/go-logr/logr v1.2.4
@@ -29,13 +29,13 @@ require (
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	go.artefactual.dev/tools v0.3.0
-	go.temporal.io/api v1.21.0
+	go.temporal.io/api v1.23.0
 	go.temporal.io/sdk v1.23.1
-	goa.design/goa/v3 v3.11.3
-	goa.design/plugins/v3 v3.11.3
+	goa.design/goa/v3 v3.12.1
+	goa.design/plugins/v3 v3.12.1
 	gocloud.dev v0.30.0
 	golang.org/x/sync v0.3.0
-	gotest.tools/v3 v3.4.0
+	gotest.tools/v3 v3.5.0
 )
 
 require (
@@ -111,17 +111,17 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
-	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.11.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
-	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.128.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/grpc v1.56.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
+	google.golang.org/grpc v1.56.1 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )