From 9309a294f472712ad1f402e63752c3f03a03a7e2 Mon Sep 17 00:00:00 2001 From: Howard Wu Date: Wed, 26 Jun 2024 15:52:40 +1200 Subject: [PATCH] chore: reusable workflow --- .github/workflows/build.yml | 79 ++--- .../data_holdings_test.go | 3 +- cmd/fdsn-holdings-consumer/log.go | 3 +- cmd/fdsn-quake-consumer/log.go | 3 +- cmd/fdsn-ws/data_holdings.go | 3 +- cmd/fdsn-ws/data_holdings_test.go | 7 +- cmd/fdsn-ws/fdsn_dataselect.go | 2 +- cmd/fdsn-ws/fdsn_event.go | 2 +- cmd/fdsn-ws/fdsn_station.go | 2 +- cmd/fdsn-ws/fdsn_station_test.go | 27 +- cmd/fdsn-ws/fdsn_station_type.go | 2 + cmd/fdsn-ws/log.go | 5 +- cmd/fdsn-ws/routes.go | 2 +- cmd/fdsn-ws/routes_integration_test.go | 4 +- cmd/fdsn-ws/sc3ml.go | 3 +- cmd/fdsn-ws/server.go | 19 +- cmd/fdsn-ws/server_test.go | 7 + cmd/s3-notify/s3-notify.go | 3 + go.mod | 1 + go.sum | 2 + internal/fdsn/dataselect.go | 10 +- internal/holdings/holdings_test.go | 3 +- internal/valid/valid_test.go | 3 +- vendor/github.com/joho/godotenv/.gitignore | 1 + vendor/github.com/joho/godotenv/LICENCE | 23 ++ vendor/github.com/joho/godotenv/README.md | 202 +++++++++++++ vendor/github.com/joho/godotenv/godotenv.go | 228 +++++++++++++++ vendor/github.com/joho/godotenv/parser.go | 271 ++++++++++++++++++ vendor/modules.txt | 3 + 29 files changed, 830 insertions(+), 93 deletions(-) create mode 100644 vendor/github.com/joho/godotenv/.gitignore create mode 100644 vendor/github.com/joho/godotenv/LICENCE create mode 100644 vendor/github.com/joho/godotenv/README.md create mode 100644 vendor/github.com/joho/godotenv/godotenv.go create mode 100644 vendor/github.com/joho/godotenv/parser.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6c809e6..db975132 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,35 @@ jobs: - name: check output run: | jq . <<< '${{ steps.set.outputs.matrix }}' + build-app: + uses: GeoNet/Actions/.github/workflows/reusable-go-apps.yml@main + with: + buildSetup: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + testSetup: | + sudo apt-get -yq update + sudo apt-get install -y xsltproc + docker \ + run -d \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_USER=fdsn_w \ + -e POSTGRES_DB=fdsn \ + --name postgres \ + docker.io/postgis/postgis:15-3.3-alpine + echo "Waiting until Postgres is ready..." + until nc -zv -w 1 127.0.0.1 5432; do + sleep 1s + done + sleep 5s + docker logs postgres + echo "Postgres is ready" + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl + psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl + goTestExtraArgs: -p 1 build: - needs: prepare + needs: [prepare, build-app] strategy: matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} uses: GeoNet/Actions/.github/workflows/reusable-docker-build.yml@main @@ -66,52 +93,4 @@ jobs: aws-region: ap-southeast-2 aws-role-arn-to-assume: arn:aws:iam::862640294325:role/github-actions-geonet-ecr-push aws-role-duration-seconds: "3600" - go-build: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-go-build-smoke-test.yml@main - with: - paths: ${{ inputs.paths }} - gofmt: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-gofmt.yml@main - golangci-lint: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-golangci-lint.yml@main - go-vet: - if: ${{ contains(fromJSON('["workflow_call", "push", "pull_request"]'), github.event_name) && startsWith(github.repository, 'GeoNet/') != false }} - uses: GeoNet/Actions/.github/workflows/reusable-go-vet.yml@main - go-test: - runs-on: ubuntu-latest - env: - AWS_REGION: ap-southeast-2 - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 - with: - go-version-file: go.mod - cache-dependency-path: go.sum - check-latest: true - - name: setup - run: | - sudo apt-get -yq update - sudo apt-get install -y xsltproc - docker \ - run -d \ - -p 5432:5432 \ - -e POSTGRES_PASSWORD=test \ - -e POSTGRES_USER=fdsn_w \ - -e POSTGRES_DB=fdsn \ - --name postgres \ - docker.io/postgis/postgis:15-3.3-alpine - echo "Waiting until Postgres is ready..." - until nc -zv -w 1 127.0.0.1 5432; do - sleep 1s - done - sleep 5s - docker logs postgres - echo "Postgres is ready" - psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/drop-create.ddl - psql postgresql://fdsn_w:test@127.0.0.1/fdsn --file=./etc/ddl/create-users.ddl - - name: test - run: | - ./all.sh + diff --git a/cmd/fdsn-holdings-consumer/data_holdings_test.go b/cmd/fdsn-holdings-consumer/data_holdings_test.go index b781db7e..c5b145fa 100644 --- a/cmd/fdsn-holdings-consumer/data_holdings_test.go +++ b/cmd/fdsn-holdings-consumer/data_holdings_test.go @@ -2,9 +2,10 @@ package main import ( "database/sql" - "github.com/GeoNet/fdsn/internal/holdings" "testing" "time" + + "github.com/GeoNet/fdsn/internal/holdings" ) func TestSaveHoldings(t *testing.T) { diff --git a/cmd/fdsn-holdings-consumer/log.go b/cmd/fdsn-holdings-consumer/log.go index 97301950..8f831791 100644 --- a/cmd/fdsn-holdings-consumer/log.go +++ b/cmd/fdsn-holdings-consumer/log.go @@ -1,9 +1,10 @@ package main import ( - "github.com/GeoNet/kit/metrics" "log" "os" + + "github.com/GeoNet/kit/metrics" ) var Prefix string diff --git a/cmd/fdsn-quake-consumer/log.go b/cmd/fdsn-quake-consumer/log.go index 97301950..8f831791 100644 --- a/cmd/fdsn-quake-consumer/log.go +++ b/cmd/fdsn-quake-consumer/log.go @@ -1,9 +1,10 @@ package main import ( - "github.com/GeoNet/kit/metrics" "log" "os" + + "github.com/GeoNet/kit/metrics" ) var Prefix string diff --git a/cmd/fdsn-ws/data_holdings.go b/cmd/fdsn-ws/data_holdings.go index 5b6d03f9..82a1221a 100644 --- a/cmd/fdsn-ws/data_holdings.go +++ b/cmd/fdsn-ws/data_holdings.go @@ -2,8 +2,9 @@ package main import ( "database/sql" - "github.com/GeoNet/fdsn/internal/fdsn" "time" + + "github.com/GeoNet/fdsn/internal/fdsn" ) type metric struct { diff --git a/cmd/fdsn-ws/data_holdings_test.go b/cmd/fdsn-ws/data_holdings_test.go index 690b3770..b661b7cd 100644 --- a/cmd/fdsn-ws/data_holdings_test.go +++ b/cmd/fdsn-ws/data_holdings_test.go @@ -1,12 +1,13 @@ package main import ( - "github.com/GeoNet/fdsn/internal/fdsn" - "github.com/GeoNet/fdsn/internal/holdings" - "github.com/lib/pq" "log" "testing" "time" + + "github.com/GeoNet/fdsn/internal/fdsn" + "github.com/GeoNet/fdsn/internal/holdings" + "github.com/lib/pq" ) // http://www.postgresql.org/docs/9.4/static/errcodes-appendix.html diff --git a/cmd/fdsn-ws/fdsn_dataselect.go b/cmd/fdsn-ws/fdsn_dataselect.go index 07bd2d53..05d5d50b 100644 --- a/cmd/fdsn-ws/fdsn_dataselect.go +++ b/cmd/fdsn-ws/fdsn_dataselect.go @@ -41,7 +41,7 @@ type dataSelect struct { keys []string } -func init() { +func initDataselectTemplate() { // Handle comma separated parameters (eg: net, sta, loc, cha, etc) decoder.RegisterConverter([]string{}, func(input string) reflect.Value { return reflect.ValueOf(strings.Split(input, ",")) diff --git a/cmd/fdsn-ws/fdsn_event.go b/cmd/fdsn-ws/fdsn_event.go index 19bbcbde..f13f4153 100644 --- a/cmd/fdsn-ws/fdsn_event.go +++ b/cmd/fdsn-ws/fdsn_event.go @@ -95,7 +95,7 @@ var validEventTypes = strings.Split( ", ", ","), ",") // remove spaces after comma -func init() { +func initEventTemplate() { var err error var b bytes.Buffer diff --git a/cmd/fdsn-ws/fdsn_station.go b/cmd/fdsn-ws/fdsn_station.go index a86d3869..3188f843 100644 --- a/cmd/fdsn-ws/fdsn_station.go +++ b/cmd/fdsn-ws/fdsn_station.go @@ -112,7 +112,7 @@ var ( s3Meta string ) -func init() { +func initStationTemplate() { var err error var b bytes.Buffer diff --git a/cmd/fdsn-ws/fdsn_station_test.go b/cmd/fdsn-ws/fdsn_station_test.go index 1b36a502..42fd7653 100644 --- a/cmd/fdsn-ws/fdsn_station_test.go +++ b/cmd/fdsn-ws/fdsn_station_test.go @@ -12,9 +12,6 @@ import ( // NOTE: To run the test, please export : // STATION_XML_META_KEY=fdsn-station-test.xml -func init() { -} - func TestStationFilter(t *testing.T) { var e fdsnStationV1Search var err error @@ -207,13 +204,16 @@ Testdata timeline Station 1: 2007-05-20T23 2011-03-06T22 2011-06-20T04 - |------3 cha----| - |-----3 cha----| - |-----3 cha-----------> + + |------3 cha----| + |-----3 cha----| + |-----3 cha-----------> + Station 2: - 2010-03-11T21 2012-01-19T22 - |-----3 cha----| - |-----3 cha-------> + + 2010-03-11T21 2012-01-19T22 + |-----3 cha----| + |-----3 cha-------> */ func TestStartEnd(t *testing.T) { var e fdsnStationV1Search @@ -408,10 +408,11 @@ NZ|ARAZ|-38.627690|176.120060|420.000000|Aratiatia Landcorp Farm|2007-05-20T23:0 } // To profiling, you'll have to use full fdsn-station xml as data source: -// 1. Put full fdsn-station.xml in etc/. -// 2. export FDSN_STATION_XML_META_KEY=fdsn-station.xml -// 3. Run `go test -bench=StationQuery -benchmem -run=^$`. -// Note: You must specify -run=^$ to skip test functions since you're not using test fdsn-station xml. +// 1. Put full fdsn-station.xml in etc/. +// 2. export FDSN_STATION_XML_META_KEY=fdsn-station.xml +// 3. Run `go test -bench=StationQuery -benchmem -run=^$`. +// Note: You must specify -run=^$ to skip test functions since you're not using test fdsn-station xml. +// // Currently the benchmark result for my MacBookPro 2017 is: // BenchmarkStationQuery/post-4 20000 74472 ns/op 78376 B/op 706 allocs/op func BenchmarkStationQuery(b *testing.B) { diff --git a/cmd/fdsn-ws/fdsn_station_type.go b/cmd/fdsn-ws/fdsn_station_type.go index cae58bf1..f46cefe9 100644 --- a/cmd/fdsn-ws/fdsn_station_type.go +++ b/cmd/fdsn-ws/fdsn_station_type.go @@ -247,6 +247,7 @@ type NetworkType struct { } // May be one of NOMINAL, CALCULATED +// //nolint:deadcode,unused // Struct based on FDSN spec so keep it type NominalType string @@ -364,6 +365,7 @@ type SampleRateType struct { } // A time value in seconds. +// //nolint:deadcode,unused // Struct based on FDSN spec so keep it type SecondType struct { Value float64 `xml:",chardata"` diff --git a/cmd/fdsn-ws/log.go b/cmd/fdsn-ws/log.go index 1cd0696f..cdf56ba7 100644 --- a/cmd/fdsn-ws/log.go +++ b/cmd/fdsn-ws/log.go @@ -1,10 +1,11 @@ package main import ( - "github.com/GeoNet/kit/metrics" - "github.com/GeoNet/kit/weft" "log" "os" + + "github.com/GeoNet/kit/metrics" + "github.com/GeoNet/kit/weft" ) var Prefix string diff --git a/cmd/fdsn-ws/routes.go b/cmd/fdsn-ws/routes.go index a6e65f61..76b276ae 100644 --- a/cmd/fdsn-ws/routes.go +++ b/cmd/fdsn-ws/routes.go @@ -12,7 +12,7 @@ import ( var mux *http.ServeMux -func init() { +func initRoutes() { mux = http.NewServeMux() mux.HandleFunc("/", weft.MakeHandler(weft.NoMatch, weft.TextError)) diff --git a/cmd/fdsn-ws/routes_integration_test.go b/cmd/fdsn-ws/routes_integration_test.go index 36f7ffc8..108b9af8 100644 --- a/cmd/fdsn-ws/routes_integration_test.go +++ b/cmd/fdsn-ws/routes_integration_test.go @@ -1,10 +1,12 @@ +//go:build integration // +build integration package main import ( - wt "github.com/GeoNet/kit/weft/wefttest" "testing" + + wt "github.com/GeoNet/kit/weft/wefttest" ) var routesIntegration = wt.Requests{ diff --git a/cmd/fdsn-ws/sc3ml.go b/cmd/fdsn-ws/sc3ml.go index 40db1034..1313f222 100644 --- a/cmd/fdsn-ws/sc3ml.go +++ b/cmd/fdsn-ws/sc3ml.go @@ -2,9 +2,10 @@ package main import ( "bytes" + "net/http" + "github.com/GeoNet/fdsn/internal/valid" "github.com/GeoNet/kit/weft" - "net/http" ) func s3ml(r *http.Request, h http.Header, b *bytes.Buffer) error { diff --git a/cmd/fdsn-ws/server.go b/cmd/fdsn-ws/server.go index 3ffa8099..6f118b18 100644 --- a/cmd/fdsn-ws/server.go +++ b/cmd/fdsn-ws/server.go @@ -13,20 +13,16 @@ import ( ) var ( - db *sql.DB - decoder = schema.NewDecoder() // decoder for URL queries. - S3_BUCKET string // the S3 bucket storing the miniseed files used by dataselect - LOG_EXTRA bool // Whether POST body is logged. - zeroDateTime time.Time + db *sql.DB + decoder = schema.NewDecoder() // decoder for URL queries. + S3_BUCKET string // the S3 bucket storing the miniseed files used by dataselect + LOG_EXTRA bool // Whether POST body is logged. ) var stationVersion = "1.1" var eventVersion = "1.2" var dataselectVersion = "1.1" - -func init() { - zeroDateTime = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC) -} +var zeroDateTime = time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC) func main() { var err error @@ -60,6 +56,11 @@ func main() { log.Println("ERROR: problem pinging DB - is it up and contactable? 500s will be served") } + initDataselectTemplate() + initEventTemplate() + initStationTemplate() + initRoutes() + setupStationXMLUpdater() log.Println("starting server") diff --git a/cmd/fdsn-ws/server_test.go b/cmd/fdsn-ws/server_test.go index aa0aac3c..87e0cbdb 100644 --- a/cmd/fdsn-ws/server_test.go +++ b/cmd/fdsn-ws/server_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "os" "testing" + + "github.com/joho/godotenv" ) var ts *httptest.Server @@ -14,6 +16,11 @@ var ts *httptest.Server func setup(t *testing.T) { var err error + err = godotenv.Load("env.list") + if err != nil { + t.Fatal(err) + } + S3_BUCKET = os.Getenv("S3_BUCKET") // need a db write user for adding test data. diff --git a/cmd/s3-notify/s3-notify.go b/cmd/s3-notify/s3-notify.go index 22eae8fe..c4bc3bf4 100644 --- a/cmd/s3-notify/s3-notify.go +++ b/cmd/s3-notify/s3-notify.go @@ -24,7 +24,9 @@ func init() { flag.StringVar(&keyPrefix, "key-prefix", "", "Key prefix to search in the S3 bucket.") flag.StringVar(&sqsUrl, "sqs-url", "", "SQS queue url to send notifications to. Omit this parameter to show the list of matched keys only.") flag.Parse() +} +func initAWS() { var err error s3c, err := s3.NewWithMaxRetries(100) if err != nil { @@ -50,6 +52,7 @@ func main() { fmt.Println("Send to SQS:", sqsUrl) } + initAWS() keys, err := s3Client.ListAll(bucketName, keyPrefix) if err != nil { log.Fatalf("error listing S3 objects: %s", err) diff --git a/go.mod b/go.mod index 9a67bd03..860c567e 100644 --- a/go.mod +++ b/go.mod @@ -30,4 +30,5 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 // indirect github.com/aws/smithy-go v1.20.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect ) diff --git a/go.sum b/go.sum index 1c2e1e7e..60306a5d 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/fdsn/dataselect.go b/internal/fdsn/dataselect.go index 6b24ad44..afc040be 100644 --- a/internal/fdsn/dataselect.go +++ b/internal/fdsn/dataselect.go @@ -5,7 +5,6 @@ import ( "bufio" "errors" "fmt" - "github.com/gorilla/schema" "io" "net/url" "reflect" @@ -13,6 +12,8 @@ import ( "strconv" "strings" "time" + + "github.com/gorilla/schema" ) var decoder = schema.NewDecoder() @@ -71,9 +72,10 @@ func init() { /* parses the time in text as per the FDSN spec. Pads text for parsing with time.RFC3339Nano. Accepted formats are (UTC): - YYYY-MM-DDTHH:MM:SS.ssssss - YYYY-MM-DDTHH:MM:SS - YYYY-MM-DD + + YYYY-MM-DDTHH:MM:SS.ssssss + YYYY-MM-DDTHH:MM:SS + YYYY-MM-DD Implements the encoding.TextUnmarshaler interface. */ diff --git a/internal/holdings/holdings_test.go b/internal/holdings/holdings_test.go index 8896ca1d..7a255b95 100644 --- a/internal/holdings/holdings_test.go +++ b/internal/holdings/holdings_test.go @@ -1,11 +1,12 @@ package holdings_test import ( - "github.com/GeoNet/fdsn/internal/holdings" "os" "reflect" "testing" "time" + + "github.com/GeoNet/fdsn/internal/holdings" ) type result struct { diff --git a/internal/valid/valid_test.go b/internal/valid/valid_test.go index ab8135b1..3b3785f1 100644 --- a/internal/valid/valid_test.go +++ b/internal/valid/valid_test.go @@ -1,11 +1,12 @@ package valid_test import ( - "github.com/GeoNet/fdsn/internal/valid" "net/http" "runtime" "strconv" "testing" + + "github.com/GeoNet/fdsn/internal/valid" ) var bad = &valid.Error{Code: http.StatusBadRequest} diff --git a/vendor/github.com/joho/godotenv/.gitignore b/vendor/github.com/joho/godotenv/.gitignore new file mode 100644 index 00000000..e43b0f98 --- /dev/null +++ b/vendor/github.com/joho/godotenv/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/vendor/github.com/joho/godotenv/LICENCE b/vendor/github.com/joho/godotenv/LICENCE new file mode 100644 index 00000000..e7ddd51b --- /dev/null +++ b/vendor/github.com/joho/godotenv/LICENCE @@ -0,0 +1,23 @@ +Copyright (c) 2013 John Barton + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/joho/godotenv/README.md b/vendor/github.com/joho/godotenv/README.md new file mode 100644 index 00000000..bfbe66a0 --- /dev/null +++ b/vendor/github.com/joho/godotenv/README.md @@ -0,0 +1,202 @@ +# GoDotEnv ![CI](https://github.com/joho/godotenv/workflows/CI/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/joho/godotenv)](https://goreportcard.com/report/github.com/joho/godotenv) + +A Go (golang) port of the Ruby [dotenv](https://github.com/bkeepers/dotenv) project (which loads env vars from a .env file). + +From the original Library: + +> Storing configuration in the environment is one of the tenets of a twelve-factor app. Anything that is likely to change between deployment environments–such as resource handles for databases or credentials for external services–should be extracted from the code into environment variables. +> +> But it is not always practical to set environment variables on development machines or continuous integration servers where multiple projects are run. Dotenv load variables from a .env file into ENV when the environment is bootstrapped. + +It can be used as a library (for loading in env for your own daemons etc.) or as a bin command. + +There is test coverage and CI for both linuxish and Windows environments, but I make no guarantees about the bin version working on Windows. + +## Installation + +As a library + +```shell +go get github.com/joho/godotenv +``` + +or if you want to use it as a bin command + +go >= 1.17 +```shell +go install github.com/joho/godotenv/cmd/godotenv@latest +``` + +go < 1.17 +```shell +go get github.com/joho/godotenv/cmd/godotenv +``` + +## Usage + +Add your application configuration to your `.env` file in the root of your project: + +```shell +S3_BUCKET=YOURS3BUCKET +SECRET_KEY=YOURSECRETKEYGOESHERE +``` + +Then in your Go app you can do something like + +```go +package main + +import ( + "log" + "os" + + "github.com/joho/godotenv" +) + +func main() { + err := godotenv.Load() + if err != nil { + log.Fatal("Error loading .env file") + } + + s3Bucket := os.Getenv("S3_BUCKET") + secretKey := os.Getenv("SECRET_KEY") + + // now do something with s3 or whatever +} +``` + +If you're even lazier than that, you can just take advantage of the autoload package which will read in `.env` on import + +```go +import _ "github.com/joho/godotenv/autoload" +``` + +While `.env` in the project root is the default, you don't have to be constrained, both examples below are 100% legit + +```go +godotenv.Load("somerandomfile") +godotenv.Load("filenumberone.env", "filenumbertwo.env") +``` + +If you want to be really fancy with your env file you can do comments and exports (below is a valid env file) + +```shell +# I am a comment and that is OK +SOME_VAR=someval +FOO=BAR # comments at line end are OK too +export BAR=BAZ +``` + +Or finally you can do YAML(ish) style + +```yaml +FOO: bar +BAR: baz +``` + +as a final aside, if you don't want godotenv munging your env you can just get a map back instead + +```go +var myEnv map[string]string +myEnv, err := godotenv.Read() + +s3Bucket := myEnv["S3_BUCKET"] +``` + +... or from an `io.Reader` instead of a local file + +```go +reader := getRemoteFile() +myEnv, err := godotenv.Parse(reader) +``` + +... or from a `string` if you so desire + +```go +content := getRemoteFileContent() +myEnv, err := godotenv.Unmarshal(content) +``` + +### Precedence & Conventions + +Existing envs take precedence of envs that are loaded later. + +The [convention](https://github.com/bkeepers/dotenv#what-other-env-files-can-i-use) +for managing multiple environments (i.e. development, test, production) +is to create an env named `{YOURAPP}_ENV` and load envs in this order: + +```go +env := os.Getenv("FOO_ENV") +if "" == env { + env = "development" +} + +godotenv.Load(".env." + env + ".local") +if "test" != env { + godotenv.Load(".env.local") +} +godotenv.Load(".env." + env) +godotenv.Load() // The Original .env +``` + +If you need to, you can also use `godotenv.Overload()` to defy this convention +and overwrite existing envs instead of only supplanting them. Use with caution. + +### Command Mode + +Assuming you've installed the command as above and you've got `$GOPATH/bin` in your `$PATH` + +``` +godotenv -f /some/path/to/.env some_command with some args +``` + +If you don't specify `-f` it will fall back on the default of loading `.env` in `PWD` + +By default, it won't override existing environment variables; you can do that with the `-o` flag. + +### Writing Env Files + +Godotenv can also write a map representing the environment to a correctly-formatted and escaped file + +```go +env, err := godotenv.Unmarshal("KEY=value") +err := godotenv.Write(env, "./.env") +``` + +... or to a string + +```go +env, err := godotenv.Unmarshal("KEY=value") +content, err := godotenv.Marshal(env) +``` + +## Contributing + +Contributions are welcome, but with some caveats. + +This library has been declared feature complete (see [#182](https://github.com/joho/godotenv/issues/182) for background) and will not be accepting issues or pull requests adding new functionality or breaking the library API. + +Contributions would be gladly accepted that: + +* bring this library's parsing into closer compatibility with the mainline dotenv implementations, in particular [Ruby's dotenv](https://github.com/bkeepers/dotenv) and [Node.js' dotenv](https://github.com/motdotla/dotenv) +* keep the library up to date with the go ecosystem (ie CI bumps, documentation changes, changes in the core libraries) +* bug fixes for use cases that pertain to the library's purpose of easing development of codebases deployed into twelve factor environments + +*code changes without tests and references to peer dotenv implementations will not be accepted* + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Added some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +## Releases + +Releases should follow [Semver](http://semver.org/) though the first couple of releases are `v1` and `v1.1`. + +Use [annotated tags for all releases](https://github.com/joho/godotenv/issues/30). Example `git tag -a v1.2.1` + +## Who? + +The original library [dotenv](https://github.com/bkeepers/dotenv) was written by [Brandon Keepers](http://opensoul.org/), and this port was done by [John Barton](https://johnbarton.co/) based off the tests/fixtures in the original library. diff --git a/vendor/github.com/joho/godotenv/godotenv.go b/vendor/github.com/joho/godotenv/godotenv.go new file mode 100644 index 00000000..61b0ebba --- /dev/null +++ b/vendor/github.com/joho/godotenv/godotenv.go @@ -0,0 +1,228 @@ +// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv) +// +// Examples/readme can be found on the GitHub page at https://github.com/joho/godotenv +// +// The TL;DR is that you make a .env file that looks something like +// +// SOME_ENV_VAR=somevalue +// +// and then in your go code you can call +// +// godotenv.Load() +// +// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR") +package godotenv + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strconv" + "strings" +) + +const doubleQuoteSpecialChars = "\\\n\r\"!$`" + +// Parse reads an env file from io.Reader, returning a map of keys and values. +func Parse(r io.Reader) (map[string]string, error) { + var buf bytes.Buffer + _, err := io.Copy(&buf, r) + if err != nil { + return nil, err + } + + return UnmarshalBytes(buf.Bytes()) +} + +// Load will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main). +// +// If you call Load without any args it will default to loading .env in the current path. +// +// You can otherwise tell it which files to load (there can be more than one) like: +// +// godotenv.Load("fileone", "filetwo") +// +// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults. +func Load(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, false) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Overload will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main). +// +// If you call Overload without any args it will default to loading .env in the current path. +// +// You can otherwise tell it which files to load (there can be more than one) like: +// +// godotenv.Overload("fileone", "filetwo") +// +// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefully set all vars. +func Overload(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, true) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Read all env (with same file loading semantics as Load) but return values as +// a map rather than automatically writing values into env +func Read(filenames ...string) (envMap map[string]string, err error) { + filenames = filenamesOrDefault(filenames) + envMap = make(map[string]string) + + for _, filename := range filenames { + individualEnvMap, individualErr := readFile(filename) + + if individualErr != nil { + err = individualErr + return // return early on a spazout + } + + for key, value := range individualEnvMap { + envMap[key] = value + } + } + + return +} + +// Unmarshal reads an env file from a string, returning a map of keys and values. +func Unmarshal(str string) (envMap map[string]string, err error) { + return UnmarshalBytes([]byte(str)) +} + +// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. +func UnmarshalBytes(src []byte) (map[string]string, error) { + out := make(map[string]string) + err := parseBytes(src, out) + + return out, err +} + +// Exec loads env vars from the specified filenames (empty map falls back to default) +// then executes the cmd specified. +// +// Simply hooks up os.Stdin/err/out to the command and calls Run(). +// +// If you want more fine grained control over your command it's recommended +// that you use `Load()`, `Overload()` or `Read()` and the `os/exec` package yourself. +func Exec(filenames []string, cmd string, cmdArgs []string, overload bool) error { + op := Load + if overload { + op = Overload + } + if err := op(filenames...); err != nil { + return err + } + + command := exec.Command(cmd, cmdArgs...) + command.Stdin = os.Stdin + command.Stdout = os.Stdout + command.Stderr = os.Stderr + return command.Run() +} + +// Write serializes the given environment and writes it to a file. +func Write(envMap map[string]string, filename string) error { + content, err := Marshal(envMap) + if err != nil { + return err + } + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + _, err = file.WriteString(content + "\n") + if err != nil { + return err + } + return file.Sync() +} + +// Marshal outputs the given environment as a dotenv-formatted environment file. +// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. +func Marshal(envMap map[string]string) (string, error) { + lines := make([]string, 0, len(envMap)) + for k, v := range envMap { + if d, err := strconv.Atoi(v); err == nil { + lines = append(lines, fmt.Sprintf(`%s=%d`, k, d)) + } else { + lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) + } + } + sort.Strings(lines) + return strings.Join(lines, "\n"), nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func loadFile(filename string, overload bool) error { + envMap, err := readFile(filename) + if err != nil { + return err + } + + currentEnv := map[string]bool{} + rawEnv := os.Environ() + for _, rawEnvLine := range rawEnv { + key := strings.Split(rawEnvLine, "=")[0] + currentEnv[key] = true + } + + for key, value := range envMap { + if !currentEnv[key] || overload { + _ = os.Setenv(key, value) + } + } + + return nil +} + +func readFile(filename string) (envMap map[string]string, err error) { + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + + return Parse(file) +} + +func doubleQuoteEscape(line string) string { + for _, c := range doubleQuoteSpecialChars { + toReplace := "\\" + string(c) + if c == '\n' { + toReplace = `\n` + } + if c == '\r' { + toReplace = `\r` + } + line = strings.Replace(line, string(c), toReplace, -1) + } + return line +} diff --git a/vendor/github.com/joho/godotenv/parser.go b/vendor/github.com/joho/godotenv/parser.go new file mode 100644 index 00000000..cc709af8 --- /dev/null +++ b/vendor/github.com/joho/godotenv/parser.go @@ -0,0 +1,271 @@ +package godotenv + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strings" + "unicode" +) + +const ( + charComment = '#' + prefixSingleQuote = '\'' + prefixDoubleQuote = '"' + + exportPrefix = "export" +) + +func parseBytes(src []byte, out map[string]string) error { + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) + cutset := src + for { + cutset = getStatementStart(cutset) + if cutset == nil { + // reached end of file + break + } + + key, left, err := locateKeyName(cutset) + if err != nil { + return err + } + + value, left, err := extractVarValue(left, out) + if err != nil { + return err + } + + out[key] = value + cutset = left + } + + return nil +} + +// getStatementPosition returns position of statement begin. +// +// It skips any comment line or non-whitespace character. +func getStatementStart(src []byte) []byte { + pos := indexOfNonSpaceChar(src) + if pos == -1 { + return nil + } + + src = src[pos:] + if src[0] != charComment { + return src + } + + // skip comment section + pos = bytes.IndexFunc(src, isCharFunc('\n')) + if pos == -1 { + return nil + } + + return getStatementStart(src[pos:]) +} + +// locateKeyName locates and parses key name and returns rest of slice +func locateKeyName(src []byte) (key string, cutset []byte, err error) { + // trim "export" and space at beginning + src = bytes.TrimLeftFunc(src, isSpace) + if bytes.HasPrefix(src, []byte(exportPrefix)) { + trimmed := bytes.TrimPrefix(src, []byte(exportPrefix)) + if bytes.IndexFunc(trimmed, isSpace) == 0 { + src = bytes.TrimLeftFunc(trimmed, isSpace) + } + } + + // locate key name end and validate it in single loop + offset := 0 +loop: + for i, char := range src { + rchar := rune(char) + if isSpace(rchar) { + continue + } + + switch char { + case '=', ':': + // library also supports yaml-style value declaration + key = string(src[0:i]) + offset = i + 1 + break loop + case '_': + default: + // variable name should match [A-Za-z0-9_.] + if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) || rchar == '.' { + continue + } + + return "", nil, fmt.Errorf( + `unexpected character %q in variable name near %q`, + string(char), string(src)) + } + } + + if len(src) == 0 { + return "", nil, errors.New("zero length string") + } + + // trim whitespace + key = strings.TrimRightFunc(key, unicode.IsSpace) + cutset = bytes.TrimLeftFunc(src[offset:], isSpace) + return key, cutset, nil +} + +// extractVarValue extracts variable value and returns rest of slice +func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) { + quote, hasPrefix := hasQuotePrefix(src) + if !hasPrefix { + // unquoted value - read until end of line + endOfLine := bytes.IndexFunc(src, isLineEnd) + + // Hit EOF without a trailing newline + if endOfLine == -1 { + endOfLine = len(src) + + if endOfLine == 0 { + return "", nil, nil + } + } + + // Convert line to rune away to do accurate countback of runes + line := []rune(string(src[0:endOfLine])) + + // Assume end of line is end of var + endOfVar := len(line) + if endOfVar == 0 { + return "", src[endOfLine:], nil + } + + // Work backwards to check if the line ends in whitespace then + // a comment (ie asdasd # some comment) + for i := endOfVar - 1; i >= 0; i-- { + if line[i] == charComment && i > 0 { + if isSpace(line[i-1]) { + endOfVar = i + break + } + } + } + + trimmed := strings.TrimFunc(string(line[0:endOfVar]), isSpace) + + return expandVariables(trimmed, vars), src[endOfLine:], nil + } + + // lookup quoted string terminator + for i := 1; i < len(src); i++ { + if char := src[i]; char != quote { + continue + } + + // skip escaped quote symbol (\" or \', depends on quote) + if prevChar := src[i-1]; prevChar == '\\' { + continue + } + + // trim quotes + trimFunc := isCharFunc(rune(quote)) + value = string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc)) + if quote == prefixDoubleQuote { + // unescape newlines for double quote (this is compat feature) + // and expand environment variables + value = expandVariables(expandEscapes(value), vars) + } + + return value, src[i+1:], nil + } + + // return formatted error if quoted string is not terminated + valEndIndex := bytes.IndexFunc(src, isCharFunc('\n')) + if valEndIndex == -1 { + valEndIndex = len(src) + } + + return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) +} + +func expandEscapes(str string) string { + out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return match + } + }) + return unescapeCharsRegex.ReplaceAllString(out, "$1") +} + +func indexOfNonSpaceChar(src []byte) int { + return bytes.IndexFunc(src, func(r rune) bool { + return !unicode.IsSpace(r) + }) +} + +// hasQuotePrefix reports whether charset starts with single or double quote and returns quote character +func hasQuotePrefix(src []byte) (prefix byte, isQuored bool) { + if len(src) == 0 { + return 0, false + } + + switch prefix := src[0]; prefix { + case prefixDoubleQuote, prefixSingleQuote: + return prefix, true + default: + return 0, false + } +} + +func isCharFunc(char rune) func(rune) bool { + return func(v rune) bool { + return v == char + } +} + +// isSpace reports whether the rune is a space character but not line break character +// +// this differs from unicode.IsSpace, which also applies line break as space +func isSpace(r rune) bool { + switch r { + case '\t', '\v', '\f', '\r', ' ', 0x85, 0xA0: + return true + } + return false +} + +func isLineEnd(r rune) bool { + if r == '\n' || r == '\r' { + return true + } + return false +} + +var ( + escapeRegex = regexp.MustCompile(`\\.`) + expandVarRegex = regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`) + unescapeCharsRegex = regexp.MustCompile(`\\([^$])`) +) + +func expandVariables(v string, m map[string]string) string { + return expandVarRegex.ReplaceAllStringFunc(v, func(s string) string { + submatch := expandVarRegex.FindStringSubmatch(s) + + if submatch == nil { + return s + } + if submatch[1] == "\\" || submatch[2] == "(" { + return submatch[0][1:] + } else if submatch[4] != "" { + return m[submatch[4]] + } + return s + }) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6377ee4c..471230b4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -148,6 +148,9 @@ github.com/gorilla/schema # github.com/jmespath/go-jmespath v0.4.0 ## explicit; go 1.14 github.com/jmespath/go-jmespath +# github.com/joho/godotenv v1.5.1 +## explicit; go 1.12 +github.com/joho/godotenv # github.com/lib/pq v1.10.3 ## explicit; go 1.13 github.com/lib/pq