Generate multiple collections (#73)

* [wip] add support for multiple collections

* tie generators in the source connector

* fix rawdata test

* rate limit test

* generate time as nanoseconds timestamp

* update connector SDK

* small fixes

* rename config parameters

* update readme

* move note about collections parameters

.github/workflows/lint.yml

@@ -12,9 +12,20 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version-file: 'go.mod'
+
+      # This step sets up the variable steps.golangci-lint-version.outputs.v
+      # to contain the version of golangci-lint (e.g. v1.54.2).
+      # The version is taken from go.mod.
+      - name: Golangci-lint version
+        id: golangci-lint-version
+        run: |
+          GOLANGCI_LINT_VERSION=$( go list -m -f '{{.Version}}' github.com/golangci/golangci-lint )
+          echo "v=$GOLANGCI_LINT_VERSION" >> "$GITHUB_OUTPUT"
+
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v4.0.0
+        uses: golangci/golangci-lint-action@v4
         with:
-          version: v1.51.2
+          version: ${{ steps.golangci-lint-version.outputs.v }}
+          skip-pkg-cache: true
           args: --timeout=2m

.github/workflows/build.yml → .github/workflows/test.yml

@@ -1,20 +1,20 @@
-name: build
+name: test
 
 on:
   push:
     branches: [ main ]
   pull_request:
 
 jobs:
-  build:
+  test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version-file: 'go.mod'
 
       - name: Test
         run: make test GOTEST_FLAGS="-v -count=1"

.github/workflows/validate-generated-files.yml

@@ -0,0 +1,23 @@
+name: validate-generated-files
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  validate-generated-files:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Check generated files
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin
+          make install-tools generate
+          git diff --exit-code --numstat

.golangci.yml

@@ -11,6 +11,9 @@ linters-settings:
     min-complexity: 20
   goconst:
     ignore-tests: true
+  gosec:
+    excludes:
+      - G404 # Insecure random number source (rand) - we use math/rand as we don't care about security in this context
   goheader:
     template-path: '.golangci.goheader.template'
     values:

Makefile

@@ -1,12 +1,23 @@
-.PHONY: build test
-
 VERSION=$(shell git describe --tags --dirty --always)
 
+.PHONY: build
 build:
 	go build -ldflags "-X 'github.com/conduitio/conduit-connector-generator.version=${VERSION}'" -o conduit-connector-generator cmd/connector/main.go
 
+.PHONY: test
 test:
 	go test $(GOTEST_FLAGS) -race ./...
 
+.PHONY: lint
 lint:
 	golangci-lint run
+
+.PHONY: generate
+generate:
+	go generate ./...
+
+.PHONY: install-tools
+install-tools:
+	@echo Installing tools from tools.go
+	@go list -e -f '{{ join .Imports "\n" }}' tools.go | xargs -I % go list -f "%@{{.Module.Version}}" % | xargs -tI % go install %
+	@go mod tidy