From ceeb0e7bc29c241d9d6a02b75866e423dc0a473b Mon Sep 17 00:00:00 2001 From: Anton Tayanovskyy Date: Thu, 5 Oct 2023 20:45:28 -0400 Subject: [PATCH] Template PULUMI_CONVERT rollout (#655) To be reviewed with https://github.com/pulumi/pulumi-azuread/pull/484 Part of https://github.com/pulumi/pulumi-terraform-bridge/issues/1280 As a pre-requisite to PULUMI_CONVERT being enabled, it's necessary to pin Pulumi CLI so that examples continue to be deterministic. This PR borrows targets from pulumi-azure-native that perform CLI pinning. ## Updates ### CLI Unlike azure-native, it assumes that the CLI version we want to use to convert examples is the same as pulumi/pkg version we want to use to build this provider. This removes a degree of freedom but buys automatic upgrades - the existing mechanisms that update pulumi/pkg (in upgrade-provider) should also update this Pulumi CLI version. ### Converter This is not enforced here but https://github.com/pulumi/pulumi-azuread/pull/484 adds TF converter pinning to the CI-mgmt plugins section. This PR makes changes to allow opting into converter and not just resource plugins. There is currently no mechanism to upgrade this, but presumably upgrade-provider can build mechanisms to upgrade all plugins referenced in this config file. Any such upgrade is likely to trigger the need to regenerate SDKs. --- provider-ci/Makefile | 3 +-- .../pkg/templates/bridged-provider/Makefile | 20 +++++++++++--- provider-ci/test-workflows/aws/Makefile | 26 ++++++++++++------- .../test-workflows/cloudflare/Makefile | 18 +++++++++---- provider-ci/test-workflows/docker/Makefile | 16 +++++++++--- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/provider-ci/Makefile b/provider-ci/Makefile index 1a36462a82..e89bbc1fca 100644 --- a/provider-ci/Makefile +++ b/provider-ci/Makefile @@ -50,8 +50,7 @@ lint-providers/%/repo: $(ACTIONLINT) providers/%/repo cd providers/$*/repo && git init && ../../../$(ACTIONLINT) -config-file ../../../actionlint.yml && rm -rf .git @scripts/shellcheck.sh providers/$*/repo -LINT_RULES := $(addprefix lint-, $(PROVIDER_REPOS)) -lint-providers: $(LINT_RULES) +lint-providers: lint-providers/aws/repo lint-providers/docker/repo lint-providers/cloudflare/repo test-workflows/%: providers/%/repo rm -r test-workflows/$* diff --git a/provider-ci/internal/pkg/templates/bridged-provider/Makefile b/provider-ci/internal/pkg/templates/bridged-provider/Makefile index 990a43bbca..798c640d5c 100644 --- a/provider-ci/internal/pkg/templates/bridged-provider/Makefile +++ b/provider-ci/internal/pkg/templates/bridged-provider/Makefile @@ -19,6 +19,11 @@ WORKING_DIR := $(shell pwd) #{{- if .Config.goBuildParallelism }}# PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p #{{ .Config.goBuildParallelism }}# #{{- end }}# +#{{- if .Config.pulumiConvert }}# +PULUMI_CONVERT := 1 +#{{- else }}# +PULUMI_CONVERT := 0 +#{{- end }}# development: install_plugins provider build_sdks install_sdks @@ -106,10 +111,9 @@ install_dotnet_sdk: install_nodejs_sdk: yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin -install_plugins: - [ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh +install_plugins: .pulumi/bin/pulumi #{{- range .Config.plugins }}# - pulumi plugin install resource #{{ .name }}# #{{ .version }}# + .pulumi/bin/pulumi plugin install #{{ or .kind "resource" }}# #{{ .name }}# #{{ .version }}# #{{- end }}# lint_provider: provider @@ -123,7 +127,7 @@ test: tfgen: install_plugins#{{ if eq .Config.team "ecosystem" }}# upstream#{{ end }}##{{ if .Config.docsCmd }}# docs#{{ end }}# (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) - $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) + PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go) #{{- if eq .Config.team "ecosystem" }}# @@ -167,5 +171,13 @@ ci-mgmt: .ci-mgmt.yaml --template bridged-provider \ --config $< +.pulumi/bin/pulumi: HOME := $(WORKING_DIR) +.pulumi/bin/pulumi: .pulumi/version + curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version) + +# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider. +.pulumi/version: + @mkdir -p .pulumi + @cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@ .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup#{{ if .Config.docsCmd }}# docs#{{end}}# help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen#{{ if eq .Config.team "ecosystem" }}# upstream upstream.finalize upstream.rebase#{{end}}# ci-mgmt diff --git a/provider-ci/test-workflows/aws/Makefile b/provider-ci/test-workflows/aws/Makefile index bbca5cee01..d998bdc0dc 100644 --- a/provider-ci/test-workflows/aws/Makefile +++ b/provider-ci/test-workflows/aws/Makefile @@ -13,6 +13,7 @@ JAVA_GEN_VERSION := v0.9.5 TESTPARALLELISM := 10 WORKING_DIR := $(shell pwd) PULUMI_PROVIDER_BUILD_PARALLELISM ?= -p 2 +PULUMI_CONVERT := 0 development: install_plugins provider build_sdks install_sdks @@ -94,14 +95,13 @@ install_dotnet_sdk: install_nodejs_sdk: yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin -install_plugins: - [ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh - pulumi plugin install resource archive 0.0.1 - pulumi plugin install resource tls 4.10.0 - pulumi plugin install resource github 4.10.0 - pulumi plugin install resource kubernetes 3.17.0 - pulumi plugin install resource random 4.8.2 - pulumi plugin install resource github 5.14.0 +install_plugins: .pulumi/bin/pulumi + .pulumi/bin/pulumi plugin install resource archive 0.0.1 + .pulumi/bin/pulumi plugin install resource tls 4.10.0 + .pulumi/bin/pulumi plugin install resource github 4.10.0 + .pulumi/bin/pulumi plugin install resource kubernetes 3.17.0 + .pulumi/bin/pulumi plugin install resource random 4.8.2 + .pulumi/bin/pulumi plugin install resource github 5.14.0 lint_provider: provider cd provider && golangci-lint run -c ../.golangci.yml @@ -114,7 +114,7 @@ test: tfgen: install_plugins upstream (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) - $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) + PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go) upstream: @@ -151,5 +151,13 @@ ci-mgmt: .ci-mgmt.yaml --template bridged-provider \ --config $< +.pulumi/bin/pulumi: HOME := $(WORKING_DIR) +.pulumi/bin/pulumi: .pulumi/version + curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version) + +# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider. +.pulumi/version: + @mkdir -p .pulumi + @cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@ .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase ci-mgmt diff --git a/provider-ci/test-workflows/cloudflare/Makefile b/provider-ci/test-workflows/cloudflare/Makefile index f93f5b0a0f..b3d4b78b6c 100644 --- a/provider-ci/test-workflows/cloudflare/Makefile +++ b/provider-ci/test-workflows/cloudflare/Makefile @@ -12,6 +12,7 @@ JAVA_GEN := pulumi-java-gen JAVA_GEN_VERSION := v0.9.3 TESTPARALLELISM := 10 WORKING_DIR := $(shell pwd) +PULUMI_CONVERT := 0 development: install_plugins provider build_sdks install_sdks @@ -93,10 +94,9 @@ install_dotnet_sdk: install_nodejs_sdk: yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin -install_plugins: - [ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh - pulumi plugin install resource gcp 5.0.0 - pulumi plugin install resource tls 4.0.0 +install_plugins: .pulumi/bin/pulumi + .pulumi/bin/pulumi plugin install resource gcp 5.0.0 + .pulumi/bin/pulumi plugin install resource tls 4.0.0 lint_provider: provider cd provider && golangci-lint run -c ../.golangci.yml @@ -109,7 +109,7 @@ test: tfgen: install_plugins upstream (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) - $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) + PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go) upstream: @@ -139,5 +139,13 @@ ci-mgmt: .ci-mgmt.yaml --template bridged-provider \ --config $< +.pulumi/bin/pulumi: HOME := $(WORKING_DIR) +.pulumi/bin/pulumi: .pulumi/version + curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version) + +# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider. +.pulumi/version: + @mkdir -p .pulumi + @cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@ .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase ci-mgmt diff --git a/provider-ci/test-workflows/docker/Makefile b/provider-ci/test-workflows/docker/Makefile index 47b1bda0df..736a809daa 100644 --- a/provider-ci/test-workflows/docker/Makefile +++ b/provider-ci/test-workflows/docker/Makefile @@ -12,6 +12,7 @@ JAVA_GEN := pulumi-java-gen JAVA_GEN_VERSION := v0.9.7 TESTPARALLELISM := 10 WORKING_DIR := $(shell pwd) +PULUMI_CONVERT := 0 development: install_plugins provider build_sdks install_sdks @@ -96,9 +97,8 @@ install_dotnet_sdk: install_nodejs_sdk: yarn link --cwd $(WORKING_DIR)/sdk/nodejs/bin -install_plugins: - [ -x "$(shell command -v pulumi 2>/dev/null)" ] || curl -fsSL https://get.pulumi.com | sh - pulumi plugin install resource aws 5.37.0 +install_plugins: .pulumi/bin/pulumi + .pulumi/bin/pulumi plugin install resource aws 5.37.0 lint_provider: provider cd provider && golangci-lint run -c ../.golangci.yml @@ -111,7 +111,7 @@ test: tfgen: install_plugins upstream docs (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "-X $(PROJECT)/$(VERSION_PATH)=$(VERSION)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) - $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) + PATH=${PWD}/.pulumi/bin:$$PATH PULUMI_CONVERT=$(PULUMI_CONVERT) $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION) go generate cmd/$(PROVIDER)/main.go) upstream: @@ -141,5 +141,13 @@ ci-mgmt: .ci-mgmt.yaml --template bridged-provider \ --config $< +.pulumi/bin/pulumi: HOME := $(WORKING_DIR) +.pulumi/bin/pulumi: .pulumi/version + curl -fsSL https://get.pulumi.com | sh -s -- --version $(cat .pulumi/version) + +# Compute the version of Pulumi to use by inspecting the Go dependencies of the provider. +.pulumi/version: + @mkdir -p .pulumi + @cd provider && go list -f "{{slice .Version 1}}" -m github.com/pulumi/pulumi/pkg/v3 | tee ../$@ .PHONY: development build build_sdks install_go_sdk install_java_sdk install_python_sdk install_sdks only_build build_dotnet build_go build_java build_nodejs build_python clean cleanup docs help install_dotnet_sdk install_nodejs_sdk install_plugins lint_provider provider test tfgen upstream upstream.finalize upstream.rebase ci-mgmt