diff --git a/.github/actions/download-codegen/action.yml b/.github/actions/download-codegen/action.yml new file mode 100644 index 000000000..eebafb711 --- /dev/null +++ b/.github/actions/download-codegen/action.yml @@ -0,0 +1,17 @@ +name: Download the code generator binary +description: Downloads the code generator binary to `bin/`. + +runs: + using: "composite" + steps: + + - name: Download the code generator binary for random + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: pulumi-tfgen-random + path: ${{ github.workspace }}/bin + + - name: Ensure the code generator binary for random is executable + shell: bash + run: | + find ${{ github.workspace }} -name "pulumi-*-random" -print -exec chmod +x {} \; diff --git a/.github/actions/setup-tools/action.yml b/.github/actions/setup-tools/action.yml index 14cd282f7..75b82a9a7 100644 --- a/.github/actions/setup-tools/action.yml +++ b/.github/actions/setup-tools/action.yml @@ -47,7 +47,7 @@ runs: - name: Install Pulumi CLI if: inputs.tools == 'all' || contains(inputs.tools, 'pulumicli') - uses: pulumi/actions@c7fad9e2f0b79653172b36538b8b34b3c0291952 # v6 + uses: pulumi/actions@13b8b7177d6fb736766875dac9b78aab07bd785f # v6 with: pulumi-version: "dev" diff --git a/.github/workflows/build_sdk.yml b/.github/workflows/build_sdk.yml index 5564e77d4..fdd42004f 100644 --- a/.github/workflows/build_sdk.yml +++ b/.github/workflows/build_sdk.yml @@ -60,8 +60,8 @@ jobs: tools: pulumictl, pulumicli, ${{ matrix.language }} - name: Prepare local workspace run: make prepare_local_workspace - - name: Download tfgen - uses: ./.github/actions/download-tfgen + - name: Download codegen + uses: ./.github/actions/download-codegen - name: Update path run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH" - name: Restore makefile progress diff --git a/.github/workflows/prerequisites.yml b/.github/workflows/prerequisites.yml index 86f276693..2dcddab65 100644 --- a/.github/workflows/prerequisites.yml +++ b/.github/workflows/prerequisites.yml @@ -97,7 +97,7 @@ jobs: Maintainer note: consult the [runbook](https://github.com/pulumi/platform-providers-team/blob/main/playbooks/tf-provider-updating.md) for dealing with any breaking changes. - - name: Upload pulumi-tfgen-random + - name: Upload codegen binary for random uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: pulumi-tfgen-random diff --git a/Makefile b/Makefile index bc2469d2c..5c10aaa36 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ ORG := pulumi PROJECT := github.com/$(ORG)/pulumi-$(PACK) PROVIDER_PATH := provider/v4 VERSION_PATH := $(PROVIDER_PATH)/pkg/version.Version -TFGEN := pulumi-tfgen-$(PACK) +CODEGEN := pulumi-tfgen-$(PACK) PROVIDER := pulumi-resource-$(PACK) JAVA_GEN := pulumi-java-gen TESTPARALLELISM := 10 @@ -14,11 +14,17 @@ PULUMI_PROVIDER_BUILD_PARALLELISM ?= PULUMI_CONVERT := 1 PULUMI_MISSING_DOCS_ERROR := true +PULUMICTL_VERSION := v0.0.46 +PULUMICTL := $(shell which pulumictl || \ + (test ! -e $(WORKING_DIR)/bin/pulumictl && \ + GOPATH="$(WORKING_DIR)" go install "github.com/pulumi/pulumictl/cmd/pulumictl@$(PULUMICTL_VERSION)"; \ + echo "$(WORKING_DIR)/bin/puluimctl")) + # Override during CI using `make [TARGET] PROVIDER_VERSION=""` or by setting a PROVIDER_VERSION environment variable # Local & branch builds will just used this fixed default version unless specified PROVIDER_VERSION ?= 4.0.0-alpha.0+dev # Use this normalised version everywhere rather than the raw input to ensure consistency. -VERSION_GENERIC = $(shell pulumictl convert-version --language generic --version "$(PROVIDER_VERSION)") +VERSION_GENERIC = $(shell $(PULUMICTL) convert-version --language generic --version "$(PROVIDER_VERSION)") # Strips debug information from the provider binary to reduce its size and speed up builds LDFLAGS_STRIP_SYMBOLS=-s -w @@ -95,8 +101,8 @@ GEN_ENVS := PULUMI_HOME=$(GEN_PULUMI_HOME) PULUMI_CONVERT_EXAMPLES_CACHE_DIR=$(G generate_dotnet: .make/generate_dotnet build_dotnet: .make/build_dotnet .make/generate_dotnet: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_dotnet: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) dotnet --out sdk/dotnet/ +.make/generate_dotnet: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) dotnet --out sdk/dotnet/ cd sdk/dotnet/ && \ printf "module fake_dotnet_module // Exclude this directory from Go tools\n\ngo 1.17\n" > go.mod && \ echo "$(VERSION_GENERIC)" >version.txt @@ -109,8 +115,8 @@ build_dotnet: .make/build_dotnet generate_go: .make/generate_go build_go: .make/build_go .make/generate_go: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_go: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) go --out sdk/go/ +.make/generate_go: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) go --out sdk/go/ @touch $@ .make/build_go: .make/generate_go cd sdk && go list "$$(grep -e "^module" go.mod | cut -d ' ' -f 2)/go/..." | xargs -I {} bash -c 'go build {} && go clean -i {}' @@ -136,8 +142,8 @@ build_java: .make/build_java generate_nodejs: .make/generate_nodejs build_nodejs: .make/build_nodejs .make/generate_nodejs: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_nodejs: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) nodejs --out sdk/nodejs/ +.make/generate_nodejs: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) nodejs --out sdk/nodejs/ printf "module fake_nodejs_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/nodejs/go.mod @touch $@ .make/build_nodejs: .make/generate_nodejs @@ -151,8 +157,8 @@ build_nodejs: .make/build_nodejs generate_python: .make/generate_python build_python: .make/build_python .make/generate_python: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) -.make/generate_python: .make/install_plugins bin/$(TFGEN) - $(GEN_ENVS) $(WORKING_DIR)/bin/$(TFGEN) python --out sdk/python/ +.make/generate_python: .make/install_plugins bin/$(CODEGEN) + $(GEN_ENVS) $(WORKING_DIR)/bin/$(CODEGEN) python --out sdk/python/ printf "module fake_python_module // Exclude this directory from Go tools\n\ngo 1.17\n" > sdk/python/go.mod cp README.md sdk/python/ @touch $@ @@ -168,8 +174,8 @@ build_python: .make/build_python .PHONY: generate_python build_python # Run the bridge's registry-docs command to generated the content of the installation docs/ folder at provider repo root build_registry_docs: .make/build_registry_docs -.make/build_registry_docs: .make/install_plugins bin/$(TFGEN) - bin/$(TFGEN) registry-docs --out $(WORKING_DIR)/docs +.make/build_registry_docs: .make/install_plugins bin/$(CODEGEN) + bin/$(CODEGEN) registry-docs --out $(WORKING_DIR)/docs @touch $@ .PHONY: build_registry_docs @@ -199,7 +205,7 @@ install_nodejs_sdk: .make/install_nodejs_sdk install_python_sdk: .PHONY: install_dotnet_sdk install_go_sdk install_java_sdk install_nodejs_sdk install_python_sdk -# Install Pulumi plugins required for TFGen to resolve references +# Install Pulumi plugins required for CODEGEN to resolve references install_plugins: .make/install_plugins .make/install_plugins: export PULUMI_HOME := $(WORKING_DIR)/.pulumi .make/install_plugins: export PATH := $(WORKING_DIR)/.pulumi/bin:$(PATH) @@ -222,12 +228,12 @@ lint_provider.fix: # `make provider_no_deps` builds the provider binary directly, without ensuring that # `cmd/pulumi-resource-random/schema.json` is valid and up to date. # To create a release ready binary, you should use `make provider`. -build_provider_cmd = cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(PROVIDER) -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER) +build_provider_cmd = cd provider && CGO_ENABLED=0 go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o "$(1)" -ldflags "$(LDFLAGS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER) provider: bin/$(PROVIDER) provider_no_deps: - $(call build_provider_cmd) + $(call build_provider_cmd,$(WORKING_DIR)/bin/$(PROVIDER)) bin/$(PROVIDER): .make/schema - $(call build_provider_cmd) + $(call build_provider_cmd,$(WORKING_DIR)/bin/$(PROVIDER)) .PHONY: provider provider_no_deps test: export PATH := $(WORKING_DIR)/bin:$(PATH) @@ -253,13 +259,13 @@ tfgen_no_deps: .make/schema .make/schema: export PULUMI_CONVERT_EXAMPLES_CACHE_DIR := $(WORKING_DIR)/.pulumi/examples-cache .make/schema: export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := $(PULUMI_CONVERT) .make/schema: export PULUMI_MISSING_DOCS_ERROR := $(PULUMI_MISSING_DOCS_ERROR) -.make/schema: bin/$(TFGEN) .make/install_plugins .make/upstream - $(WORKING_DIR)/bin/$(TFGEN) schema --out provider/cmd/$(PROVIDER) +.make/schema: bin/$(CODEGEN) .make/install_plugins .make/upstream + $(WORKING_DIR)/bin/$(CODEGEN) schema --out provider/cmd/$(PROVIDER) (cd provider && VERSION=$(VERSION_GENERIC) go generate cmd/$(PROVIDER)/main.go) @touch $@ -tfgen_build_only: bin/$(TFGEN) -bin/$(TFGEN): provider/*.go provider/go.* .make/upstream - (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(TFGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(TFGEN)) +tfgen_build_only: bin/$(CODEGEN) +bin/$(CODEGEN): provider/*.go provider/go.* .make/upstream + (cd provider && go build $(PULUMI_PROVIDER_BUILD_PARALLELISM) -o $(WORKING_DIR)/bin/$(CODEGEN) -ldflags "$(LDFLAGS_PROJ_VERSION) $(LDFLAGS_EXTRAS)" $(PROJECT)/$(PROVIDER_PATH)/cmd/$(CODEGEN)) .PHONY: tfgen schema tfgen_no_deps tfgen_build_only # Apply patches to the upstream submodule, if it exists @@ -273,7 +279,7 @@ endif .PHONY: upstream bin/pulumi-java-gen: .pulumi-java-gen.version - pulumictl download-binary -n pulumi-language-java -v v$(shell cat .pulumi-java-gen.version) -r pulumi/pulumi-java + $(PULUMICTL) download-binary -n pulumi-language-java -v v$(shell cat .pulumi-java-gen.version) -r pulumi/pulumi-java # To make an immediately observable change to .ci-mgmt.yaml: # @@ -305,7 +311,7 @@ ci-mgmt: .ci-mgmt.yaml # Start debug server for tfgen debug_tfgen: - dlv --listen=:2345 --headless=true --api-version=2 exec $(WORKING_DIR)/bin/$(TFGEN) -- schema --out provider/cmd/$(PROVIDER) + dlv --listen=:2345 --headless=true --api-version=2 exec $(WORKING_DIR)/bin/$(CODEGEN) -- schema --out provider/cmd/$(PROVIDER) .PHONY: debug_tfgen # Provider cross-platform build & packaging @@ -319,25 +325,24 @@ SKIP_SIGNING ?= # These targets assume that the schema-embed.json exists - it's generated by tfgen. # We disable CGO to ensure that the binary is statically linked. -bin/linux-amd64/$(PROVIDER): TARGET := linux-amd64 -bin/linux-arm64/$(PROVIDER): TARGET := linux-arm64 -bin/darwin-amd64/$(PROVIDER): TARGET := darwin-amd64 -bin/darwin-arm64/$(PROVIDER): TARGET := darwin-arm64 -bin/windows-amd64/$(PROVIDER).exe: TARGET := windows-amd64 +bin/linux-amd64/$(PROVIDER): export GOOS := linux +bin/linux-amd64/$(PROVIDER): export GOARCH := amd64 +bin/linux-arm64/$(PROVIDER): export GOOS := linux +bin/linux-arm64/$(PROVIDER): export GOARCH := arm64 +bin/darwin-amd64/$(PROVIDER): export GOOS := darwin +bin/darwin-amd64/$(PROVIDER): export GOARCH := amd64 +bin/darwin-arm64/$(PROVIDER): export GOOS := darwin +bin/darwin-arm64/$(PROVIDER): export GOARCH := arm64 +bin/windows-amd64/$(PROVIDER).exe: export GOOS := windows +bin/windows-amd64/$(PROVIDER).exe: export GOARCH := amd64 bin/%/$(PROVIDER) bin/%/$(PROVIDER).exe: bin/jsign-6.0.jar - @# check the TARGET is set - @test $(TARGET) - @cd provider && \ - export GOOS=$$(echo "$(TARGET)" | cut -d "-" -f 1) && \ - export GOARCH=$$(echo "$(TARGET)" | cut -d "-" -f 2) && \ - export CGO_ENABLED=0 && \ - go build -o "${WORKING_DIR}/$@" $(PULUMI_PROVIDER_BUILD_PARALLELISM) -ldflags "$(LDFLAGS)" "$(PROJECT)/$(PROVIDER_PATH)/cmd/$(PROVIDER)" + $(call build_provider_cmd,$(WORKING_DIR)/$@) @# Only sign windows binary if fully configured. @# Test variables set by joining with | between and looking for || showing at least one variable is empty. @# Move the binary to a temporary location and sign it there to avoid the target being up-to-date if signing fails. @set -e; \ - if [[ "${TARGET}" = "windows-amd64" && "${SKIP_SIGNING}" != "true" ]]; then \ + if [[ "${GOOS}-${GOARCH}" = "windows-amd64" && "${SKIP_SIGNING}" != "true" ]]; then \ if [[ "|${AZURE_SIGNING_CLIENT_ID}|${AZURE_SIGNING_CLIENT_SECRET}|${AZURE_SIGNING_TENANT_ID}|${AZURE_SIGNING_KEY_VAULT_URI}|" == *"||"* ]]; then \ echo "Can't sign windows binaries as required configuration not set: AZURE_SIGNING_CLIENT_ID, AZURE_SIGNING_CLIENT_SECRET, AZURE_SIGNING_TENANT_ID, AZURE_SIGNING_KEY_VAULT_URI"; \ echo "To rebuild with signing delete the unsigned $@ and rebuild with the fixed configuration"; \