diff --git a/README.md b/README.md index 0b440e05a..88f2a8d87 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,9 @@ This repo on its own isn't particularly interesting, until it is used to create We use git tags and [GitHub Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository). 1. Maintainers will push a new semver [tag](https://github.com/pulumi/pulumi-terraform-bridge/tags) when appropriate -1. Maintainers will then generate a Release with Changelog using [GitHub Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), using the tag pushed in the +1. Maintainers will then generate a Release with Changelog using [GitHub Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), using the tag pushed in the first step. -1. Finally, maintainers will roll out bridge updates across Pulumi providers via manually running the [bridge update +1. Finally, maintainers will roll out bridge updates across Pulumi providers via manually running the [bridge update workflow in CI Management](https://github.com/pulumi/ci-mgmt/actions/workflows/update-bridge-ecosystem-providers.yml). ### Adapting a New Terraform Provider @@ -106,3 +106,4 @@ tfgen, the command that generates Pulumi schema/code for a bridged provider supp * `PULUMI_SKIP_MISSING_MAPPING_ERROR`: If truthy, tfgen will not fail if a data source or resource in the TF provider is not mapped to the Pulumi provider. Instead, a warning is printed. Default is `false`. * `PULUMI_SKIP_EXTRA_MAPPING_ERROR`: If truthy, tfgen will not fail if a mapped data source or resource does not exist in the TF provider. Instead, warning is printed. Default is `false`. * `PULUMI_MISSING_DOCS_ERROR`: If truthy, tfgen will fail if docs cannot be found for a data source or resource. Default is `false`. +* `PULUMI_CONVERT`: If truthy, tfgen will shell out to `pulumi convert` for converting example code from TF HCL to Pulumi PCL diff --git a/pkg/tfgen/convert_cli.go b/pkg/tfgen/convert_cli.go new file mode 100644 index 000000000..c6398562b --- /dev/null +++ b/pkg/tfgen/convert_cli.go @@ -0,0 +1,468 @@ +// Copyright 2016-2023, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tfgen + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + + "github.com/hashicorp/hcl/v2" + + hcl2java "github.com/pulumi/pulumi-java/pkg/codegen/java" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge" + hcl2yaml "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/codegen" + hcl2dotnet "github.com/pulumi/pulumi/pkg/v3/codegen/dotnet" + hcl2go "github.com/pulumi/pulumi/pkg/v3/codegen/go" + "github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax" + hcl2nodejs "github.com/pulumi/pulumi/pkg/v3/codegen/nodejs" + "github.com/pulumi/pulumi/pkg/v3/codegen/pcl" + hcl2python "github.com/pulumi/pulumi/pkg/v3/codegen/python" + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" +) + +func cliConverterEnabled() bool { + return cmdutil.IsTruthy(os.Getenv("PULUMI_CONVERT")) +} + +// Integrates with `pulumi convert` command for converting TF examples. +// +// Pulumi CLI now supprts a handy `pulumi convert` command. This file implements integrating with +// this command for the purposes of initial conversion of Terraform examples into PCL language. This +// integration is preferable to linking the functionality in as it allows bridged providers to not +// build-depend on the TF converter. +// +// Note that once examples are converted to PCL, they continue to be processed with in-process +// target language specific generators to produce TypeScript, YAML, Python etc target code. +type cliConverter struct { + packageName string // name of the provider such as "gcp" + info tfbridge.ProviderInfo // provider declaration + pluginHost plugin.Host // the plugin host for PCL conversion + packageCache *pcl.PackageCache // the package cache for PCL conversion + + hcls map[string]struct{} // set of observed HCL snippets + + generator interface { + convertHCL(hcl, path, exampleTitle string, languages []string) (string, error) + convertExamplesInner( + docs string, + path examplePath, + stripSubsectionsWithErrors bool, + convertHCL func(hcl, path, exampleTitle string, + languages []string) (string, error), + ) string + } + + convertExamplesList []struct { + docs string + path examplePath + stripSubsectionsWithErrors bool + } + + currentPackageSpec *pschema.PackageSpec + + pcls map[string]translatedExample // translations indexed by HCL + opts []pcl.BindOption // options cache; do not set +} + +// Represents a partially converted example. PCL is the Pulumi dialect of HCL. +type translatedExample struct { + PCL string `json:"pcl"` + Diagnostics hcl.Diagnostics `json:"diagnostics"` +} + +// Get or create the cliConverter associated with the Generator. +func (g *Generator) cliConverter() *cliConverter { + if g.cliConverterState != nil { + return g.cliConverterState + } + g.cliConverterState = &cliConverter{ + generator: g, + hcls: map[string]struct{}{}, + info: g.info, + packageCache: g.packageCache, + packageName: string(g.pkg), + pluginHost: g.pluginHost, + pcls: map[string]translatedExample{}, + } + return g.cliConverterState +} + +// Instead of converting examples, detect HCL literals involved and record placeholders for later. +func (cc *cliConverter) StartConvertingExamples( + docs string, + path examplePath, + stripSubsectionsWithErrors bool, +) string { + // Record inner HCL conversions and discard the result. + cc.generator.convertExamplesInner(docs, path, stripSubsectionsWithErrors, cc.recordHCL) + // Record the convertExamples job for later. + e := struct { + docs string + path examplePath + stripSubsectionsWithErrors bool + }{ + docs: docs, + path: path, + stripSubsectionsWithErrors: stripSubsectionsWithErrors, + } + cc.convertExamplesList = append(cc.convertExamplesList, e) + // Return a placeholder referencing the convertExampleJob by position. + return fmt.Sprintf("{convertExamples:%d}", len(cc.convertExamplesList)-1) +} + +// Replace all convertExamples placeholders with actual values by rendering them. +func (cc *cliConverter) FinishConvertingExamples(p pschema.PackageSpec) pschema.PackageSpec { + // Remember partially constructed PackageSpec so that Convert can access it. + cc.currentPackageSpec = &p + + err := cc.bulkConvert() + contract.AssertNoErrorf(err, "bulk converting examples failed") + + bytes, err := json.Marshal(p) + contract.AssertNoErrorf(err, "json.Marshal failed on PackageSpec") + re := regexp.MustCompile("[{]convertExamples[:]([^}]+)[}]") + + // Convert all stubs populated by StartConvertingExamples. + fixedBytes := re.ReplaceAllFunc(bytes, func(match []byte) []byte { + groups := re.FindSubmatch(match) + i, err := strconv.Atoi(string(groups[1])) + contract.AssertNoErrorf(err, "strconv.Atoi") + ex := cc.convertExamplesList[i] + source := cc.generator.convertExamplesInner(ex.docs, ex.path, + ex.stripSubsectionsWithErrors, cc.generator.convertHCL) + // JSON-escaping to splice into JSON string literals. + bytes, err := json.Marshal(source) + contract.AssertNoErrorf(err, "json.Masrhal(sourceCode)") + return bytes[1 : len(bytes)-1] + }) + + var result pschema.PackageSpec + err = json.Unmarshal(fixedBytes, &result) + contract.AssertNoErrorf(err, "json.Unmarshal failed to recover PackageSpec") + return result +} + +// During FinishConvertingExamples pass, generator calls back into this function to continue +// PCL->lang translation from a pre-computed HCL->PCL translation table cc.pcls. +func (cc *cliConverter) Convert(hclCode string, lang string) (string, hcl.Diagnostics, error) { + example, ok := cc.pcls[hclCode] + // Cannot assert here because panics are still possible for some reason. + // Example: gcp:gameservices/gameServerCluster:GameServerCluster + // + // Something skips adding failing conversion diagnostics to cc.pcls when pre-converting. The + // end-user experience is not affected much, the above example does not regress. + if !ok { + return "", hcl.Diagnostics{}, fmt.Errorf("unexpected HCL snippet in Convert") + } + if example.Diagnostics.HasErrors() { + return "", example.Diagnostics, nil + } + source, diags, err := cc.convertPCL(cc.currentPackageSpec, example.PCL, lang) + return source, cc.removeFileName(diags).Extend(example.Diagnostics), err +} + +// Convert all observed HCL snippets from cc.hcls to PCL in one pass, populate cc.pcls. +func (cc *cliConverter) bulkConvert() error { + examples := map[string]string{} + n := 0 + for hcl := range cc.hcls { + fileName := fmt.Sprintf("e%d", n) + examples[fileName] = hcl + n++ + } + result, err := cc.convertViaPulumiCLI(examples, []struct { + name string + info tfbridge.ProviderInfo + }{ + { + name: cc.packageName, + info: cc.info, + }, + }) + if err != nil { + return err + } + for fileName, hcl := range examples { + r := result[fileName] + cc.pcls[hcl] = translatedExample{ + PCL: r.PCL, + Diagnostics: cc.removeFileName(r.Diagnostics), + } + } + return nil +} + +// Calls pulumi convert to bulk-convert examples. +// +// To facilitate high-throughput conversion, an `examples.json` protocol is employed to convert +// examples in batches. See pulumi/pulumi-converter-terraform#29 for where the support is +// introduced. +// +// Source examples are passed in as a map from ID to raw TF code. +// +// This may need to be coarse-grain parallelized to speed up larger providers at the cost of more +// memory, for example run 4 instances of `pulumi convert` on 25% of examples each. +func (*cliConverter) convertViaPulumiCLI( + examples map[string]string, + mappings []struct { + name string + info tfbridge.ProviderInfo + }, +) ( + output map[string]translatedExample, + finalError error, +) { + outDir, err := os.MkdirTemp("", "bridge-examples-output") + if err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to create a temp dir "+ + " bridge-examples-output: %w", err) + } + defer func() { + if err := os.RemoveAll(outDir); err != nil { + if finalError == nil { + finalError = fmt.Errorf("convertViaPulumiCLI: failed to clean up "+ + "temp bridge-examples-output dir: %w", err) + } + } + }() + + examplesJSON, err := os.CreateTemp("", "bridge-examples.json") + if err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to create a temp "+ + " bridge-examples.json file: %w", err) + } + defer func() { + if err := os.Remove(examplesJSON.Name()); err != nil { + if finalError == nil { + finalError = fmt.Errorf("convertViaPulumiCLI: failed to clean up "+ + "temp bridge-examples.json file: %w", err) + } + } + }() + + // Write example to bridge-examples.json. + examplesBytes, err := json.Marshal(examples) + if err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to marshal examples "+ + "to JSON: %w", err) + } + if err := os.WriteFile(examplesJSON.Name(), examplesBytes, 0600); err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to write a temp "+ + "bridge-examples.json file: %w", err) + } + + mappingsDir := filepath.Join(outDir, "mappings") + + mappingsFile := func(name string) string { + return filepath.Join(mappingsDir, fmt.Sprintf("%s.json", name)) + } + + // Prepare mappings folder if necessary. + if len(mappings) > 0 { + if err := os.MkdirAll(mappingsDir, 0755); err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to write "+ + "mappings folder: %w", err) + } + } + + // Write out mappings files if necessary. + for _, m := range mappings { + mpi := tfbridge.MarshalProviderInfo(&m.info) + bytes, err := json.Marshal(mpi) + if err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to write "+ + "mappings folder: %w", err) + } + mf := mappingsFile(m.name) + if err := os.WriteFile(mf, bytes, 0600); err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to write "+ + "mappings file: %w", err) + } + } + + pulumiPath, err := exec.LookPath("pulumi") + if err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: pulumi executalbe not "+ + "in PATH: %w", err) + } + + var mappingsArgs []string + for _, m := range mappings { + mappingsArgs = append(mappingsArgs, "--mappings", mappingsFile(m.name)) + } + + cmdArgs := []string{ + "convert", + "--from", "terraform", + "--language", "pcl", + "--out", outDir, + "--generate-only", + } + + cmdArgs = append(cmdArgs, mappingsArgs...) + cmdArgs = append(cmdArgs, "--", "--convert-examples", filepath.Base(examplesJSON.Name())) + + cmd := exec.Command(pulumiPath, cmdArgs...) + + cmd.Dir = filepath.Dir(examplesJSON.Name()) + + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: pulumi command failed: %w\n"+ + "Stdout:\n%s\n\n"+ + "Stderr:\n%s\n\n", + err, stdout.String(), stderr.String()) + } + + outputFile := filepath.Join(outDir, filepath.Base(examplesJSON.Name())) + + outputFileBytes, err := os.ReadFile(outputFile) + if err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to read output file: %w, "+ + "check if your Pulumi CLI version is recent enough to include pulumi-converter-terraform v1.0.9", + err) + } + + var result map[string]translatedExample + if err := json.Unmarshal(outputFileBytes, &result); err != nil { + return nil, fmt.Errorf("convertViaPulumiCLI: failed to unmarshal output "+ + "file: %w", err) + } + + return result, nil +} + +// Conversion from PCL to the target language still happens in-process temporarily, which is really +// unfortunate because it makes another plugin loader necessary. This should eventually also happen +// through pulumi convert, but it needs to have bulk interface enabled for every language. +func (cc *cliConverter) convertPCL( + spec *pschema.PackageSpec, + source string, + languageName string, +) (string, hcl.Diagnostics, error) { + pulumiParser := syntax.NewParser() + + err := pulumiParser.ParseFile(bytes.NewBufferString(source), "example.pp") + contract.AssertNoErrorf(err, "pulumiParser.ParseFile returned an error") + + var diagnostics hcl.Diagnostics + + diagnostics = append(diagnostics, pulumiParser.Diagnostics...) + if diagnostics.HasErrors() { + return "", diagnostics, nil + } + + if cc.opts == nil { + var opts []pcl.BindOption + opts = append(opts, pcl.AllowMissingProperties) + opts = append(opts, pcl.AllowMissingVariables) + opts = append(opts, pcl.SkipResourceTypechecking) + if cc.pluginHost != nil { + opts = append(opts, pcl.PluginHost(cc.pluginHost)) + loader := newLoader(cc.pluginHost) + opts = append(opts, pcl.Loader(loader)) + } + if cc.packageCache != nil { + opts = append(opts, pcl.Cache(cc.packageCache)) + } + cc.opts = opts + } + + program, programDiags, err := pcl.BindProgram(pulumiParser.Files, cc.opts...) + if err != nil { + return "", diagnostics, fmt.Errorf("pcl.BindProgram failed: %w", err) + } + + diagnostics = append(diagnostics, programDiags...) + if diagnostics.HasErrors() { + return "", diagnostics, nil + } + + var genDiags hcl.Diagnostics + var generatedFiles map[string][]byte + + switch languageName { + case "typescript": + generatedFiles, genDiags, err = hcl2nodejs.GenerateProgram(program) + diagnostics = append(diagnostics, genDiags...) + case "python": + generatedFiles, genDiags, err = hcl2python.GenerateProgram(program) + diagnostics = append(diagnostics, genDiags...) + case "csharp": + generatedFiles, genDiags, err = hcl2dotnet.GenerateProgram(program) + diagnostics = append(diagnostics, genDiags...) + case "go": + generatedFiles, genDiags, err = hcl2go.GenerateProgram(program) + diagnostics = append(diagnostics, genDiags...) + case "yaml": + generatedFiles, genDiags, err = hcl2yaml.GenerateProgram(program) + diagnostics = append(diagnostics, genDiags...) + case "java": + generatedFiles, genDiags, err = hcl2java.GenerateProgram(program) + diagnostics = append(diagnostics, genDiags...) + default: + err = fmt.Errorf("Unsupported language: %q", languageName) + } + if err != nil { + return "", diagnostics, fmt.Errorf("GenerateProgram failed: %w", err) + } + if len(generatedFiles) != 1 { + err := fmt.Errorf("expected 1 file to be generated, got %d", len(generatedFiles)) + return "", diagnostics, err + } + var key string + for n := range generatedFiles { + key = n + } + return string(generatedFiles[key]), diagnostics, nil +} + +// Act as a convertHCL stub that does not actually convert but spies on the literals involved. +func (cc *cliConverter) recordHCL( + hcl, path, exampleTitle string, languages []string, +) (string, error) { + h := cc.hcls + h[hcl] = struct{}{} + return "{convertHCL}", nil +} + +func (cc *cliConverter) removeFileName(diag hcl.Diagnostics) hcl.Diagnostics { + var out []*hcl.Diagnostic + for _, d := range diag { + if d == nil { + continue + } + copy := *d + if copy.Subject != nil { + copy.Subject.Filename = "" + } + if copy.Context != nil { + copy.Context.Filename = "" + } + out = append(out, ©) + } + return out +} diff --git a/pkg/tfgen/convert_cli_test.go b/pkg/tfgen/convert_cli_test.go new file mode 100644 index 000000000..37d45957a --- /dev/null +++ b/pkg/tfgen/convert_cli_test.go @@ -0,0 +1,178 @@ +// Copyright 2016-2023, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tfgen + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "testing" + + "encoding/json" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + bridgetesting "github.com/pulumi/pulumi-terraform-bridge/v3/internal/testing" + "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge" + sdkv2 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2" + pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema" + "github.com/pulumi/pulumi/sdk/v3/go/common/diag" + "github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors" +) + +func TestConvertViaPulumiCLI(t *testing.T) { + if runtime.GOOS == "windows" { + // Currently there is a test issue in CI/test setup: + // + // convertViaPulumiCLI: failed to clean up temp bridge-examples.json file: The + // process cannot access the file because it is being used by another process.. + t.Skipf("Skipping on Windows due to a test setup issue") + } + t.Setenv("PULUMI_CONVERT", "1") + + simpleResourceTF := ` +resource "simple_resource" "a_resource" { + input_one = "hello" + input_two = true +} + +output "some_output" { + value = simple_resource.a_resource.result +}` + + p := tfbridge.ProviderInfo{ + Name: "simple", + P: sdkv2.NewProvider(&schema.Provider{ + ResourcesMap: map[string]*schema.Resource{ + "simple_resource": { + Schema: map[string]*schema.Schema{}, + }, + }, + DataSourcesMap: map[string]*schema.Resource{ + "simple_data_source": { + Schema: map[string]*schema.Schema{}, + }, + }, + }), + Resources: map[string]*tfbridge.ResourceInfo{ + "simple_resource": { + Tok: "simple:index:resource", + Fields: map[string]*tfbridge.SchemaInfo{ + "input_one": { + Name: "renamedInput1", + }, + }, + Docs: &tfbridge.DocInfo{ + Markdown: []byte(fmt.Sprintf( + "Sample resource.\n## Example Usage\n\n"+ + "```hcl\n%s\n```\n\n##Extras\n\n", + simpleResourceTF, + )), + }, + }, + }, + DataSources: map[string]*tfbridge.DataSourceInfo{ + "simple_data_source": { + Tok: "simple:index:dataSource", + }, + }, + } + + simpleDataSourceTF := ` +data "simple_data_source" "a_data_source" { + input_one = "hello" + input_two = true +} + +output "some_output" { + value = data.simple_data_source.a_data_source.result +}` + + simpleResourceExpectPCL := `resource "aResource" "simple:index:resource" { + __logicalName = "a_resource" + renamedInput1 = "hello" + inputTwo = true +} + +output "someOutput" { + value = aResource.result +} +` + + simpleDataSourceExpectPCL := `aDataSource = invoke("simple:index:dataSource", { + inputOne = "hello" + inputTwo = true +}) + +output "someOutput" { + value = aDataSource.result +} +` + + t.Run("convertViaPulumiCLI", func(t *testing.T) { + cc := &cliConverter{} + out, err := cc.convertViaPulumiCLI(map[string]string{ + "example1": simpleResourceTF, + "example2": simpleDataSourceTF, + }, []struct { + name string + info tfbridge.ProviderInfo + }{{info: p, name: "simple"}}) + + require.NoError(t, err) + assert.Equal(t, 2, len(out)) + + assert.Equal(t, simpleResourceExpectPCL, out["example1"].PCL) + assert.Equal(t, simpleDataSourceExpectPCL, out["example2"].PCL) + + assert.Empty(t, out["example1"].Diagnostics) + assert.Empty(t, out["example2"].Diagnostics) + }) + + t.Run("GenerateSchema", func(t *testing.T) { + info := p + tempdir := t.TempDir() + fs := afero.NewBasePathFs(afero.NewOsFs(), tempdir) + + g, err := NewGenerator(GeneratorOptions{ + Package: info.Name, + Version: info.Version, + Language: Schema, + ProviderInfo: info, + Root: fs, + Sink: diag.DefaultSink(io.Discard, io.Discard, diag.FormatOptions{ + Color: colors.Never, + }), + }) + assert.NoError(t, err) + + err = g.Generate() + assert.NoError(t, err) + + d, err := os.ReadFile(filepath.Join(tempdir, "schema.json")) + assert.NoError(t, err) + + var schema pschema.PackageSpec + err = json.Unmarshal(d, &schema) + assert.NoError(t, err) + + bridgetesting.AssertEqualsJSONFile(t, + "test_data/TestConvertViaPulumiCLI/schema.json", schema) + }) +} diff --git a/pkg/tfgen/docs.go b/pkg/tfgen/docs.go index e04e53e19..d11cbcb63 100644 --- a/pkg/tfgen/docs.go +++ b/pkg/tfgen/docs.go @@ -30,6 +30,7 @@ import ( "unicode" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl/v2" bf "github.com/russross/blackfriday/v2" "github.com/spf13/afero" "golang.org/x/text/cases" @@ -1301,6 +1302,23 @@ func (g *Generator) convertExamples(docs string, path examplePath, stripSubsecti return fmt.Sprintf("{{%% examples %%}}\n%s\n{{%% /examples %%}}", docs) } + if cliConverterEnabled() { + return g.cliConverter().StartConvertingExamples(docs, path, + stripSubsectionsWithErrors) + } + + return g.convertExamplesInner(docs, path, stripSubsectionsWithErrors, g.convertHCL) +} + +// The inner implementation of examples conversion is parameterized by convertHCL so that it can be +// executed either normally or in symbolic mode. +func (g *Generator) convertExamplesInner( + docs string, + path examplePath, + stripSubsectionsWithErrors bool, + convertHCL func(hcl, path, exampleTitle string, languages []string) (string, error), +) (result string) { + output := &bytes.Buffer{} writeTrailingNewline := func(buf *bytes.Buffer) { @@ -1363,7 +1381,7 @@ func (g *Generator) convertExamples(docs string, path examplePath, stripSubsecti } langs := genLanguageToSlice(g.language) - codeBlock, err := g.convertHCL(hcl, path.String(), exampleTitle, langs) + codeBlock, err := convertHCL(hcl, path.String(), exampleTitle, langs) if err != nil { skippedExamples = true @@ -1510,18 +1528,45 @@ func (g *Generator) convert(input afero.Fs, languageName string) (files map[stri return } -// convertHCLToString hides the implementation details of the upstream implementation for HCL conversion and provides -// simplified parameters and return values -func (g *Generator) convertHCLToString(hcl, path, languageName string) (string, error) { +func (g *Generator) legacyConvert( + hclCode, fileName, languageName string, +) (string, hcl.Diagnostics, error) { input := afero.NewMemMapFs() - fileName := fmt.Sprintf("/%s.tf", strings.ReplaceAll(path, "/", "-")) f, err := input.Create(fileName) contract.AssertNoErrorf(err, "err != nil") - _, err = f.Write([]byte(hcl)) + _, err = f.Write([]byte(hclCode)) contract.AssertNoErrorf(err, "err != nil") contract.IgnoreClose(f) files, diags, err := g.convert(input, languageName) + if diags.All.HasErrors() || err != nil { + return "", diags.All, err + } + + contract.Assertf(len(files) == 1, `len(files) == 1`) + + convertedHcl := "" + for _, output := range files { + convertedHcl = strings.TrimSpace(string(output)) + } + return convertedHcl, diags.All, nil +} + +// convertHCLToString hides the implementation details of the upstream implementation for HCL conversion and provides +// simplified parameters and return values +func (g *Generator) convertHCLToString(hclCode, path, languageName string) (string, error) { + + fileName := fmt.Sprintf("/%s.tf", strings.ReplaceAll(path, "/", "-")) + + var convertedHcl string + var diags hcl.Diagnostics + var err error + + if cliConverterEnabled() { + convertedHcl, diags, err = g.cliConverter().Convert(hclCode, languageName) + } else { + convertedHcl, diags, err = g.legacyConvert(hclCode, fileName, languageName) + } // By observation on the GCP provider, convert.Convert() will either panic (in which case the wrapped method above // will return an error) or it will return a non-zero value for diags. @@ -1534,25 +1579,18 @@ func (g *Generator) convertHCLToString(hcl, path, languageName string) (string, } return "", fmt.Errorf("failed to convert HCL for %s to %v: %w", path, languageName, err) } - if diags.All.HasErrors() { + if diags.HasErrors() { // Remove the temp filename from the error, since it will be confusing to users of the bridge who do not know // we write an example to a temp file internally in order to pass to convert.Convert(). // // fileName starts with a "/" which is not present in the resulting error, so we need to skip the first rune. - errMsg := strings.ReplaceAll(diags.All.Error(), fileName[1:], "") + errMsg := strings.ReplaceAll(diags.Error(), fileName[1:], "") g.warn("failed to convert HCL for %s to %v: %v", path, languageName, errMsg) - g.coverageTracker.languageConversionFailure(languageName, diags.All) + g.coverageTracker.languageConversionFailure(languageName, diags) return "", fmt.Errorf(errMsg) } - contract.Assertf(len(files) == 1, `len(files) == 1`) - - convertedHcl := "" - for _, output := range files { - convertedHcl = strings.TrimSpace(string(output)) - } - g.coverageTracker.languageConversionSuccess(languageName, convertedHcl) return convertedHcl, nil } diff --git a/pkg/tfgen/generate.go b/pkg/tfgen/generate.go index 38b8010cd..e19ab93d7 100644 --- a/pkg/tfgen/generate.go +++ b/pkg/tfgen/generate.go @@ -79,6 +79,8 @@ type Generator struct { // Set if we can't find the docs repo and we have already printed a warning // message. noDocsRepo bool + + cliConverterState *cliConverter } type Language string diff --git a/pkg/tfgen/generate_schema.go b/pkg/tfgen/generate_schema.go index 65190af75..65d70340c 100644 --- a/pkg/tfgen/generate_schema.go +++ b/pkg/tfgen/generate_schema.go @@ -946,6 +946,11 @@ func (g *Generator) convertExamplesInSchema(spec pschema.PackageSpec) pschema.Pa path := newExamplePathForFunction(token) spec.Functions[token] = g.convertExamplesInFunctionSpec(path, function) } + + if cliConverterEnabled() { + return g.cliConverter().FinishConvertingExamples(spec) + } + return spec } diff --git a/pkg/tfgen/test_data/TestConvertViaPulumiCLI/schema.json b/pkg/tfgen/test_data/TestConvertViaPulumiCLI/schema.json new file mode 100644 index 000000000..8ea73a584 --- /dev/null +++ b/pkg/tfgen/test_data/TestConvertViaPulumiCLI/schema.json @@ -0,0 +1,49 @@ +{ + "name": "simple", + "attribution": "This Pulumi package is based on the [`simple` Terraform Provider](https://github.com/terraform-providers/terraform-provider-simple).", + "meta": { + "moduleFormat": "(.*)(?:/[^/]*)" + }, + "language": { + "nodejs": { + "readme": "\u003e This provider is a derived work of the [Terraform Provider](https://github.com/terraform-providers/terraform-provider-simple)\n\u003e distributed under [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). If you encounter a bug or missing feature,\n\u003e first check the [`pulumi-simple` repo](/issues); however, if that doesn't turn up anything,\n\u003e please consult the source [`terraform-provider-simple` repo](https://github.com/terraform-providers/terraform-provider-simple/issues).", + "compatibility": "tfbridge20", + "disableUnionOutputTypes": true + }, + "python": { + "readme": "\u003e This provider is a derived work of the [Terraform Provider](https://github.com/terraform-providers/terraform-provider-simple)\n\u003e distributed under [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/). If you encounter a bug or missing feature,\n\u003e first check the [`pulumi-simple` repo](/issues); however, if that doesn't turn up anything,\n\u003e please consult the source [`terraform-provider-simple` repo](https://github.com/terraform-providers/terraform-provider-simple/issues).", + "compatibility": "tfbridge20", + "pyproject": {} + } + }, + "config": {}, + "provider": { + "description": "The provider type for the simple package. By default, resources use package-wide configuration\nsettings, however an explicit `Provider` instance may be created and passed during resource\nconstruction to achieve fine-grained programmatic control over provider settings. See the\n[documentation](https://www.pulumi.com/docs/reference/programming-model/#providers) for more information.\n" + }, + "resources": { + "simple:index:resource": { + "description": "{{% examples %}}\n## Example Usage\n{{% example %}}\n\n```java\npackage generated_program;\n\nimport com.pulumi.Context;\nimport com.pulumi.Pulumi;\nimport com.pulumi.core.Output;\nimport com.pulumi.simple.resource;\nimport com.pulumi.simple.ResourceArgs;\nimport java.util.List;\nimport java.util.ArrayList;\nimport java.util.Map;\nimport java.io.File;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\n\npublic class App {\n public static void main(String[] args) {\n Pulumi.run(App::stack);\n }\n\n public static void stack(Context ctx) {\n var aResource = new Resource(\"aResource\", ResourceArgs.builder() \n .renamedInput1(\"hello\")\n .inputTwo(true)\n .build());\n\n ctx.export(\"someOutput\", aResource.result());\n }\n}\n```\n```yaml\nresources:\n a_resource:\n type: simple:resource\n properties:\n renamedInput1: hello\n inputTwo: true\noutputs:\n someOutput: ${a_resource.result}\n```\n\n##Extras\n{{% /example %}}\n{{% /examples %}}", + "stateInputs": { + "description": "Input properties used for looking up and filtering resource resources.\n", + "type": "object" + } + } + }, + "functions": { + "simple:index:dataSource": { + "outputs": { + "description": "A collection of values returned by dataSource.\n", + "properties": { + "id": { + "type": "string", + "description": "The provider-assigned unique ID for this managed resource.\n" + } + }, + "type": "object", + "required": [ + "id" + ] + } + } + } +}