From c7b6a1880fe80b439055593e0b083bfcce339347 Mon Sep 17 00:00:00 2001 From: Christian Lechner Date: Thu, 6 Feb 2025 08:38:44 +0100 Subject: [PATCH] feat: Support OpenTofu (#245) --- .../default-opentofu/devcontainer.json | 30 +++++++ .devcontainer/{ => default}/devcontainer.json | 2 +- .../withenvfile-opentofu/devcontainer.json | 37 ++++++++ .devcontainer/withenvfile/devcontainer.json | 2 +- .wordlist.txt | 3 + docs/concepts.md | 8 +- docs/index.md | 6 +- docs/install.md | 6 +- docs/limitations.md | 2 +- docs/prerequisites.md | 12 ++- pkg/tfutils/tfConfig.go | 10 +-- pkg/tfutils/tfImport.go | 41 ++------- pkg/tfutils/tfWrapper.go | 85 ++++++++++++++++++ pkg/tfutils/types_tfState.go | 90 +++++++++++++++++++ sonar-project.properties | 2 +- 15 files changed, 279 insertions(+), 57 deletions(-) create mode 100644 .devcontainer/default-opentofu/devcontainer.json rename .devcontainer/{ => default}/devcontainer.json (92%) create mode 100644 .devcontainer/withenvfile-opentofu/devcontainer.json create mode 100644 pkg/tfutils/tfWrapper.go create mode 100644 pkg/tfutils/types_tfState.go diff --git a/.devcontainer/default-opentofu/devcontainer.json b/.devcontainer/default-opentofu/devcontainer.json new file mode 100644 index 0000000..8c25a24 --- /dev/null +++ b/.devcontainer/default-opentofu/devcontainer.json @@ -0,0 +1,30 @@ +{ + "name": "Terraform exporter for SAP BTP - Dev OpenTofu", + //https://mcr.microsoft.com/en-us/product/devcontainers/go/tags + "image": "mcr.microsoft.com/devcontainers/go:1.23-bullseye", + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/robbert229/devcontainer-features/opentofu:1": { + "version": "1.9.0" + }, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + "customizations": { + "vscode": { + "settings": {}, + "extensions": [ + "golang.go", + "gamunu.opentofu", + "ms-vscode.makefile-tools" + ] + }, + "codespaces": {} + }, + // If you want to use SSO in the devcontainer, you must install the xdg-utils package (see .devcontainer/scripts/install-xdg.sh). + "hostRequirements": { + "memory": "4gb" + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + "remoteUser": "vscode" +} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/default/devcontainer.json similarity index 92% rename from .devcontainer/devcontainer.json rename to .devcontainer/default/devcontainer.json index 0fe6c28..4532b94 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/default/devcontainer.json @@ -1,5 +1,5 @@ { - "name": "Terraform exporter for SAP BTP - Development", + "name": "Terraform exporter for SAP BTP - Dev", //https://mcr.microsoft.com/en-us/product/devcontainers/go/tags "image": "mcr.microsoft.com/devcontainers/go:1.23-bullseye", // Features to add to the dev container. More info: https://containers.dev/features. diff --git a/.devcontainer/withenvfile-opentofu/devcontainer.json b/.devcontainer/withenvfile-opentofu/devcontainer.json new file mode 100644 index 0000000..a14f340 --- /dev/null +++ b/.devcontainer/withenvfile-opentofu/devcontainer.json @@ -0,0 +1,37 @@ +{ + "name": "Terraform exporter for SAP BTP - Dev OpenTofu (with env file)", + "image": "mcr.microsoft.com/devcontainers/go:1.23-bullseye", + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/robbert229/devcontainer-features/opentofu:1": { + "version": "1.9.0" + }, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + "customizations": { + "vscode": { + "settings": {}, + "extensions": [ + "golang.go", + "gamunu.opentofu", + "ms-vscode.makefile-tools" + ] + }, + "codespaces": {} + }, + // If you want to use SSO in the devcontainer, you must install the xdg-utils package (see .devcontainer/scripts/install-xdg.sh). + "hostRequirements": { + "memory": "4gb" + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + "remoteUser": "vscode", + // This devcontainer expects a file named .devcontainer/devcontainer.env to exist. + // you should place the following environment variables in that file: + // - BTP_USERNAME + // - BTP_PASSWORD + "runArgs": [ + "--env-file", + ".devcontainer/devcontainer.env" + ] +} diff --git a/.devcontainer/withenvfile/devcontainer.json b/.devcontainer/withenvfile/devcontainer.json index 72629b6..dab296f 100644 --- a/.devcontainer/withenvfile/devcontainer.json +++ b/.devcontainer/withenvfile/devcontainer.json @@ -1,5 +1,5 @@ { - "name": "Terraform exporter for SAP BTP - Development (with env file)", + "name": "Terraform exporter for SAP BTP - Dev (with env file)", "image": "mcr.microsoft.com/devcontainers/go:1.23-bullseye", // Features to add to the dev container. More info: https://containers.dev/features. "features": { diff --git a/.wordlist.txt b/.wordlist.txt index 9cc7055..2082f48 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -8,6 +8,7 @@ btptf btpResources CLA CLI +CLIs CodeQL config Copilot @@ -15,6 +16,7 @@ customizations DCO dependabot devcontainer +devcontainers dir faq Github @@ -38,6 +40,7 @@ md NextSteps OAuth OpenSSF +OpenTofu PEM Pre pre diff --git a/docs/concepts.md b/docs/concepts.md index 799b383..bd415fd 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -1,15 +1,15 @@ # Concepts -The Terraform Exporter for SAP BTP (btptf CLI) provides a convenience functionality to import existing subaccounts on SAP BTP into Terraform configurations. The configurations delivered by the btptf CLI are: +The Terraform Exporter for SAP BTP (btptf CLI) provides a convenience functionality to import existing subaccounts on SAP BTP into Terraform/OpenTofu configurations. The configurations delivered by the btptf CLI are: - Provider configuration (excluding credentials) - [Import](https://developer.hashicorp.com/terraform/language/import) blocks for the resources - - Resource configuration retrieved from the platform + - Resource configuration retrieved from the platform The btptf CLI offers two options for the import: 1. As a one-step process via creating the import configuration by naming the resource types. -2. As a two-step process via creating a local JSON file with the resources to be imported. This file can be adjusted and then used as a configuration for the import. +2. As a two-step process via creating a local JSON file with the resources to be imported. This file can be adjusted and then used as a configuration for the import. ## Basic Flow @@ -19,7 +19,7 @@ The btptf CLI offers two options for the import: 2. Creating the files with the import block based on the information from the documentation and reading the data from the platform leveraging the corresponding Terraform [data sources](https://registry.terraform.io/providers/SAP/btp/latest/docs). -3. Executing the Terraform commands via Terraform CLI to generate the resource configuration and store the results in the file system. +3. Executing the Terraform/OpenTofu commands via Terraform/OpenTofu CLI to generate the resource configuration and store the results in the file system. The following points should be mentioned: diff --git a/docs/index.md b/docs/index.md index f937031..b72347a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,8 @@ # Terraform Exporter for SAP BTP -The *Terraform Exporter for SAP BTP* (btptf CLI) is a handy tool that makes it easier to bring your existing SAP Business Technology Platform ([BTP](https://www.sap.com/products/technology-platform/what-is-sap-business-technology-platform.html)) resources into [Terraform](https://www.terraform.io/). With it, you can take things like subaccounts and directories in BTP and turn them into Terraform state and configuration files. It's especially useful for teams who are moving to Terraform but still need to manage older infrastructure or SAP BTP accounts that are already set up. +The *Terraform Exporter for SAP BTP* (btptf CLI) is a handy tool that makes it easier to bring your existing SAP Business Technology Platform ([BTP](https://www.sap.com/products/technology-platform/what-is-sap-business-technology-platform.html)) resources into [Terraform](https://www.terraform.io/) or [OpenTofu](https://opentofu.org/). + +With it, you can take things like subaccounts and directories in BTP and turn them into Terraform/OpenTofu state and configuration files. It is especially useful for teams who are moving to Terraform/OpenTofu but still need to manage older infrastructure or SAP BTP accounts that are already set up. ## How does it work @@ -9,4 +11,4 @@ The *Terraform Exporter for SAP BTP* (btptf CLI) is a handy tool that makes it e - **Import Process**: The tool utilizes Terraform's [import](https://developer.hashicorp.com/terraform/cli/import) function to integrate each resource into Terraform's state. - **Configuration Generation**: After import, it generates the Terraform code (in HashiCorp Configuration Language - HCL) for each resource, enabling further customizations as needed. - +The same steps apply when using OpenTofu. diff --git a/docs/install.md b/docs/install.md index af9965c..91375fc 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2,7 +2,7 @@ You have two options to install the btptf CLI: -1. Download the pre-built binary. +1. Download the pre-built binary. 2. Local build The following sections describe the details for the two options. @@ -19,7 +19,7 @@ If you want to build the binary from scratch, follow these steps: 1. Open [this](https://github.com/SAP/terraform-exporter-btp) repository inside VS Code Editor -2. We have setup a [devcontainer](https://code.visualstudio.com/docs/devcontainers/tutorial), so reopen the repository in the devcontainer. +2. We have setup a [devcontainer](https://code.visualstudio.com/docs/devcontainers/tutorial), so reopen the repository in the devcontainer. We provide devcontainers for Terraform and OpenTofu. 3. Open a terminal in VS Code and install the binary by running @@ -31,4 +31,4 @@ If you want to build the binary from scratch, follow these steps: 4. The system will store the binary as `btptf` (`btptf.exe` in case of Windows) in the default binary path of your Go installation `$GOPATH/bin`. !!! tip - You find the value of the GOPATH via `go env GOPATH` \ No newline at end of file + You find the value of the GOPATH via `go env GOPATH` diff --git a/docs/limitations.md b/docs/limitations.md index 0fb1309..3f522fc 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -2,7 +2,7 @@ ## Supported Resources for Import -The btptf CLI can create import blocks and the corresponding configurations only for resources that support the import functionality of Terraform. Not all resources available in the Terraform providers support this feature and can hence not be imported. +The btptf CLI can create import blocks and the corresponding configurations only for resources that support the import functionality of Terraform/OpenTofu. Not all resources available in the Terraform providers support this feature and can hence not be imported. You find a list of supported resources for the Terraform Provider for SAP BTP in the corresponding repository on GitHub under the [Overview on importable resources](https://github.com/SAP/terraform-provider-btp/blob/main/guides/IMPORT.md). diff --git a/docs/prerequisites.md b/docs/prerequisites.md index aa28938..224c09a 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -1,10 +1,16 @@ # Prerequisite -## Terraform CLI +## Terraform/OpenTofu CLI -The btptf CLI requires a installation of the Terraform CLI. The Terraform CLI will be called by the btptf CLI. +The btptf CLI requires a installation of the Terraform CLI or the OpenTofu CLI. The corresponding CLI will be detected and called by the btptf CLI. -You find the necessary information in the [official Terraform documentation](https://developer.hashicorp.com/terraform/install#darwin). +You find the necessary information in the official documentation: + +- [Terraform](https://developer.hashicorp.com/terraform/install). +- [OpenTofu](https://opentofu.org/docs/intro/install/). + +!!! info + If you have both CLIs installed you can enforce the usage of one via the environment variable `BTPTF_IAC_TOOL` namely setting its value to `terraform`or `tofu`. ## Setting of Environment Variables diff --git a/pkg/tfutils/tfConfig.go b/pkg/tfutils/tfConfig.go index a8542d1..eb6cd05 100644 --- a/pkg/tfutils/tfConfig.go +++ b/pkg/tfutils/tfConfig.go @@ -68,16 +68,16 @@ func GenerateConfig(resourceFileName string, configFolder string, isMainCmd bool return fmt.Errorf("error changing directory to %s: %v", terraformConfigPath, err) } - if err := runTerraformCommand("init"); err != nil { + if err := runTfCmdGeneric("init"); err != nil { return fmt.Errorf("error running Terraform init: %v", err) } planOption := "--generate-config-out=" + resourceFileName - if err := runTerraformCommand("plan", planOption); err != nil { + if err := runTfCmdGeneric("plan", planOption); err != nil { return fmt.Errorf("error running Terraform plan: %v", err) } - if err := runTerraformCommand("fmt", "-recursive", "-list=false"); err != nil { + if err := runTfCmdGeneric("fmt", "-recursive", "-list=false"); err != nil { return fmt.Errorf("error running Terraform fmt: %v", err) } @@ -436,13 +436,13 @@ func FinalizeTfConfig(configFolder string) { log.Fatalf("error changing directory to %s: %v \n", terraformConfigPath, err) } - if err := runTerraformCommand("init"); err != nil { + if err := runTfCmdGeneric("init"); err != nil { CleanupProviderConfig() fmt.Print("\r\n") log.Fatalf("error initializing Terraform: %v", err) } - if err := runTerraformCommand("fmt", "-recursive", "-list=false"); err != nil { + if err := runTfCmdGeneric("fmt", "-recursive", "-list=false"); err != nil { CleanupProviderConfig() fmt.Print("\r\n") log.Fatalf("error running Terraform fmt: %v", err) diff --git a/pkg/tfutils/tfImport.go b/pkg/tfutils/tfImport.go index 22efcbb..c13e755 100644 --- a/pkg/tfutils/tfImport.go +++ b/pkg/tfutils/tfImport.go @@ -1,20 +1,16 @@ package tfutils import ( - "context" "encoding/json" "fmt" "log" "os" - "os/exec" "path/filepath" "slices" "strings" files "github.com/SAP/terraform-exporter-btp/pkg/files" output "github.com/SAP/terraform-exporter-btp/pkg/output" - "github.com/hashicorp/terraform-exec/tfexec" - "github.com/spf13/viper" ) // Constants for TF version for Terraform providers @@ -304,34 +300,21 @@ func readDataSource(subaccountId string, directoryId string, organizationId stri } func getTfStateData(configDir string, resourceName string, identifier string) ([]byte, error) { - execPath, err := exec.LookPath("terraform") - if err != nil { - fmt.Print("\r\n") - log.Fatalf("error finding Terraform: %v", err) - return nil, err - } + chDir := fmt.Sprintf("-chdir=%s", configDir) // Set custom user agent for call of TF Provider via exporter addUserAgent() defer removeUserAgent() - // create a new Terraform instance - tf, err := tfexec.NewTerraform(configDir, execPath) - if err != nil { - removeUserAgent() - fmt.Print("\r\n") - log.Fatalf("error running NewTerraform: %v", err) - return nil, err - } - - err = tf.Init(context.Background(), tfexec.Upgrade(true)) + err := runTfCmdGeneric(chDir, "init", "-upgrade") if err != nil { removeUserAgent() fmt.Print("\r\n") log.Fatalf("error running Init: %v", err) return nil, err } - err = tf.Apply(context.Background()) + + err = runTfCmdGeneric(chDir, "apply", "-auto-approve") if err != nil { err = handleNotFoundError(err, resourceName, identifier) removeUserAgent() @@ -340,7 +323,7 @@ func getTfStateData(configDir string, resourceName string, identifier string) ([ return nil, err } - state, err := tf.Show(context.Background()) + state, err := runTfShowJson(configDir) if err != nil { removeUserAgent() fmt.Print("\r\n") @@ -448,20 +431,6 @@ func generateDataSourcesForList(subaccountId string, directoryId string, organiz return transformDataToStringArray(btpResourceType, data), extractFeatureList(data, btpResourceType), nil } -func runTerraformCommand(args ...string) error { - - verbose := viper.GetViper().GetBool("verbose") - cmd := exec.Command("terraform", args...) - if verbose { - cmd.Stdout = os.Stdout - } else { - cmd.Stdout = nil - } - - cmd.Stderr = os.Stderr - return cmd.Run() -} - func GetExecutionLevelAndId(subaccountID string, directoryID string, organizationID string) (level string, id string) { if subaccountID != "" { return SubaccountLevel, subaccountID diff --git a/pkg/tfutils/tfWrapper.go b/pkg/tfutils/tfWrapper.go new file mode 100644 index 0000000..7ecb3bf --- /dev/null +++ b/pkg/tfutils/tfWrapper.go @@ -0,0 +1,85 @@ +package tfutils + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + + "github.com/spf13/viper" +) + +func getIaCTool() (tool string, err error) { + + //For TESTING purposes, we can set the tool to be used + tool = os.Getenv("BTPTF_IAC_TOOL") + if tool != "" { + return tool, nil + } + + _, localerr := exec.LookPath("terraform") + if localerr == nil { + tool = "terraform" + return tool, nil + } + + _, localerr = exec.LookPath("tofu") + if localerr == nil { + tool = "tofu" + return tool, nil + } + + fmt.Print("\r\n") + log.Fatalf("error finding Terraform or OpenTofu executable: %v", err) + return "", err +} + +func runTfCmdGeneric(args ...string) error { + tool, err := getIaCTool() + if err != nil { + return err + } + + verbose := viper.GetViper().GetBool("verbose") + cmd := exec.Command(tool, args...) + if verbose { + cmd.Stdout = os.Stdout + } else { + cmd.Stdout = nil + } + + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func runTfShowJson(directory string) (*State, error) { + chDir := fmt.Sprintf("-chdir=%s", directory) + + tool, err := getIaCTool() + if err != nil { + return nil, err + } + + cmd := exec.Command(tool, chDir, "show", "-json") + + var outBuffer bytes.Buffer + cmd.Stdout = &outBuffer + + err = cmd.Run() + if err != nil { + fmt.Println("Error executing command:", err) + return nil, err + } + + var state State + + err = json.Unmarshal(outBuffer.Bytes(), &state) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + return nil, err + } + + return &state, nil +} diff --git a/pkg/tfutils/types_tfState.go b/pkg/tfutils/types_tfState.go new file mode 100644 index 0000000..48dae2d --- /dev/null +++ b/pkg/tfutils/types_tfState.go @@ -0,0 +1,90 @@ +package tfutils + +import "encoding/json" + +// Reduced definition of state as provided by https://github.com/hashicorp/terraform-json + +type State struct { + // The version of the state format. This should always match the + // StateFormatVersion constant in this package, or else am + // unmarshal will be unstable. + FormatVersion string `json:"format_version,omitempty"` + + // The Terraform version used to make the state. + TerraformVersion string `json:"terraform_version,omitempty"` + + // The values that make up the state. + Values *StateValues `json:"values,omitempty"` +} + +type StateValues struct { + // The root module in this state representation. + RootModule *StateModule `json:"root_module,omitempty"` +} + +type StateModule struct { + // All resources or data sources within this module. + Resources []*StateResource `json:"resources,omitempty"` + + // The absolute module address, omitted for the root module. + Address string `json:"address,omitempty"` + + // Any child modules within this module. + ChildModules []*StateModule `json:"child_modules,omitempty"` +} + +type StateResource struct { + // The absolute resource address. + Address string `json:"address,omitempty"` + + // The resource mode. + Mode ResourceMode `json:"mode,omitempty"` + + // The resource type, example: "aws_instance" for aws_instance.foo. + Type string `json:"type,omitempty"` + + // The resource name, example: "foo" for aws_instance.foo. + Name string `json:"name,omitempty"` + + // The instance key for any resources that have been created using + // "count" or "for_each". If neither of these apply the key will be + // empty. + // + // This value can be either an integer (int) or a string. + Index interface{} `json:"index,omitempty"` + + // The name of the provider this resource belongs to. This allows + // the provider to be interpreted unambiguously in the unusual + // situation where a provider offers a resource type whose name + // does not start with its own name, such as the "googlebeta" + // provider offering "google_compute_instance". + ProviderName string `json:"provider_name,omitempty"` + + // The version of the resource type schema the "values" property + // conforms to. + SchemaVersion uint64 `json:"schema_version,"` + + // The JSON representation of the attribute values of the resource, + // whose structure depends on the resource type schema. Any unknown + // values are omitted or set to null, making them indistinguishable + // from absent values. + AttributeValues map[string]interface{} `json:"values,omitempty"` + + // The JSON representation of the sensitivity of the resource's + // attribute values. Only attributes which are sensitive + // are included in this structure. + SensitiveValues json.RawMessage `json:"sensitive_values,omitempty"` + + // The addresses of the resources that this resource depends on. + DependsOn []string `json:"depends_on,omitempty"` + + // If true, the resource has been marked as tainted and will be + // re-created on the next update. + Tainted bool `json:"tainted,omitempty"` + + // DeposedKey is set if the resource instance has been marked Deposed and + // will be destroyed on the next apply. + DeposedKey string `json:"deposed_key,omitempty"` +} + +type ResourceMode string diff --git a/sonar-project.properties b/sonar-project.properties index e77a230..778449e 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,7 +4,7 @@ sonar.projectName=terraform-exporter-btp sonar.language=go sonar.sources=. sonar.inclusions=**/*.go -sonar.exclusions=**/*_test.go,test/**,**/zz-generated*,cmd/**,format/output.go,cmd/**,pkg/output/output.go,pkg/files/**,main.go,pkg/tfimportprovider/tfImportProviderFactory.go,pkg/tfimportprovider/TfimportProvider.go +sonar.exclusions=**/*_test.go,test/**,**/zz-generated*,cmd/**,format/output.go,cmd/**,pkg/output/output.go,pkg/files/**,main.go,pkg/tfimportprovider/tfImportProviderFactory.go,pkg/tfimportprovider/TfimportProvider.go,pkg/tfutils/tfWrapper.go,pkg/tfutils/types_tfState.go sonar.tests=, sonar.test.inclusions=**/*_test.go sonar.test.exclusions=**/vendor/**