From ad236b3a19315509941f45dfd0c0826c0db1ac78 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 12 Mar 2021 08:54:47 -0800 Subject: [PATCH 1/9] Added test of project update This is necessary because project numbers are only available from existing projects, not from projects being created --- .gitignore | 7 +- test/cli_test.go | 13 +- test/init_test.go | 1 + test/read_test.go | 3 +- .../project_match_target/lib/.gitignore | 0 .../always_violates_project_match_target.yaml | 27 ++++ .../templates/gcp_always_violates_v1.yaml | 57 +++++++ ...oject.json => example_project_create.json} | 0 ...e_project.tf => example_project_create.tf} | 0 ...son => example_project_create.tfplan.json} | 0 .../templates/example_project_update.json | 40 +++++ testdata/templates/example_project_update.tf | 40 +++++ .../example_project_update.tfplan.json | 144 ++++++++++++++++++ .../templates/example_project_update.tfstate | 36 +++++ 14 files changed, 363 insertions(+), 5 deletions(-) create mode 100644 testdata/sample_policies/project_match_target/lib/.gitignore create mode 100644 testdata/sample_policies/project_match_target/policies/constraints/always_violates_project_match_target.yaml create mode 100644 testdata/sample_policies/project_match_target/policies/templates/gcp_always_violates_v1.yaml rename testdata/templates/{example_project.json => example_project_create.json} (100%) rename testdata/templates/{example_project.tf => example_project_create.tf} (100%) rename testdata/templates/{example_project.tfplan.json => example_project_create.tfplan.json} (100%) create mode 100644 testdata/templates/example_project_update.json create mode 100644 testdata/templates/example_project_update.tf create mode 100644 testdata/templates/example_project_update.tfplan.json create mode 100644 testdata/templates/example_project_update.tfstate diff --git a/.gitignore b/.gitignore index 3fd1bad38..992ce4f87 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,12 @@ credentials.json # Terraform (possibly sensitive) .terraform/ *.tfplan -*.tfstate* +examples/*.tfstate* # Visual Studio Code .vscode/ # Intellij Golang -.idea/ \ No newline at end of file +.idea/ + +# Mac OSX +.DS_Store diff --git a/test/cli_test.go b/test/cli_test.go index 5396716fa..47c54e38e 100644 --- a/test/cli_test.go +++ b/test/cli_test.go @@ -75,7 +75,8 @@ func TestCLI(t *testing.T) { {name: "example_organization_iam_policy"}, {name: "example_pubsub_topic"}, {name: "example_pubsub_subscription"}, - {name: "example_project"}, + {name: "example_project_create", constraints: []constraint{alwaysViolate, constraint{name: "project_match_target", wantViolation: false, wantOutputRegex: ""}}}, + {name: "example_project_update", constraints: []constraint{alwaysViolate, constraint{name: "project_match_target", wantViolation: true, wantOutputRegex: "Constraint GCPAlwaysViolatesConstraintV1.always_violates_project_match_target on resource"}}}, {name: "example_project_in_org"}, {name: "example_project_in_folder"}, {name: "example_project_iam"}, @@ -129,6 +130,14 @@ func TestCLI(t *testing.T) { // Generate the .tf and _assets.json files into the temporary directory. generateTestFiles(t, "../testdata/templates", dir, c.name+".tf") generateTestFiles(t, "../testdata/templates", dir, c.name+".json") + generateTestFiles(t, "../testdata/templates", dir, c.name+".tfstate") + err = os.Rename( + filepath.Join(dir, c.name + ".tfstate"), + filepath.Join(dir, "terraform.tfstate"), + ) + if err != nil { + log.Printf("Error renaming tfstate: %v", err) + } terraform(t, dir, c.name) @@ -288,7 +297,7 @@ func terraformInit(t *testing.T, executable, dir string) { } func terraformPlan(t *testing.T, executable, dir, tfplan string) { - terraformExec(t, executable, dir, "plan", "-input=false", "--out", tfplan) + terraformExec(t, executable, dir, "plan", "-input=false", "-refresh=false", "-out", tfplan) } func terraformShow(t *testing.T, executable, dir, tfplan string) []byte { diff --git a/test/init_test.go b/test/init_test.go index be9cc19a2..19a3eb4ac 100644 --- a/test/init_test.go +++ b/test/init_test.go @@ -137,6 +137,7 @@ func generateTestFiles(t *testing.T, sourceDir string, targetDir string, selecto if err := f.Close(); err != nil { t.Fatalf("closing file %v: %v", path, err) } + log.Printf("Successfully created file %v", path) } } diff --git a/test/read_test.go b/test/read_test.go index b0b650be1..9bd8030d3 100644 --- a/test/read_test.go +++ b/test/read_test.go @@ -39,7 +39,8 @@ func TestReadPlannedAssetsCoverage(t *testing.T) { {name: "example_organization_iam_policy"}, {name: "example_pubsub_topic"}, {name: "example_pubsub_subscription"}, - {name: "example_project"}, + {name: "example_project_create"}, + {name: "example_project_update"}, {name: "example_project_in_org"}, {name: "example_project_in_folder"}, {name: "example_project_organization_policy"}, diff --git a/testdata/sample_policies/project_match_target/lib/.gitignore b/testdata/sample_policies/project_match_target/lib/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/testdata/sample_policies/project_match_target/policies/constraints/always_violates_project_match_target.yaml b/testdata/sample_policies/project_match_target/policies/constraints/always_violates_project_match_target.yaml new file mode 100644 index 000000000..3b42dc9e2 --- /dev/null +++ b/testdata/sample_policies/project_match_target/policies/constraints/always_violates_project_match_target.yaml @@ -0,0 +1,27 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: constraints.gatekeeper.sh/v1alpha1 +kind: GCPAlwaysViolatesConstraintV1 +metadata: + name: always_violates_project_match_target + annotations: + description: Testing policy, will always violate. +spec: + constraintVersion: 0.1.0 + severity: high + match: + target: + - "organizations/12345/projects/1234567890" + parameters: {} diff --git a/testdata/sample_policies/project_match_target/policies/templates/gcp_always_violates_v1.yaml b/testdata/sample_policies/project_match_target/policies/templates/gcp_always_violates_v1.yaml new file mode 100644 index 000000000..10f74c0ed --- /dev/null +++ b/testdata/sample_policies/project_match_target/policies/templates/gcp_always_violates_v1.yaml @@ -0,0 +1,57 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: templates.gatekeeper.sh/v1alpha1 +kind: ConstraintTemplate +metadata: + name: gcp-always-violates-v1 +spec: + crd: + spec: + names: + kind: GCPAlwaysViolatesConstraintV1 + validation: + openAPIV3Schema: + properties: {} + targets: + validation.gcp.forsetisecurity.org: + rego: | #INLINE("validator/always_violates.rego") + # + # Copyright 2018 Google LLC + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + package templates.gcp.GCPAlwaysViolatesConstraintV1 + + import data.validator.gcp.lib as lib + + deny[{ + "msg": message, + "details": metadata, + }] { + message := "violates on all resources." + metadata := {"asset": input.asset} + } + #ENDINLINE diff --git a/testdata/templates/example_project.json b/testdata/templates/example_project_create.json similarity index 100% rename from testdata/templates/example_project.json rename to testdata/templates/example_project_create.json diff --git a/testdata/templates/example_project.tf b/testdata/templates/example_project_create.tf similarity index 100% rename from testdata/templates/example_project.tf rename to testdata/templates/example_project_create.tf diff --git a/testdata/templates/example_project.tfplan.json b/testdata/templates/example_project_create.tfplan.json similarity index 100% rename from testdata/templates/example_project.tfplan.json rename to testdata/templates/example_project_create.tfplan.json diff --git a/testdata/templates/example_project_update.json b/testdata/templates/example_project_update.json new file mode 100644 index 000000000..b10efbd54 --- /dev/null +++ b/testdata/templates/example_project_update.json @@ -0,0 +1,40 @@ +[ + { + "name": "//cloudbilling.googleapis.com/projects/{{.Provider.project}}/billingInfo", + "asset_type": "cloudbilling.googleapis.com/ProjectBillingInfo", + "ancestry_path": "organization/{{.OrgID}}/project/{{.Provider.project}}", + "resource": { + "version": "v1", + "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/cloudbilling/v1/rest", + "discovery_name": "ProjectBillingInfo", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "data": { + "billingAccountName": "billingAccounts/{{.Project.BillingAccountName}}", + "name": "projects/{{.Provider.project}}/billingInfo", + "projectId": "{{.Provider.project}}" + } + } + }, + { + "name": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "asset_type": "cloudresourcemanager.googleapis.com/Project", + "ancestry_path": "organization/{{.OrgID}}/project/{{.Provider.project}}", + "resource": { + "version": "v1", + "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest", + "discovery_name": "Project", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "data": { + "labels": { + "project-label-key-a": "project-label-val-a" + }, + "name": "My New Project", + "parent": { + "id": "{{.OrgID}}", + "type": "organization" + }, + "projectId": "{{.Provider.project}}" + } + } + } +] \ No newline at end of file diff --git a/testdata/templates/example_project_update.tf b/testdata/templates/example_project_update.tf new file mode 100644 index 000000000..0bd91a4a2 --- /dev/null +++ b/testdata/templates/example_project_update.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> {{.Provider.version}}" + } + } +} + +provider "google" { + {{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}} +} + +resource "google_project" "my_project" { + name = "My New Project" + project_id = "{{.Provider.project}}" + org_id = "{{.OrgID}}" + + billing_account = "{{.Project.BillingAccountName}}" + + labels = { + "project-label-key-a" = "project-label-val-a" + } +} diff --git a/testdata/templates/example_project_update.tfplan.json b/testdata/templates/example_project_update.tfplan.json new file mode 100644 index 000000000..8c602f702 --- /dev/null +++ b/testdata/templates/example_project_update.tfplan.json @@ -0,0 +1,144 @@ +{ + "format_version": "0.1", + "terraform_version": "0.14.7", + "planned_values": { + "root_module": { + "resources": [ + { + "address": "google_project.my_project", + "mode": "managed", + "type": "google_project", + "name": "my_project", + "provider_name": "registry.terraform.io/hashicorp/google", + "schema_version": 1, + "values": { + "auto_create_network": true, + "billing_account": "{{.Project.BillingAccountName}}", + "folder_id": "", + "id": "projects/{{.Provider.project}}", + "labels": { + "project-label-key-a": "project-label-val-a" + }, + "name": "My New Project", + "number": "1234567890", + "org_id": "{{.OrgID}}", + "project_id": "{{.Provider.project}}", + "skip_delete": null, + "timeouts": null + } + } + ] + } + }, + "resource_changes": [ + { + "address": "google_project.my_project", + "mode": "managed", + "type": "google_project", + "name": "my_project", + "provider_name": "registry.terraform.io/hashicorp/google", + "change": { + "actions": [ + "update" + ], + "before": { + "auto_create_network": true, + "billing_account": "{{.Project.BillingAccountName}}", + "folder_id": "", + "id": "projects/{{.Provider.project}}", + "labels": { + "project-label-key-a": "project-label-val-a" + }, + "name": "My Project", + "number": "1234567890", + "org_id": "{{.OrgID}}", + "project_id": "{{.Provider.project}}", + "skip_delete": null, + "timeouts": null + }, + "after": { + "auto_create_network": true, + "billing_account": "{{.Project.BillingAccountName}}", + "folder_id": "", + "id": "projects/{{.Provider.project}}", + "labels": { + "project-label-key-a": "project-label-val-a" + }, + "name": "My New Project", + "number": "1234567890", + "org_id": "{{.OrgID}}", + "project_id": "{{.Provider.project}}", + "skip_delete": null, + "timeouts": null + }, + "after_unknown": {} + } + } + ], + "prior_state": { + "format_version": "0.1", + "terraform_version": "0.14.7", + "values": { + "root_module": { + "resources": [ + { + "address": "google_project.my_project", + "mode": "managed", + "type": "google_project", + "name": "my_project", + "provider_name": "registry.terraform.io/hashicorp/google", + "schema_version": 1, + "values": { + "auto_create_network": true, + "billing_account": "{{.Project.BillingAccountName}}", + "folder_id": "", + "id": "projects/{{.Provider.project}}", + "labels": { + "project-label-key-a": "project-label-val-a" + }, + "name": "My Project", + "number": "1234567890", + "org_id": "{{.OrgID}}", + "project_id": "{{.Provider.project}}", + "skip_delete": null, + "timeouts": null + } + } + ] + } + } + }, + "configuration": { + "root_module": { + "resources": [ + { + "address": "google_project.my_project", + "mode": "managed", + "type": "google_project", + "name": "my_project", + "provider_config_key": "google", + "expressions": { + "billing_account": { + "constant_value": "{{.Project.BillingAccountName}}" + }, + "org_id": { + "constant_value": "organizations/{{.OrgID}}" + }, + "labels": { + "constant_value": { + "project-label-key-a": "project-label-val-a" + } + }, + "name": { + "constant_value": "My New Project" + }, + "project_id": { + "constant_value": "{{.Provider.project}}" + } + }, + "schema_version": 1 + } + ] + } + } +} diff --git a/testdata/templates/example_project_update.tfstate b/testdata/templates/example_project_update.tfstate new file mode 100644 index 000000000..de3b7ab0a --- /dev/null +++ b/testdata/templates/example_project_update.tfstate @@ -0,0 +1,36 @@ +{ + "version": 4, + "terraform_version": "0.14.7", + "serial": 3, + "lineage": "2d6ba1d4-001c-be50-e951-42833952b512", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "google_project", + "name": "my_project", + "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "auto_create_network": true, + "billing_account": "{{.Project.BillingAccountName}}", + "folder_id": "", + "id": "projects/{{.Provider.project}}", + "labels": { + "project-label-key-a": "project-label-val-a" + }, + "name": "My Project", + "number": "1234567890", + "org_id": "{{.OrgID}}", + "project_id": "{{.Provider.project}}", + "skip_delete": null, + "timeouts": null + }, + "sensitive_attributes": [] + } + ] + } + ] +} From 05a695aa999e8319fc69ae892aad9a0c01f20286 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 12 Mar 2021 09:29:44 -0800 Subject: [PATCH 2/9] Made tfstate only get generated if it exists --- test/cli_test.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/cli_test.go b/test/cli_test.go index 47c54e38e..da26341e6 100644 --- a/test/cli_test.go +++ b/test/cli_test.go @@ -130,13 +130,21 @@ func TestCLI(t *testing.T) { // Generate the .tf and _assets.json files into the temporary directory. generateTestFiles(t, "../testdata/templates", dir, c.name+".tf") generateTestFiles(t, "../testdata/templates", dir, c.name+".json") - generateTestFiles(t, "../testdata/templates", dir, c.name+".tfstate") - err = os.Rename( - filepath.Join(dir, c.name + ".tfstate"), - filepath.Join(dir, "terraform.tfstate"), - ) + + // Uses glob matching to match generateTestFiles internals. + tfstateMatches, err := filepath.Glob(filepath.Join("../testdata/templates", c.name+".tfstate")) if err != nil { - log.Printf("Error renaming tfstate: %v", err) + t.Fatalf("malformed glob: %v", err) + } + if tfstateMatches != nil { + generateTestFiles(t, "../testdata/templates", dir, c.name+".tfstate") + err = os.Rename( + filepath.Join(dir, c.name + ".tfstate"), + filepath.Join(dir, "terraform.tfstate"), + ) + if err != nil { + t.Fatalf("renaming tfstate: %v", err) + } } terraform(t, dir, c.name) From a0ba7753c6c60b7b4784ab422480d9cb4b053df7 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Wed, 17 Mar 2021 15:39:12 -0700 Subject: [PATCH 3/9] Made project update conversion test expect project number --- test/init_test.go | 1 + testdata/templates/example_project_update.json | 6 +++--- testdata/templates/example_project_update.tfplan.json | 8 ++++---- testdata/templates/example_project_update.tfstate | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/init_test.go b/test/init_test.go index 19a3eb4ac..24014e885 100644 --- a/test/init_test.go +++ b/test/init_test.go @@ -103,6 +103,7 @@ func init() { "Name": "My Project Name", "ProjectId": "my-project-id", "BillingAccountName": "012345-567890-ABCDEF", + "Number": "1234567890", }, OrgID: org, FolderID: folder, diff --git a/testdata/templates/example_project_update.json b/testdata/templates/example_project_update.json index b10efbd54..f4a643c3d 100644 --- a/testdata/templates/example_project_update.json +++ b/testdata/templates/example_project_update.json @@ -7,7 +7,7 @@ "version": "v1", "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/cloudbilling/v1/rest", "discovery_name": "ProjectBillingInfo", - "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}", "data": { "billingAccountName": "billingAccounts/{{.Project.BillingAccountName}}", "name": "projects/{{.Provider.project}}/billingInfo", @@ -16,14 +16,14 @@ } }, { - "name": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "name": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}", "asset_type": "cloudresourcemanager.googleapis.com/Project", "ancestry_path": "organization/{{.OrgID}}/project/{{.Provider.project}}", "resource": { "version": "v1", "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest", "discovery_name": "Project", - "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}", "data": { "labels": { "project-label-key-a": "project-label-val-a" diff --git a/testdata/templates/example_project_update.tfplan.json b/testdata/templates/example_project_update.tfplan.json index 8c602f702..000296711 100644 --- a/testdata/templates/example_project_update.tfplan.json +++ b/testdata/templates/example_project_update.tfplan.json @@ -20,7 +20,7 @@ "project-label-key-a": "project-label-val-a" }, "name": "My New Project", - "number": "1234567890", + "number": "{{.Project.Number}}", "org_id": "{{.OrgID}}", "project_id": "{{.Provider.project}}", "skip_delete": null, @@ -50,7 +50,7 @@ "project-label-key-a": "project-label-val-a" }, "name": "My Project", - "number": "1234567890", + "number": "{{.Project.Number}}", "org_id": "{{.OrgID}}", "project_id": "{{.Provider.project}}", "skip_delete": null, @@ -65,7 +65,7 @@ "project-label-key-a": "project-label-val-a" }, "name": "My New Project", - "number": "1234567890", + "number": "{{.Project.Number}}", "org_id": "{{.OrgID}}", "project_id": "{{.Provider.project}}", "skip_delete": null, @@ -97,7 +97,7 @@ "project-label-key-a": "project-label-val-a" }, "name": "My Project", - "number": "1234567890", + "number": "{{.Project.Number}}", "org_id": "{{.OrgID}}", "project_id": "{{.Provider.project}}", "skip_delete": null, diff --git a/testdata/templates/example_project_update.tfstate b/testdata/templates/example_project_update.tfstate index de3b7ab0a..e2ec15781 100644 --- a/testdata/templates/example_project_update.tfstate +++ b/testdata/templates/example_project_update.tfstate @@ -22,7 +22,7 @@ "project-label-key-a": "project-label-val-a" }, "name": "My Project", - "number": "1234567890", + "number": "{{.Project.Number}}", "org_id": "{{.OrgID}}", "project_id": "{{.Provider.project}}", "skip_delete": null, From 6983e5efe6845d40bb370c4663655c9d822ea570 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 18 Mar 2021 09:18:08 -0700 Subject: [PATCH 4/9] support / expect project number in billinginfo names --- converters/google/vendor_utils.go | 7 ++++++- testdata/templates/example_project_update.json | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/converters/google/vendor_utils.go b/converters/google/vendor_utils.go index a4594e58d..29245e31f 100644 --- a/converters/google/vendor_utils.go +++ b/converters/google/vendor_utils.go @@ -30,7 +30,12 @@ func getProject(d converter.TerraformResourceData, config *converter.Config, cai switch cai.Type { case "cloudresourcemanager.googleapis.com/Project", "cloudbilling.googleapis.com/ProjectBillingInfo": - res, ok := d.GetOk("project_id") + res, ok := d.GetOk("number") + if ok { + return res.(string), nil + } + // Fall back to project_id if number is not available. + res, ok = d.GetOk("project_id") if ok { return res.(string), nil } else { diff --git a/testdata/templates/example_project_update.json b/testdata/templates/example_project_update.json index f4a643c3d..c61cd295b 100644 --- a/testdata/templates/example_project_update.json +++ b/testdata/templates/example_project_update.json @@ -1,6 +1,6 @@ [ { - "name": "//cloudbilling.googleapis.com/projects/{{.Provider.project}}/billingInfo", + "name": "//cloudbilling.googleapis.com/projects/{{.Project.Number}}/billingInfo", "asset_type": "cloudbilling.googleapis.com/ProjectBillingInfo", "ancestry_path": "organization/{{.OrgID}}/project/{{.Provider.project}}", "resource": { @@ -10,7 +10,7 @@ "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}", "data": { "billingAccountName": "billingAccounts/{{.Project.BillingAccountName}}", - "name": "projects/{{.Provider.project}}/billingInfo", + "name": "projects/{{.Project.Number}}/billingInfo", "projectId": "{{.Provider.project}}" } } From 6d5c1619acc2b1604dc89813e1d96db8cfb42f56 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 18 Mar 2021 09:41:26 -0700 Subject: [PATCH 5/9] Made ancestrymanager prefer project number if available --- ancestrymanager/ancestrymanager.go | 11 ++++++++--- testdata/templates/example_project_update.json | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ancestrymanager/ancestrymanager.go b/ancestrymanager/ancestrymanager.go index ced45d8f5..55f6e834a 100644 --- a/ancestrymanager/ancestrymanager.go +++ b/ancestrymanager/ancestrymanager.go @@ -51,9 +51,14 @@ func (m *resourceAncestryManager) getAncestryFromResource(tfData converter.Terra switch cai.Type { case "cloudresourcemanager.googleapis.com/Project", "cloudbilling.googleapis.com/ProjectBillingInfo": - projectID, ok := tfData.GetOk("project_id") - if !ok || projectID == "" { - return nil, false + // Prefer project number to project id if available; + // CAI exports use project number. + projectID, ok := tfData.GetOk("number") + if !ok { + projectID, ok = tfData.GetOk("project_id") + if !ok || projectID == "" { + return nil, false + } } ancestry := []*cloudresourcemanager.Ancestor{ diff --git a/testdata/templates/example_project_update.json b/testdata/templates/example_project_update.json index c61cd295b..8981ec543 100644 --- a/testdata/templates/example_project_update.json +++ b/testdata/templates/example_project_update.json @@ -2,7 +2,7 @@ { "name": "//cloudbilling.googleapis.com/projects/{{.Project.Number}}/billingInfo", "asset_type": "cloudbilling.googleapis.com/ProjectBillingInfo", - "ancestry_path": "organization/{{.OrgID}}/project/{{.Provider.project}}", + "ancestry_path": "organization/{{.OrgID}}/project/{{.Project.Number}}", "resource": { "version": "v1", "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/cloudbilling/v1/rest", @@ -18,7 +18,7 @@ { "name": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}", "asset_type": "cloudresourcemanager.googleapis.com/Project", - "ancestry_path": "organization/{{.OrgID}}/project/{{.Provider.project}}", + "ancestry_path": "organization/{{.OrgID}}/project/{{.Project.Number}}", "resource": { "version": "v1", "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest", From b3dceee125a29ff4e7baf9a51b1328140634182c Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Thu, 18 Mar 2021 14:57:01 -0700 Subject: [PATCH 6/9] Upgraded terraform-google-conversion --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 423f431f4..9d13f3941 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/GoogleCloudPlatform/terraform-validator require ( - github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210311162721-d97a3783d011 + github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210318171059-9ab40040d220 github.com/forseti-security/config-validator v0.0.0-20200812033229-7388761cc9ca github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.4.3 diff --git a/go.sum b/go.sum index e473131b2..5058f05b4 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210310165035 github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210310165035-fe5eb1b46daa/go.mod h1:ZmY0Ua5EVNaxHXJe5x3VQfaghzCmORy2fksFH4Wf6Eg= github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210311162721-d97a3783d011 h1:dlyKNzKVp7js3QyDFI3XaS+WzFF+zsEIdf5bgTFPwBg= github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210311162721-d97a3783d011/go.mod h1:ZmY0Ua5EVNaxHXJe5x3VQfaghzCmORy2fksFH4Wf6Eg= +github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210318171059-9ab40040d220 h1:jst9/iXj83wIXE8kvzy6jTo37ZYpabY5hvNe2brKa9Q= +github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210318171059-9ab40040d220/go.mod h1:ZmY0Ua5EVNaxHXJe5x3VQfaghzCmORy2fksFH4Wf6Eg= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= From 3578fba41180bc1e8705097f4b06c382b0727f7c Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 19 Mar 2021 08:47:11 -0700 Subject: [PATCH 7/9] Removed data sources from tf examples It turns out that running tests with -refresh=false that use data sources to generate json doesn't work in tf 0.12 --- .../templates/example_organization_iam_policy.tf | 12 +----------- testdata/templates/example_project_iam_policy.tf | 12 +----------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/testdata/templates/example_organization_iam_policy.tf b/testdata/templates/example_organization_iam_policy.tf index ffd1c2ce3..57d4e7940 100644 --- a/testdata/templates/example_organization_iam_policy.tf +++ b/testdata/templates/example_organization_iam_policy.tf @@ -29,15 +29,5 @@ provider "google" { resource "google_organization_iam_policy" "policy" { org_id = "123456789" - policy_data = data.google_iam_policy.admin.policy_data -} - -data "google_iam_policy" "admin" { - binding { - role = "roles/editor" - - members = [ - "user:jane@example.com", - ] - } + policy_data = "{\"bindings\":[{\"members\":[\"user:jane@example.com\"],\"role\":\"roles/editor\"}]}" } diff --git a/testdata/templates/example_project_iam_policy.tf b/testdata/templates/example_project_iam_policy.tf index 7a63418b2..5be740863 100644 --- a/testdata/templates/example_project_iam_policy.tf +++ b/testdata/templates/example_project_iam_policy.tf @@ -29,15 +29,5 @@ provider "google" { resource "google_project_iam_policy" "project" { project = "{{.Provider.project}}" - policy_data = data.google_iam_policy.admin.policy_data -} - -data "google_iam_policy" "admin" { - binding { - role = "roles/editor" - - members = [ - "user:jane@example.com", - ] - } + policy_data = "{\"bindings\":[{\"members\":[\"user:jane@example.com\"],\"role\":\"roles/editor\"}]}" } From ca981354511c8debdb94fbc6193bf6b453dd401e Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 19 Mar 2021 09:30:09 -0700 Subject: [PATCH 8/9] Downgraded project update state to be 0.12-compatible --- testdata/templates/example_project_update.tfstate | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testdata/templates/example_project_update.tfstate b/testdata/templates/example_project_update.tfstate index e2ec15781..6ef04dea8 100644 --- a/testdata/templates/example_project_update.tfstate +++ b/testdata/templates/example_project_update.tfstate @@ -1,15 +1,15 @@ { "version": 4, - "terraform_version": "0.14.7", + "terraform_version": "0.12.30", "serial": 3, - "lineage": "2d6ba1d4-001c-be50-e951-42833952b512", + "lineage": "b430721c-2a1f-e6b3-157b-733a455c961d", "outputs": {}, "resources": [ { "mode": "managed", "type": "google_project", "name": "my_project", - "provider": "provider[\"registry.terraform.io/hashicorp/google\"]", + "provider": "provider.google", "instances": [ { "schema_version": 1, From cfbd4dd177732a629969383b53ada4b300acaa93 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Fri, 19 Mar 2021 09:36:28 -0700 Subject: [PATCH 9/9] Downgraded project update state to be 0.12.24-compatible for ci/cd --- testdata/templates/example_project_update.tfstate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testdata/templates/example_project_update.tfstate b/testdata/templates/example_project_update.tfstate index 6ef04dea8..72f29ace6 100644 --- a/testdata/templates/example_project_update.tfstate +++ b/testdata/templates/example_project_update.tfstate @@ -1,6 +1,6 @@ { "version": 4, - "terraform_version": "0.12.30", + "terraform_version": "0.12.24", "serial": 3, "lineage": "b430721c-2a1f-e6b3-157b-733a455c961d", "outputs": {},