Merge pull request #197 from GoogleCloudPlatform/182-project-numbers

Fixed matching of rules to specific projects

.gitignore

@@ -8,9 +8,12 @@ credentials.json
 # Terraform (possibly sensitive)
 .terraform/
 *.tfplan
-*.tfstate*
+examples/*.tfstate*
 
 # Visual Studio Code
 .vscode/
 # Intellij Golang
-.idea/
+.idea/
+
+# Mac OSX
+.DS_Store

ancestrymanager/ancestrymanager.go

@@ -51,9 +51,14 @@ func (m *resourceAncestryManager) getAncestryFromResource(tfData converter.Terra
 
 	switch cai.Type {
 	case "cloudresourcemanager.googleapis.com/Project", "cloudbilling.googleapis.com/ProjectBillingInfo":
-		projectID, ok := tfData.GetOk("project_id")
-		if !ok || projectID == "" {
-			return nil, false
+		// Prefer project number to project id if available;
+		// CAI exports use project number.
+		projectID, ok := tfData.GetOk("number")
+		if !ok {
+			projectID, ok = tfData.GetOk("project_id")
+			if !ok || projectID == "" {
+				return nil, false
+			}
 		}
 
 		ancestry := []*cloudresourcemanager.Ancestor{

converters/google/vendor_utils.go

@@ -30,7 +30,12 @@ func getProject(d converter.TerraformResourceData, config *converter.Config, cai
 	switch cai.Type {
 	case "cloudresourcemanager.googleapis.com/Project",
 		"cloudbilling.googleapis.com/ProjectBillingInfo":
-		res, ok := d.GetOk("project_id")
+		res, ok := d.GetOk("number")
+		if ok {
+			return res.(string), nil
+		}
+		// Fall back to project_id if number is not available.
+		res, ok = d.GetOk("project_id")
 		if ok {
 			return res.(string), nil
 		} else {

go.mod

@@ -1,7 +1,7 @@
 module github.com/GoogleCloudPlatform/terraform-validator
 
 require (
-	github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210311162721-d97a3783d011
+	github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210318171059-9ab40040d220
 	github.com/forseti-security/config-validator v0.0.0-20200812033229-7388761cc9ca
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/protobuf v1.4.3

go.sum

@@ -69,6 +69,8 @@ github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210310165035
 github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210310165035-fe5eb1b46daa/go.mod h1:ZmY0Ua5EVNaxHXJe5x3VQfaghzCmORy2fksFH4Wf6Eg=
 github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210311162721-d97a3783d011 h1:dlyKNzKVp7js3QyDFI3XaS+WzFF+zsEIdf5bgTFPwBg=
 github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210311162721-d97a3783d011/go.mod h1:ZmY0Ua5EVNaxHXJe5x3VQfaghzCmORy2fksFH4Wf6Eg=
+github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210318171059-9ab40040d220 h1:jst9/iXj83wIXE8kvzy6jTo37ZYpabY5hvNe2brKa9Q=
+github.com/GoogleCloudPlatform/terraform-google-conversion v0.0.0-20210318171059-9ab40040d220/go.mod h1:ZmY0Ua5EVNaxHXJe5x3VQfaghzCmORy2fksFH4Wf6Eg=
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=

test/cli_test.go

@@ -75,7 +75,8 @@ func TestCLI(t *testing.T) {
 		{name: "example_organization_iam_policy"},
 		{name: "example_pubsub_topic"},
 		{name: "example_pubsub_subscription"},
-		{name: "example_project"},
+		{name: "example_project_create", constraints: []constraint{alwaysViolate, constraint{name: "project_match_target", wantViolation: false, wantOutputRegex: ""}}},
+		{name: "example_project_update", constraints: []constraint{alwaysViolate, constraint{name: "project_match_target", wantViolation: true, wantOutputRegex: "Constraint GCPAlwaysViolatesConstraintV1.always_violates_project_match_target on resource"}}},
 		{name: "example_project_in_org"},
 		{name: "example_project_in_folder"},
 		{name: "example_project_iam"},
@@ -130,6 +131,22 @@ func TestCLI(t *testing.T) {
 				generateTestFiles(t, "../testdata/templates", dir, c.name+".tf")
 				generateTestFiles(t, "../testdata/templates", dir, c.name+".json")
 
+				// Uses glob matching to match generateTestFiles internals.
+				tfstateMatches, err := filepath.Glob(filepath.Join("../testdata/templates", c.name+".tfstate"))
+				if err != nil {
+					t.Fatalf("malformed glob: %v", err)
+				}
+				if tfstateMatches != nil {
+					generateTestFiles(t, "../testdata/templates", dir, c.name+".tfstate")
+					err = os.Rename(
+						filepath.Join(dir, c.name + ".tfstate"),
+						filepath.Join(dir, "terraform.tfstate"),
+					)
+					if err != nil {
+						t.Fatalf("renaming tfstate: %v", err)
+					}
+				}
+
 				terraform(t, dir, c.name)
 
 				t.Run("cmd=convert", func(t *testing.T) {
@@ -288,7 +305,7 @@ func terraformInit(t *testing.T, executable, dir string) {
 }
 
 func terraformPlan(t *testing.T, executable, dir, tfplan string) {
-	terraformExec(t, executable, dir, "plan", "-input=false", "--out", tfplan)
+	terraformExec(t, executable, dir, "plan", "-input=false", "-refresh=false", "-out", tfplan)
 }
 
 func terraformShow(t *testing.T, executable, dir, tfplan string) []byte {

test/init_test.go

@@ -103,6 +103,7 @@ func init() {
 			"Name":               "My Project Name",
 			"ProjectId":          "my-project-id",
 			"BillingAccountName": "012345-567890-ABCDEF",
+			"Number":             "1234567890",
 		},
 		OrgID:    org,
 		FolderID: folder,
@@ -137,6 +138,7 @@ func generateTestFiles(t *testing.T, sourceDir string, targetDir string, selecto
 		if err := f.Close(); err != nil {
 			t.Fatalf("closing file %v: %v", path, err)
 		}
+		log.Printf("Successfully created file %v", path)
 	}
 }
 

test/read_test.go

@@ -39,7 +39,8 @@ func TestReadPlannedAssetsCoverage(t *testing.T) {
 		{name: "example_organization_iam_policy"},
 		{name: "example_pubsub_topic"},
 		{name: "example_pubsub_subscription"},
-		{name: "example_project"},
+		{name: "example_project_create"},
+		{name: "example_project_update"},
 		{name: "example_project_in_org"},
 		{name: "example_project_in_folder"},
 		{name: "example_project_organization_policy"},

testdata/sample_policies/project_match_target/policies/constraints/always_violates_project_match_target.yaml

@@ -0,0 +1,27 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: constraints.gatekeeper.sh/v1alpha1
+kind: GCPAlwaysViolatesConstraintV1
+metadata:
+  name: always_violates_project_match_target
+  annotations:
+    description: Testing policy, will always violate.
+spec:
+  constraintVersion: 0.1.0
+  severity: high
+  match:
+    target:
+    - "organizations/12345/projects/1234567890"
+  parameters: {}

testdata/sample_policies/project_match_target/policies/templates/gcp_always_violates_v1.yaml

@@ -0,0 +1,57 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: templates.gatekeeper.sh/v1alpha1
+kind: ConstraintTemplate
+metadata:
+  name: gcp-always-violates-v1
+spec:
+  crd:
+    spec:
+      names:
+        kind: GCPAlwaysViolatesConstraintV1
+      validation:
+        openAPIV3Schema:
+          properties: {}
+  targets:
+   validation.gcp.forsetisecurity.org:
+      rego: | #INLINE("validator/always_violates.rego")
+           #
+           # Copyright 2018 Google LLC
+           #
+           # Licensed under the Apache License, Version 2.0 (the "License");
+           # you may not use this file except in compliance with the License.
+           # You may obtain a copy of the License at
+           #
+           #      http://www.apache.org/licenses/LICENSE-2.0
+           #
+           # Unless required by applicable law or agreed to in writing, software
+           # distributed under the License is distributed on an "AS IS" BASIS,
+           # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+           # See the License for the specific language governing permissions and
+           # limitations under the License.
+           #
+           
+           package templates.gcp.GCPAlwaysViolatesConstraintV1
+           
+           import data.validator.gcp.lib as lib
+           
+           deny[{
+           	"msg": message,
+           	"details": metadata,
+           }] {
+           	message := "violates on all resources."
+           	metadata := {"asset": input.asset}
+           }
+           #ENDINLINE

testdata/templates/example_organization_iam_policy.tf

@@ -29,15 +29,5 @@ provider "google" {
 
 resource "google_organization_iam_policy" "policy" {
   org_id = "123456789"
-  policy_data = data.google_iam_policy.admin.policy_data
-}
-
-data "google_iam_policy" "admin" {
-  binding {
-    role = "roles/editor"
-
-    members = [
-      "user:[email protected]",
-    ]
-  }
+  policy_data = "{\"bindings\":[{\"members\":[\"user:[email protected]\"],\"role\":\"roles/editor\"}]}"
 }

testdata/templates/example_project_iam_policy.tf

@@ -29,15 +29,5 @@ provider "google" {
 
 resource "google_project_iam_policy" "project" {
   project     = "{{.Provider.project}}"
-  policy_data = data.google_iam_policy.admin.policy_data
-}
-
-data "google_iam_policy" "admin" {
-  binding {
-    role = "roles/editor"
-
-    members = [
-      "user:[email protected]",
-    ]
-  }
+  policy_data = "{\"bindings\":[{\"members\":[\"user:[email protected]\"],\"role\":\"roles/editor\"}]}"
 }

testdata/templates/example_project_update.json

@@ -0,0 +1,40 @@
+[
+  {
+    "name": "//cloudbilling.googleapis.com/projects/{{.Project.Number}}/billingInfo",
+    "asset_type": "cloudbilling.googleapis.com/ProjectBillingInfo",
+    "ancestry_path": "organization/{{.OrgID}}/project/{{.Project.Number}}",
+    "resource": {
+      "version": "v1",
+      "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/cloudbilling/v1/rest",
+      "discovery_name": "ProjectBillingInfo",
+      "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}",
+      "data": {
+        "billingAccountName": "billingAccounts/{{.Project.BillingAccountName}}",
+        "name": "projects/{{.Project.Number}}/billingInfo",
+        "projectId": "{{.Provider.project}}"
+      }
+    }
+  },
+  {
+    "name": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}",
+    "asset_type": "cloudresourcemanager.googleapis.com/Project",
+    "ancestry_path": "organization/{{.OrgID}}/project/{{.Project.Number}}",
+    "resource": {
+      "version": "v1",
+      "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/v1/rest",
+      "discovery_name": "Project",
+      "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Project.Number}}",
+      "data": {
+        "labels": {
+          "project-label-key-a": "project-label-val-a"
+        },
+        "name": "My New Project",
+        "parent": {
+          "id": "{{.OrgID}}",
+          "type": "organization"
+        },
+        "projectId": "{{.Provider.project}}"
+      }
+    }
+  }
+]

testdata/templates/example_project_update.tf

@@ -0,0 +1,40 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+      version = "~> {{.Provider.version}}"
+    }
+  }
+}
+
+provider "google" {
+  {{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}}
+}
+
+resource "google_project" "my_project" {
+  name = "My New Project"
+  project_id = "{{.Provider.project}}"
+  org_id  = "{{.OrgID}}"
+
+  billing_account = "{{.Project.BillingAccountName}}"
+
+  labels  = {
+    "project-label-key-a" = "project-label-val-a"
+  }
+}