chore: moving and updating sample_infrastructure terraform, adding sa…

.gitignore

@@ -5,6 +5,11 @@
 *.tfstate
 *.tfstate.*
 
+# tf plan files
+*.tfplan*
+*.tfplan
+**/.*tfplan*/*
+
 # Crash log files
 crash.log
 

asset.tf

@@ -0,0 +1,45 @@
+
+# Create a feed that sends notifications about resource updates under a
+# particular folder.
+
+# resource "google_cloud_asset_folder_feed" "folder_feed" {
+#   billing_project  = "content-eng-billing-report"
+#   folder           = data.google_folder.this.folder_id
+#   feed_id          = "observe-asset-updates"
+#   content_type     = "RESOURCE"
+
+#   asset_types = ["aiplatform.googleapis.com.*", "anthos.googleapis.com.*", "apigateway.googleapis.com.*", "apikeys.googleapis.com.*", "appengine.googleapis.com.*", "apps.k8s.io.*", "artifactregistry.googleapis.com.*", "assuredworkloads.googleapis.com.*", "batch.k8s.io.*", "beyondcorp.googleapis.com.*", "bigquery.googleapis.com.*", "bigquerymigration.googleapis.com.*", "bigtableadmin.googleapis.com.*", "cloudbilling.googleapis.com.*", "clouddeploy.googleapis.com.*", "cloudfunctions.googleapis.com.*", "cloudkms.googleapis.com.*", "cloudresourcemanager.googleapis.com.*", "composer.googleapis.com.*", "compute.googleapis.com.*", "connectors.googleapis.com.*", "container.googleapis.com.*", "containerregistry.googleapis.com.*", "dataflow.googleapis.com.*", "dataform.googleapis.com.*", "datafusion.googleapis.com.*", "datamigration.googleapis.com.*", "dataplex.googleapis.com.*", "dataproc.googleapis.com.*", "datastream.googleapis.com.*", "dialogflow.googleapis.com.*", "dlp.googleapis.com.*", "dns.googleapis.com.*", "documentai.googleapis.com.*", "domains.googleapis.com.*", "eventarc.googleapis.com.*", "extensions.k8s.io.*", "file.googleapis.com.*", "firestore.googleapis.com.*", "gameservices.googleapis.com.*", "gkebackup.googleapis.com.*", "gkehub.googleapis.com.*", "healthcare.googleapis.com.*", "iam.googleapis.com.*", "ids.googleapis.com.*", "k8s.io.*", "logging.googleapis.com.*", "managedidentities.googleapis.com.*", "memcache.googleapis.com.*", "metastore.googleapis.com.*", "monitoring.googleapis.com.*", "networkconnectivity.googleapis.com.*", "networking.k8s.io.*", "networkmanagement.googleapis.com.*", "networkservices.googleapis.com.*", "orgpolicy.googleapis.com.*", "osconfig.googleapis.com.*", "privateca.googleapis.com.*", "pubsub.googleapis.com.*", "rbac.authorization.k8s.io.*", "redis.googleapis.com.*", "run.googleapis.com.*", "secretmanager.googleapis.com.*", "servicedirectory.googleapis.com.*", "servicemanagement.googleapis.com.*", "serviceusage.googleapis.com.*", "spanner.googleapis.com.*", "speech.googleapis.com.*", "sqladmin.googleapis.com.*", "storage.googleapis.com.*", "tpu.googleapis.com.*", "transcoder.googleapis.com.*", "vpcaccess.googleapis.com.*", "workflows.googleapis.com.*"]
+
+#   feed_output_config {
+#     pubsub_destination {
+#       topic = "projects/joe-test-proj/topics/observe"
+#     }
+#   }
+
+  # condition {
+  #   expression = <<-EOT
+  #   temporal_asset.deleted &&
+  #   temporal_asset.prior_asset_state == google.cloud.asset.v1.TemporalAsset.PriorAssetState.DOES_NOT_EXIST
+  #   EOT
+  #   title = "created and deleted"
+  #   description = "Send notifications on creation events"
+  # }
+# }
+
+# # The topic where the resource change notifications will be sent.
+# resource "google_pubsub_topic" "feed_output" {
+#   project  = "my-project-name"
+#   name     = "network-updates"
+# }
+
+# # The folder that will be monitored for resource updates.
+# resource "google_folder" "my_folder" {
+#   display_name = "Networking"
+#   parent       = "organizations/123456789"
+# }
+
+# # Find the project number of the project whose identity will be used for sending
+# # the asset change notifications.
+# data "google_project" "project" {
+#   project_id = "my-project-name"
+# }

examples/gcp_create_project/main.tf

@@ -0,0 +1,11 @@
+
+module "gcp_service_proj" {
+  source          = "../../modules/gcp_project"
+  org_id          = var.org_id
+  folder_id       = var.folder_id
+  project_id      = var.project_id
+  project_name    = var.project_id
+  billing_account = var.billing_account
+  project_owners  = var.project_owners
+  #project_editors = ["serviceAccount:[email protected]"]
+}

examples/gcp_create_project/project.auto.tfvars

@@ -0,0 +1,5 @@
+billing_account = "value"
+org_id          = "value"
+folder_id       = "value"
+project_id      = "value"
+project_owners  = ["user:<replace_with_user>@observeinc.com"]

examples/gcp_create_project/variables.tf

@@ -0,0 +1,24 @@
+variable "billing_account" {
+  type = string
+}
+
+variable "org_id" {
+  type        = string
+  description = "Org ID from GCP console"
+}
+
+variable "folder_id" {
+  type        = string
+  description = "GCP folder id to deploy service project"
+}
+
+
+variable "project_id" {
+  type        = string
+  description = "GCP project used as a service/collection project"
+}
+
+variable "project_owners" {
+  description = "Add the user emails of of the project owners"
+  type        = list(string)
+}

examples/gcp_observe_collection_for_folder/ephem-proj.auto.tfvars

@@ -0,0 +1,4 @@
+name_format   = "joe-test-%s"
+project_id    = "joe-test-proj"
+folder_number = "831845457119"
+region        = "us-central1"

examples/gcp_observe_collection_for_folder/ephem-proj.auto.tfvars.back

@@ -0,0 +1,3 @@
+project_id    = "service-proj-391021"
+name_format   = "service-proj-%s"
+folder_number = "12345678900"

examples/gcp_observe_collection_for_folder/main.tf

@@ -0,0 +1,120 @@
+locals {
+  projects = data.google_projects.my_folder_projects.projects
+}
+
+#######################################################################
+# 
+# The Obseverve GCP Collection that creates the PubSub, Log Sinks,
+# and deploys a GCP Cloud Function used to collect Asset Information
+# 
+#######################################################################
+
+module "observe_gcp_collection" {
+  source   = "../../"
+  name     = var.name
+  resource = "folders/${var.folder_number}"
+}
+
+data "google_project" "service_project" {
+  project_id = var.project_id
+}
+
+#####################################################################
+# 
+# Determines all the sibling GCP Projects inside the folder the 
+# service/collection project was deployed
+#
+#####################################################################
+
+data "google_projects" "my_folder_projects" {
+  filter = "parent.id:${data.google_project.service_project.folder_id} lifecycleState:ACTIVE"
+}
+
+###############################################################
+# 
+# This enables all the GCP API Services needed for metrics in
+# each project in the Folder the service/collection
+# Project is deployed.   
+#
+##############################################################
+
+
+module "google_project_service" {
+  for_each = {
+    for index, project in local.projects :
+    project.project_id => project if project.project_id != var.project_id
+  }
+
+  source             = "../../modules/gcp_project_services"
+  project_id         = each.value.project_id
+  services_to_enable = var.metric_services
+}
+
+
+#######################################################################################
+# 
+# This will add all sibling projects that reside in the same 
+# folder as the collection/service project as Metric Montiored Projects.
+# 
+# The result will be: Metrics for all projects flowing through the collection/service
+# project and collected with a single Observe poller.
+# 
+#######################################################################################
+
+resource "google_monitoring_monitored_project" "primary" {
+  for_each = {
+    for index, project in local.projects :
+    project.project_id => project if project.project_id != var.project_id
+  }
+  metrics_scope = var.project_id
+  name          = each.value.project_id
+}
+
+#######################################################################################
+#
+# The following would replace the steps of creating connections to GPC using Observe
+# Pollers inside "Creating the required connections to GCP" 
+# found in https://docs.observeinc.com/en/latest/content/integrations/gcp/gcp.html#id1
+# 
+# The following still requires an Observe Datastream to be created.  The simplest way
+# to accomplish this is by installing the Observe Application for GCP. 
+# 
+# NOTE: YOU NEED TO UNCOMMENT OUT THE Observe PROVIER in versions.tf to use below.
+#
+#######################################################################################
+
+# locals {
+#   workspace  = data.observe_workspace.default
+#   datastream = data.observe_datastream.gcp
+# }
+
+# data "observe_workspace" "default" {
+#   name = "Default"
+# }
+
+# data "observe_datastream" "google" {
+#   workspace = data.observe_workspace.default.oid
+#   name      = "GCP"
+# }
+
+# module "observe_gcp_metrics_poller" {
+#   workspace                        = data.observe_workspace.default
+#   datastream                       = data.observe_datastream.google
+#   source                           = "../../modules/observe_metrics_poller"
+#   project_id                       = var.project_id
+#   name_format                      = "${var.project_id}-poller-%s"
+#   service_account_private_key_json = base64decode(module.observe_gcp_collection.service_account_key.private_key)
+
+#   depends_on = [module.observe_gcp_collection]
+# }
+
+# module "pubsub_poller" {
+#   source                           = "../../modules/observe_pubsub_poller"
+#   workspace                        = local.workspace
+#   datastream                       = local.datastream
+#   name                             = format(var.name_format, "assets-logs")
+#   description                      = "terraform only poller"
+#   project                          = var.project_id
+#   service_account_private_key_json = base64decode(module.observe_gcp_collection.service_account_key.private_key)
+#   subscription                     = module.observe_gcp_collection.subscription.name
+# }

examples/gcp_observe_collection_for_folder/outputs.tf

@@ -0,0 +1,18 @@
+output "subscription" {
+  description = "The Pub/Sub subscription created by this module."
+  value       = module.observe_gcp_collection.subscription
+}
+
+output "service_account_private_key" {
+  description = "A service account key sent to the pollers for Pub/Sub and Cloud Monitoring"
+  value       = base64decode(module.observe_gcp_collection.service_account_key.private_key)
+  sensitive   = true
+}
+
+output "project_info" {
+  value = data.google_project.service_project
+}
+
+output "projects" {
+  value = data.google_projects.my_folder_projects
+}

examples/gcp_observe_collection_for_folder/provider.tf

@@ -0,0 +1,28 @@
+# locals {
+#   domain     = "observeinc.com"
+#   customer   = "126329491179"
+#   user_email = "[email protected]"
+# }
+
+# provider "aws" {
+#   region = "us-west-2"
+# }
+
+# data "aws_secretsmanager_secret" "secret" {
+#   name = format("tf-password-%s-%s", local.domain, local.customer)
+# }
+
+# data "aws_secretsmanager_secret_version" "secret" {
+#   secret_id = data.aws_secretsmanager_secret.secret.id
+# }
+
+# provider "observe" {
+#   customer      = local.customer
+#   domain        = local.domain
+#   user_email    = local.user_email
+#   user_password = data.aws_secretsmanager_secret_version.secret.secret_string
+# }
+
+# provider "google"{
+#   project = "joe-test-proj"
+# }

examples/gcp_observe_collection_for_folder/variables.tf

@@ -0,0 +1,52 @@
+variable "name" {
+  type        = string
+  description = "Name of Observe's GCP Collection"
+  default     = "observe"
+}
+
+variable "project_id" {
+  type        = string
+  description = "GCP project to deploy sample env"
+}
+
+variable "folder_number" {
+  type        = string
+  description = "GCP folder number to deploy sample env"
+}
+
+variable "datastream_name" {
+  type        = string
+  description = "GCP datastream"
+  default     = "GCP"
+}
+
+variable "region" {
+  type        = string
+  description = "GCP region to deploy sample env. Required for Google provider."
+}
+
+variable "name_format" {
+  type        = string
+  description = "Format string to use for infra names."
+}
+
+variable "metric_services" {
+  description = "Default metric service prefixes to poll"
+  type        = list(string)
+  default = [
+    "cloudfunctions.googleapis.com",
+    "cloudasset.googleapis.com",
+    "logging.googleapis.com",
+    "iam.googleapis.com",
+    "monitoring.googleapis.com",
+    "pubsub.googleapis.com",
+    "storage.googleapis.com",
+    "sql-component.googleapis.com",
+    "compute.googleapis.com",
+    "serviceusage.googleapis.com",
+    "servicenetworking.googleapis.com",
+    "container.googleapis.com",
+    "redis.googleapis.com",
+    "run.googleapis.com"
+  ]
+}

examples/gcp_observe_collection_for_folder/versions.tf

@@ -0,0 +1,18 @@
+terraform {
+  required_providers {
+    # observe = {
+    #   source  = "terraform.observeinc.com/observeinc/observe"
+    #   version = "~> 0.13"
+    # }
+    google = {
+      source  = "hashicorp/google"
+      version = "<= 4.67.0"
+    }
+  }
+  required_version = ">= 1.3.0"
+}
+
+# provider "google" {
+#   project = var.project_id
+#   region  = var.region
+# }

examples/gcp_project_deploy_infra_and_observe_collection/ephem-proj-collect.auto.tfvars

@@ -0,0 +1,9 @@
+region     = "us-central1"
+project_id = "project-id"
+
+observe = {
+  customer_id           = "126329491179"
+  otel_datastream_token = "ds1J8gzPachy4fscTzSD:csenknPbvhkV8WHdjVxaShMbf5HgBY6B"
+  host_datastream_token = "ds1oYxW0CSmcWk14uwsX:hUIOgRqLbLZcrCYdNSnfnkJP0TyR_F_-"
+  domain                = "observeinc.com"
+}