Import gcp resources creation code from search-api-v2 repo

terraform/deployments/tfc-aws-config/gcp_search.tf

@@ -0,0 +1,57 @@
+# NOTE: This is used to store the state for this module itself (see `terraform` block above). It was
+# initially created using a local backend, and then migrated to a remote backend.
+resource "tfe_workspace" "meta_workspace" {
+  name        = "search-api-v2-meta"
+  project_id  = var.project_id
+  description = "Meta workspace for cross-environment TF Cloud resources (state backend only)"
+  tag_names   = ["govuk", "search-api-v2"]
+}
+
+resource "tfe_workspace_settings" "meta_workspace_settings" {
+  workspace_id = tfe_workspace.meta_workspace.id
+
+  execution_mode = "local"
+}
+
+module "environment_integration" {
+  source = "./modules/search-api-v2"
+
+  name                         = "integration"
+  google_cloud_billing_account = var.google_cloud_billing_account
+  google_cloud_folder          = var.google_cloud_folder
+  tfc_project_name             = var.tfe_project_name
+  environment_workspace_name   = var.environment_workspace_name
+}
+
+module "environment_staging" {
+  source = "./modules/search-api-v2"
+
+  name                      = "staging"
+  upstream_environment_name = "integration"
+
+  google_cloud_billing_account = var.google_cloud_billing_account
+  google_cloud_folder          = var.google_cloud_folder
+  tfc_project_name             = var.tfe_project_name
+  environment_workspace_name   = var.environment_workspace_name
+}
+
+module "environment_production" {
+  source = "./modules/search-api-v2"
+
+  name                      = "production"
+  upstream_environment_name = "staging"
+
+  google_cloud_billing_account = var.google_cloud_billing_account
+  google_cloud_folder          = var.google_cloud_folder
+  tfc_project_name             = var.tfe_project_name
+  environment_workspace_name   = var.environment_workspace_name
+
+
+  # NOTE: There are limits on the Google side on how high we are permitted to set these quotas. If
+  # you attempt to increase these beyond the ceiling, a `COMMON_QUOTA_CONSUMER_OVERRIDE_TOO_HIGH`
+  # error will be raised (including some metadata that should tell you what the current ceiling is)
+  # and you will need to manually request a quota increase from Google through the console first
+  # (see the environment module for the exact quota names you need to request increases for).
+  discovery_engine_quota_search_requests_per_minute = 5000
+  discovery_engine_quota_documents                  = 2000000
+}

terraform/deployments/tfc-aws-config/modules/search-api-v2/main.tf

@@ -0,0 +1,145 @@
+terraform {
+  required_providers {
+    tfe = {
+      source  = "hashicorp/tfe"
+      version = "~> 0.55.0"
+    }
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 5.20"
+    }
+    # required for `google_service_usage_consumer_quota_override` resources
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 5.20"
+    }
+  }
+
+  required_version = "~> 1.7"
+}
+
+locals {
+  display_name = title(var.name)
+}
+
+resource "google_project" "environment_project" {
+  name       = "Search API V2 ${local.display_name}"
+  project_id = "search-api-v2-${var.name}"
+
+  folder_id       = var.google_cloud_folder
+  billing_account = var.google_cloud_billing_account
+
+  labels = {
+    "programme"         = "govuk"
+    "team"              = "govuk-search-improvement"
+    "govuk_environment" = var.name
+  }
+}
+
+resource "google_project_iam_member" "environment_project_owner" {
+  project = google_project.environment_project.project_id
+  role    = "roles/owner"
+
+  member = "group:[email protected]"
+}
+
+resource "google_project_service" "api_service" {
+  for_each = var.google_cloud_apis
+
+  project                    = google_project.environment_project.project_id
+  service                    = each.value
+  disable_dependent_services = true
+}
+
+resource "google_service_usage_consumer_quota_override" "discoveryengine_search_requests" {
+  provider = google-beta
+  project  = google_project.environment_project.project_id
+
+  service = "discoveryengine.googleapis.com"
+  metric  = urlencode("discoveryengine.googleapis.com/search_requests")
+  force   = true
+
+  # limit is equivalent to `unit` field when making a GET request against the metric, but without
+  # leading `1/` and without curly braces
+  limit          = urlencode("/min/project")
+  override_value = var.discovery_engine_quota_search_requests_per_minute
+}
+
+resource "google_service_usage_consumer_quota_override" "discoveryengine_documents" {
+  provider = google-beta
+  project  = google_project.environment_project.project_id
+
+  service = "discoveryengine.googleapis.com"
+  metric  = urlencode("discoveryengine.googleapis.com/documents")
+  force   = true
+
+  # limit is equivalent to `unit` field when making a GET request against the metric, but without
+  # leading `1/` and without curly braces
+  limit          = urlencode("/project")
+  override_value = var.discovery_engine_quota_documents
+}
+
+data "tfe_oauth_client" "github" {
+  organization     = var.tfc_organization_name
+  service_provider = "github"
+}
+
+# Set up Workload Identity Federation between Terraform Cloud and GCP
+# see https://github.com/hashicorp/terraform-dynamic-credentials-setup-examples
+resource "google_iam_workload_identity_pool" "tfc_pool" {
+  project                   = google_project.environment_project.project_id
+  workload_identity_pool_id = "terraform-cloud-id-pool"
+
+  display_name = "Terraform Cloud ID Pool"
+  description  = "Pool to enable access to project resources for Terraform Cloud"
+}
+
+resource "google_iam_workload_identity_pool_provider" "tfc_provider" {
+  project                            = google_project.environment_project.project_id
+  workload_identity_pool_id          = google_iam_workload_identity_pool.tfc_pool.workload_identity_pool_id
+  workload_identity_pool_provider_id = "terraform-cloud-provider-oidc"
+
+  display_name = "Terraform Cloud OIDC Provider"
+  description  = "Configures Terraform Cloud as an external identity provider for this project"
+
+  attribute_mapping = {
+    "google.subject"                        = "assertion.sub",
+    "attribute.aud"                         = "assertion.aud",
+    "attribute.terraform_run_phase"         = "assertion.terraform_run_phase",
+    "attribute.terraform_project_id"        = "assertion.terraform_project_id",
+    "attribute.terraform_project_name"      = "assertion.terraform_project_name",
+    "attribute.terraform_workspace_id"      = "assertion.terraform_workspace_id",
+    "attribute.terraform_workspace_name"    = "assertion.terraform_workspace_name",
+    "attribute.terraform_organization_id"   = "assertion.terraform_organization_id",
+    "attribute.terraform_organization_name" = "assertion.terraform_organization_name",
+    "attribute.terraform_run_id"            = "assertion.terraform_run_id",
+    "attribute.terraform_full_workspace"    = "assertion.terraform_full_workspace",
+  }
+
+  oidc {
+    issuer_uri = "https://${var.tfc_hostname}"
+  }
+
+  attribute_condition = "assertion.sub.startsWith(\"organization:${var.tfc_organization_name}:project:${var.tfc_project_name}:workspace:${var.environment_workspace_name}\")"
+}
+
+resource "google_service_account" "tfc_service_account" {
+  project = google_project.environment_project.project_id
+
+  account_id   = "tfc-service-account"
+  display_name = "Terraform Cloud Service Account"
+  description  = "Used by Terraform Cloud to manage resources in this project through Workload Identity Federation"
+}
+
+resource "google_service_account_iam_member" "tfc_service_account_member" {
+  service_account_id = google_service_account.tfc_service_account.name
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.tfc_pool.name}/*"
+}
+
+resource "google_project_iam_member" "tfc_project_member" {
+  project = google_project.environment_project.project_id
+
+  role   = "roles/owner"
+  member = "serviceAccount:${google_service_account.tfc_service_account.email}"
+}

terraform/deployments/tfc-aws-config/modules/search-api-v2/variables.tf

@@ -0,0 +1,81 @@
+variable "name" {
+  type        = string
+  description = "A short name for this environment (used in resource IDs)"
+}
+
+variable "google_cloud_folder" {
+  type        = string
+  description = "The ID of the Google Cloud folder to create projects under"
+}
+
+variable "google_cloud_billing_account" {
+  type        = string
+  description = "The ID of the Google Cloud billing account to associate projects with"
+}
+
+variable "google_cloud_apis" {
+  type        = set(string)
+  description = "The Google Cloud APIs to enable for the project"
+  default = [
+    # Required to be able to manage resources using Terraform
+    "cloudresourcemanager.googleapis.com",
+    # Required to set up service accounts and manage dynamic credentials
+    "iam.googleapis.com",
+    "iamcredentials.googleapis.com",
+    "sts.googleapis.com",
+    # Required for Discovery Engine
+    "discoveryengine.googleapis.com",
+    # Required for event data pipeline
+    "bigquery.googleapis.com",
+    "bigquerystorage.googleapis.com",
+    "storage.googleapis.com",
+    "cloudbuild.googleapis.com",
+    "artifactregistry.googleapis.com",
+    "cloudfunctions.googleapis.com",
+    "run.googleapis.com",
+    "cloudscheduler.googleapis.com",
+    # Required for observability
+    "logging.googleapis.com",
+    "monitoring.googleapis.com",
+  ]
+}
+
+variable "discovery_engine_quota_search_requests_per_minute" {
+  type        = number
+  description = "The maximum number of search requests per minute for the Discovery Engine"
+  default     = 250
+}
+
+variable "discovery_engine_quota_documents" {
+  type        = number
+  description = "The maximum number of documents across Discovery Engine datastores"
+  default     = 1000000
+}
+
+variable "upstream_environment_name" {
+  type        = string
+  description = "The name of the upstream environment, if any (used to wait for a successful apply on a 'lower' environment before applying this one)"
+  default     = null
+}
+
+variable "tfc_hostname" {
+  type        = string
+  description = "The hostname of the Terraform Cloud/Enterprise instance to use"
+  default     = "app.terraform.io"
+}
+
+variable "tfc_organization_name" {
+  type        = string
+  description = "The name of the Terraform Cloud/Enterprise organization to use"
+  default     = "govuk"
+}
+
+variable "tfc_project_name" {
+  type        = string
+  description = "The  name of the overarching terraform cloud project for all workspaces"
+}
+
+variable "environment_workspace_name" {
+  type        = string
+  description = "Provisions search-api-v2 Discovery Engine resources for the environment"
+}

terraform/deployments/tfc-aws-config/variables.tf

@@ -14,3 +14,28 @@ variable "tfc_organization_name" {
   default     = "govuk"
   description = "The name of the Terraform Cloud organization"
 }
+
+variable "google_cloud_folder" {
+  type        = string
+  description = "The ID of the Google Cloud folder to create projects under"
+}
+
+variable "google_cloud_billing_account" {
+  type        = string
+  description = "The ID of the Google Cloud billing account to associate projects with"
+}
+
+variable "project_id" {
+  type        = string
+  description = "The ID of the overarching terraform cloud project for all workspaces"
+}
+
+variable "tfe_project_name" {
+  type        = string
+  description = "The  name of the overarching terraform cloud project for all workspaces"
+}
+
+variable "environment_workspace_name" {
+  type        = string
+  description = "Provisions search-api-v2 Discovery Engine resources for the environment"
+}