From 06b405f9bcc500d497077833ece9eb7ae52197a2 Mon Sep 17 00:00:00 2001 From: Charlie Wang Date: Tue, 22 Aug 2023 14:01:59 +0000 Subject: [PATCH 1/5] remove unnecessary dependency declarations --- infrastructure/terraform/main.tf | 6 ------ .../terraform/modules/feature-store/main.tf | 11 ----------- 2 files changed, 17 deletions(-) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index d5334325..a6249895 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -115,7 +115,6 @@ module "feature_store" { project_id = var.feature_store_project_id depends_on = [ - local_file.feature_store_configuration, null_resource.generate_sql_queries ] } @@ -126,7 +125,6 @@ module "pipelines" { poetry_run_alias = local.poetry_run_alias count = var.deploy_pipelines ? 1 : 0 depends_on = [ - local_file.feature_store_configuration, null_resource.poetry_install ] } @@ -139,8 +137,4 @@ module "activation" { ga4_measurement_id = var.ga4_measurement_id ga4_measurement_secret = var.ga4_measurement_secret count = var.deploy_activation ? 1 : 0 - depends_on = [ - local_file.feature_store_configuration, - null_resource.poetry_install - ] } \ No newline at end of file diff --git a/infrastructure/terraform/modules/feature-store/main.tf b/infrastructure/terraform/modules/feature-store/main.tf index cf6305b9..5ee6cd31 100644 --- a/infrastructure/terraform/modules/feature-store/main.tf +++ b/infrastructure/terraform/modules/feature-store/main.tf @@ -54,17 +54,6 @@ module "project_services" { ] } -resource "null_resource" "check_apis_" { - provisioner "local-exec" { - command = "${var.poetry_cmd} install" - working_dir = local.source_root_dir - } - depends_on = [ - module.project_services.project_id, - null_resource.poetry_install - ] -} - resource "google_artifact_registry_repository" "cloud_builder_repository" { project = local.feature_store_project_id location = var.region From 19cef64f5c190d37d24664ee20417ee2bc4307f6 Mon Sep 17 00:00:00 2001 From: Charlie Wang Date: Wed, 23 Aug 2023 11:59:32 +0000 Subject: [PATCH 2/5] automatic terraform validation --- .github/workflows/terraform.yaml | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/terraform.yaml diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml new file mode 100644 index 00000000..420e876d --- /dev/null +++ b/.github/workflows/terraform.yaml @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +name: Terraform + +on: [push, pull_request] + +jobs: + terraform: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v3 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + - name: Terraform init + working-directory: infrastructure/terraform + # Don't initialize the backend because we may not have any remote backend configuration available + run: terraform init -backend=false -input=false + - name: Terraform validate + working-directory: infrastructure/terraform + run: terraform validate +... From 82c95b9a6a6edcf6c331fbf28d31243e85f2629a Mon Sep 17 00:00:00 2001 From: Charlie Wang Date: Tue, 26 Sep 2023 17:16:31 +0000 Subject: [PATCH 3/5] add triggers on file content --- infrastructure/terraform/main.tf | 34 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 6f18b34f..3b502d95 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -57,6 +57,14 @@ locals { config_file_name = "config" poetry_run_alias = "${var.poetry_cmd} run" mds_dataset_suffix = var.create_prod_environment ? "prod" : var.create_dev_environment ? "dev" : "staging" + + generated_sql_queries_directory_path = "${local.source_root_dir}/sql/query" + generated_sql_queries_fileset = [for f in fileset(local.generated_sql_queries_directory_path, "*.sql") : "${local.generated_sql_queries_directory_path}/${f}"] + generated_sql_queries_content_hash = sha512(join("", [for f in local.generated_sql_queries_fileset : fileexists(f) ? filesha512(f) : sha512("file-not-found")])) + + generated_sql_procedures_directory_path = "${local.source_root_dir}/sql/procedure" + generated_sql_procedures_fileset = [for f in fileset(local.generated_sql_procedures_directory_path, "*.sql") : "${local.generated_sql_procedures_directory_path}/${f}"] + generated_sql_procedures_content_hash = sha512(join("", [for f in local.generated_sql_procedures_fileset : fileexists(f) ? filesha512(f) : sha512("file-not-found")])) } resource "local_file" "feature_store_configuration" { @@ -84,28 +92,36 @@ resource "null_resource" "poetry_install" { resource "null_resource" "generate_sql_queries" { triggers = { + create_command = <<-EOT + ${local.poetry_run_alias} inv apply-env-variables-queries --env-name=${local.config_file_name} + ${local.poetry_run_alias} inv apply-env-variables-procedures --env-name=${local.config_file_name} + EOT + + destroy_command = <<-EOT + rm sql/query/*.sql + rm sql/procedure/*.sql + EOT + working_dir = local.source_root_dir + + source_contents_hash = local_file.feature_store_configuration.content_sha512 + destination_queries_hash = local.generated_sql_queries_content_hash + destination_procedures_hash = local.generated_sql_procedures_content_hash } provisioner "local-exec" { - command = <<-EOT - ${local.poetry_run_alias} inv apply-env-variables-queries --env-name=${local.config_file_name} - ${local.poetry_run_alias} inv apply-env-variables-procedures --env-name=${local.config_file_name} - EOT + when = create + command = self.triggers.create_command working_dir = self.triggers.working_dir } provisioner "local-exec" { when = destroy - command = <<-EOT - rm sql/query/*.sql - rm sql/procedure/*.sql - EOT + command = self.triggers.destroy_command working_dir = self.triggers.working_dir } depends_on = [ - local_file.feature_store_configuration, null_resource.poetry_install ] } From da90621330e8de896da6813f95cfbf9be4f5d55c Mon Sep 17 00:00:00 2001 From: Charlie Wang Date: Tue, 26 Sep 2023 17:38:35 +0000 Subject: [PATCH 4/5] ignore error when files does not exist --- infrastructure/terraform/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 3b502d95..8c42f2b8 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -98,8 +98,8 @@ resource "null_resource" "generate_sql_queries" { EOT destroy_command = <<-EOT - rm sql/query/*.sql - rm sql/procedure/*.sql + rm -f sql/query/*.sql + rm -f sql/procedure/*.sql EOT working_dir = local.source_root_dir From a838c69270dddb2a48d82de19f2d8330c035d6bb Mon Sep 17 00:00:00 2001 From: Charlie Wang Date: Thu, 28 Sep 2023 09:44:16 +0000 Subject: [PATCH 5/5] add trigger on pyproject.toml configuration for poetry install --- infrastructure/terraform/main.tf | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 8c42f2b8..ba156ce8 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -58,6 +58,9 @@ locals { poetry_run_alias = "${var.poetry_cmd} run" mds_dataset_suffix = var.create_prod_environment ? "prod" : var.create_dev_environment ? "dev" : "staging" + project_toml_file_path = "${local.source_root_dir}/pyproject.toml" + project_toml_content_hash = filesha512(local.project_toml_file_path) + generated_sql_queries_directory_path = "${local.source_root_dir}/sql/query" generated_sql_queries_fileset = [for f in fileset(local.generated_sql_queries_directory_path, "*.sql") : "${local.generated_sql_queries_directory_path}/${f}"] generated_sql_queries_content_hash = sha512(join("", [for f in local.generated_sql_queries_fileset : fileexists(f) ? filesha512(f) : sha512("file-not-found")])) @@ -83,8 +86,14 @@ resource "local_file" "feature_store_configuration" { } resource "null_resource" "poetry_install" { + triggers = { + create_command = "${var.poetry_cmd} install" + source_contents_hash = local.project_toml_content_hash + } + provisioner "local-exec" { - command = "${var.poetry_cmd} install" + when = create + command = self.triggers.create_command working_dir = local.source_root_dir } } @@ -104,6 +113,8 @@ resource "null_resource" "generate_sql_queries" { working_dir = local.source_root_dir + poetry_installed = null_resource.poetry_install.id + source_contents_hash = local_file.feature_store_configuration.content_sha512 destination_queries_hash = local.generated_sql_queries_content_hash destination_procedures_hash = local.generated_sql_procedures_content_hash @@ -120,10 +131,6 @@ resource "null_resource" "generate_sql_queries" { command = self.triggers.destroy_command working_dir = self.triggers.working_dir } - - depends_on = [ - null_resource.poetry_install - ] } module "feature_store" {