Merge pull request #131 from GoogleCloudPlatform/global-vars

Adding global vars feature with CLI to replace globally

README.md

@@ -127,6 +127,10 @@ sb deploy
 - See other deployment options in [solutions_builder/modules](solutions_builder/modules).
 
 
+## CLI Usage
+
+For more information on how to use the CLI, please refer to the [CLI_USAGE.md](docs/CLI_USAGE.md).
+
 ## Additional Documentations
 
 You can find more documentations in [docs](docs) folder. In a nutshell, it covers the following:

docs/CLI_USAGE.md

@@ -0,0 +1,56 @@
+# Solution Builder CLI Usage
+
+## Global Variables
+
+### Set and apply a global variable
+
+You can add an anchor to specify where a variable to apply in the solution folder.
+
+For example, in a YAML file:
+```yaml
+detail:
+  PROJECT_ID: old-project-id # sb-var:project_id
+  OTHER: something
+```
+- It sets a variable named "project_id" as the anchor for this "PROJECT_ID" property at the same line in the YAML file.
+
+Then, you can run the following to replace the variable value:
+```
+$ sb vars set project_id new-project-id
+```
+- This will find all occurrence of the `sb-var:project_id` anchors in your folder, and replace with the new value "new-project-id"
+
+The YAML file will become:
+
+```yaml
+detail:
+  PROJECT_ID: new-project-id # sb-var:project_id
+  OTHER: something
+```
+
+### Apply all existing global variables
+
+You can apply all existing global variables to files with corresponding variable anchors.
+
+For example, in a YAML file:
+```yaml
+detail:
+  PROJECT_ID: old-project-id # sb-var:project_id
+  OTHER: something
+```
+
+And in the `sb.yaml` file in your project root folder:
+```yaml
+global_variables:
+  project_id: MY_PROJECT_ID
+  project_name: core-solution-services
+  project_number: MY_PROJECT_NUMBER
+  gcp_region: us-central1
+```
+
+Run the following to apply all these values to existing variables in all files.
+```
+$ sb vars apply-all
+```
+
+All files with the corresponding anchors will be updated altogether.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "solutions-builder"
-version = "1.17.13"
+version = "1.17.14"
 description = "A solution framework to generate a project with built-in structure and modules"
 authors = ["Jon Chen <[email protected]>"]
 license = "Apache"

solutions_builder/cli/cli.py

@@ -23,6 +23,7 @@
 from .infra import infra_app
 from .template import template_app
 from .set import set_app
+from .vars import vars_app
 from .cli_utils import *
 
 __version__ = importlib.metadata.version("solutions-builder")
@@ -48,6 +49,9 @@
 app.add_typer(set_app,
               name="set",
               help="Set properties to an existing solution folder.")
+app.add_typer(vars_app,
+              name="vars",
+              help="Set variables in an existing solutions-builder folder.")
 
 
 # Create a new solution

solutions_builder/cli/vars.py

@@ -0,0 +1,168 @@
+# Copyright 2023 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typer
+import pathlib
+import re
+import jinja2
+from typing import Optional
+from typing_extensions import Annotated
+from copier import run_auto
+from .cli_utils import *
+
+vars_app = typer.Typer()
+
+INCLUDE_PATTERNS = [
+    "*.yaml", "*.yml", "*.env", "*.tfvars", "*.tf", "*.sh", "*.md"
+]
+EXCLUDE_PATTERNS = ["**/.terraform/**/*.*", "**/node_modules/**/*.*", "**/.venv/**/*.*"]
+
+# Replace a variable with a given text content.
+def replace_var_to_template(var_name, text, custom_template=False, debug=False):
+  # This pattern matches lines with sb-var anchor in the comment at the end.
+  # For example:
+  #   PROJECT_ID: 12345          # sb-var:project_id
+  #   GCP_REGION = "us-central1" # sb-var:gcp_region
+  match_pattern = f"^([^\\r]*[:|=]\\s*)([\"\']?)([^\"^\']*)([\"\']?)\\s*#\\s*sb-var:{var_name}"
+
+  # This output patterh print the jinja2 template for the specific variable name.
+  # For example:
+  #   PROJECT_ID: {{project_id}} # sb-var:project_id
+  output_pattern = f"\\1\\2{{{{{var_name}}}}}\\4 # sb-var:{var_name}"
+
+  # In addition, if custom_template is true, the pattern will extend to the custom
+  # template string at the end of the anchor. For example:
+  #   BUCKET_NAME: my-project-bucket # sb-var:project_id:{{project_id}}-bucket
+  if custom_template:
+    match_pattern = match_pattern + ":(.*)"
+    output_pattern = f"\\1\\2\\5\\4 # sb-var:{var_name}:\\5"
+
+  if debug:
+    print(f"match_pattern = {match_pattern}")
+
+  # Replace with regex pattern and returns new text and count of changes.
+  text, count = re.subn(match_pattern, output_pattern, text)
+  return (text, count)
+
+def restore_template_in_comment(var_name, var_value, text):
+  # Restore jinja2 variables in the custom content comment.
+  match_pattern = f"(#\\s*sb-var:{var_name}:)(.*){var_value}(.*)"
+  output_pattern = f"\\1\\2{{{{{var_name}}}}}\\3"
+
+  text, count = re.subn(match_pattern, output_pattern, text)
+  return (text, count)
+
+
+def replace_var_to_value(var_name, var_value, text):
+  overall_count = 0
+
+  # Replace simple variable pattern with sb-var:var_name
+  text, count = replace_var_to_template(var_name, text)
+  overall_count += count
+
+  # Replace custom-content variable pattern with sb-var:var_name:custom_template
+  text, count = replace_var_to_template(var_name, text, custom_template=True)
+  overall_count += count
+
+  # Update variables using Jinja2
+  jinja_env = jinja2.Environment()
+  text = text.replace("# copier:raw", "# copier:raw{% raw %}")
+  text = text.replace("# copier:endraw", "# copier:endraw{% endraw %}")
+  template = jinja_env.from_string(text)
+
+  # Set vars data for jinja
+  data = {}
+  data[var_name] = var_value
+
+  # Apply variable values using Jinja2
+  text = template.render(**data)
+
+  # Restore vars to template in comment.
+  text, count = restore_template_in_comment(var_name, var_value, text)
+
+  return (text, overall_count)
+
+# Apply a specific variable with a new value.
+def apply_var_to_folder(solution_path, var_name, var_value):
+  file_set = set()
+
+  # Adding includes.
+  for pattern in INCLUDE_PATTERNS:
+    file_list = pathlib.Path(solution_path).rglob(f"{pattern}")
+    file_set.update(set([str(x) for x in file_list]))
+
+  # Removing excludes.
+  for pattern in EXCLUDE_PATTERNS:
+    file_list = pathlib.Path(solution_path).rglob(f"{pattern}")
+    file_set = file_set - set([str(x) for x in file_list])
+
+  modified_files_list = []
+  for filename in list(file_set):
+    with open(filename, "r") as file:
+      # Replace variable
+      filedata = file.read()
+      filedata, count = replace_var_to_value(var_name, var_value, filedata)
+      filedata = filedata + "\n"
+
+      if count > 0:
+        modified_files_list.append(filename)
+
+      # If there's any changes, write back to the original file.
+      if count > 0:
+        with open(filename, "w") as file:
+          file.write(filedata)
+
+  return modified_files_list
+
+# CLI command for `sb vars set <var_name> <var_value>`
+@vars_app.command(name="set")
+def set_var(
+    var_name,
+    var_value,
+    solution_path: Annotated[Optional[str], typer.Argument()] = ".",
+):
+  validate_solution_folder(solution_path)
+
+  # Update to the root sb.yaml
+  root_st_yaml = read_yaml(f"{solution_path}/sb.yaml")
+  global_variables = root_st_yaml.get("global_variables", {})
+  global_variables[var_name] = var_value
+  root_st_yaml["global_variables"] = global_variables
+  write_yaml(f"{solution_path}/sb.yaml", root_st_yaml)
+
+  # Apply vars to individual files
+  filenames = apply_var_to_folder(solution_path, var_name, var_value)
+
+  print_success(
+      f"Complete. {len(filenames)} files updated.\n"
+  )
+
+# CLI command for `sb vars apply-all`
+@vars_app.command(name="apply-all")
+def apply_all(
+    solution_path: Annotated[Optional[str], typer.Argument()] = ".",
+):
+  validate_solution_folder(solution_path)
+
+  # Update to the root sb.yaml
+  root_st_yaml = read_yaml(f"{solution_path}/sb.yaml")
+  global_variables = root_st_yaml.get("global_variables", {})
+  filenames = []
+
+  for var_name, var_value in global_variables.items():
+    filenames += apply_var_to_folder(solution_path, var_name, var_value)
+
+  print_success(
+      f"Complete. {len(filenames)} files updated.\n"
+  )

solutions_builder/cli/vars_test.py

@@ -0,0 +1,65 @@
+"""
+Copyright 2023 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import pytest
+from .vars import *
+
+def test_replace_var_to_template():
+  # Test with equals and quotes
+  text = "project_id = \"not-replaced-yet\" # sb-var:project_id"
+  text = replace_var_to_template("project_id", text)
+  assert text == "project_id = \"{{project_id}}\" # sb-var:project_id"
+
+  # Test with equals without quotes
+  text = "project_id = test # sb-var:project_id"
+  text = replace_var_to_template("project_id", text)
+  assert text == "project_id = {{project_id}} # sb-var:project_id"
+
+  # Test with comma without quotes
+  text = "project_id: test # sb-var:project_id"
+  text = replace_var_to_template("project_id", text)
+  assert text == "project_id: {{project_id}} # sb-var:project_id"
+
+def test_replace_var_to_custom_template():
+  # Test with equals and quotes
+  text = "project_id = \"not-replaced-yet\" # sb-var:project_id:prefix-{{project_id}}-suffix"
+  text = replace_var_to_template("project_id", text, custom_template=True)
+  assert text == "project_id = \"prefix-{{project_id}}-suffix\" # sb-var:project_id:prefix-{{project_id}}-suffix"
+
+def test_replace_var_to_value():
+  text = "project_id = \"not-replaced-yet\" # sb-var:project_id"
+  text = replace_var_to_value("project_id", "fake-id", text)
+  assert text == "project_id = \"fake-id\" # sb-var:project_id"
+
+  text = "  PROJECT_ID: not-replaced-yet # sb-var:project_id"
+  text = replace_var_to_value("project_id", "fake-id", text)
+  assert text == "  PROJECT_ID: fake-id # sb-var:project_id"
+
+def test_replace_with_multiple_lines():
+  text = """
+    env:
+      PROJECT_ID: not-replaced-yet # sb-var:project_id
+    """
+  text = replace_var_to_value("project_id", "fake-id", text)
+  assert text == """
+    env:
+      PROJECT_ID: fake-id # sb-var:project_id
+    """
+
+def test_replace_var_to_value_custom_template():
+  text = "project_id = \"not-replaced-yet\" # sb-var:project_id:prefix-{{project_id}}-suffix"
+  text = replace_var_to_value("project_id", "fake-id", text)
+  assert text == "project_id = \"prefix-fake-id-suffix\" # sb-var:project_id:prefix-{{project_id}}-suffix"

solutions_builder/modules/task_dispatch_service/terraform/stages/3-task-dispatch/terraform.tfvars

@@ -19,8 +19,8 @@
 # Note for modules: Jinja variables are defined in copier.yaml.
 
 
-project_id                 = "{{project_id}}"
-region                     = "{{gcp_region}}"
+project_id                 = "{{project_id}}" # sb-var:project_id
+region                     = "{{gcp_region}}" # sb-var:gcp_region
 task_pubsub_topic          = "{{task_pubsub_topic}}"
 message_retention_duration = "{{pubsub_message_retention_duration}}"
 eventarc_trigger_name      = "{{eventarc_trigger_name}}"

solutions_builder/modules/terraform_gke/terraform/stages/{{terraform_stage_name}}/backend.tf

@@ -17,7 +17,7 @@
 
 terraform {
   backend "gcs" {
-    bucket = "{{project_id}}-tfstate"
+    bucket = "{{project_id}}-tfstate" # sb-var:project_id:{{project_id}}-tfstate
     prefix = "stage/{{terraform_stage_name}}"
   }
 }

solutions_builder/modules/terraform_gke/terraform/stages/{{terraform_stage_name}}/terraform.tfvars

@@ -1,5 +1,5 @@
-project_id           = "{{project_id}}"
-region               = "{{gcp_region}}"
+project_id           = "{{project_id}}" # sb-var:project_id
+region               = "{{gcp_region}}" # sb-var:gcp_region
 kubernetes_version   = "{{kubernetes_version}}"
 node_machine_type    = "{{node_machine_type}}"
 cluster_name         = "{{cluster_name}}"

solutions_builder/modules/terraform_gke_ingress/terraform/stages/{{terraform_stage_name}}/terraform.tfvars

@@ -1,5 +1,5 @@
-project_id                = "{{project_id}}"
-region                    = "{{gcp_region}}"
+project_id                = "{{project_id}}" # sb-var:project_id
+region                    = "{{gcp_region}}" # sb-var:gcp_region
 cluster_name              = "{{cluster_name}}"
 cluster_external_endpoint = "{{cluster_external_endpoint}}"
 cluster_ca_certificate    = "{{('masterAuth.clusterCaCertificate', cluster_name, gcp_region) | get_cluster_value}}"

solutions_builder/modules/terraform_httplb_cloudrun/terraform/stages/3-httplb-cloudrun/terraform.tfvars

@@ -1,5 +1,5 @@
-project_id        = "{{project_id}}"
-region            = "{{gcp_region}}"
+project_id        = "{{project_id}}" # sb-var:project_id
+region            = "{{gcp_region}}" # sb-var:gcp_region
 domains           = "{{domains}}"
 cloudrun_services = "{{cloudrun_services}}"
 loadbalancer_name = "{{loadbalancer_name}}"

solutions_builder/template_root/.github/workflows/deployment_cloudrun_dev.yaml

@@ -16,7 +16,7 @@ env:
   REGION: {{gcp_region}}
   ENV: dev
 
-# copoier:raw {% raw %}
+# copier:raw {% raw %}
 jobs:
   build_common:
     name: Build Common image
@@ -85,4 +85,4 @@ jobs:
           _IMAGE="${{ matrix.target-folder[1] }}",\
           _SERVICE_ACCOUNT="deployment-${{ env.ENV }}@${{ env.PROJECT_ID }}.iam.gserviceaccount.com"
 
-# copoier:endraw {% endraw %}
+# copier:endraw {% endraw %}