diff --git a/.docker/forge-github-app-register/app.py b/.docker/forge-github-app-register/app.py index 62edd6c8..00ce5247 100644 --- a/.docker/forge-github-app-register/app.py +++ b/.docker/forge-github-app-register/app.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import io import json import os diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3df0a1c6..7fe19310 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ repos: # General Hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-yaml always_run: true @@ -32,8 +32,6 @@ repos: always_run: true - id: debug-statements always_run: true - - id: fix-encoding-pragma - always_run: true - id: name-tests-test always_run: true - id: destroyed-symlinks @@ -100,9 +98,14 @@ repos: args: - --ignore=E501 + - repo: https://github.com/asottile/pyupgrade + rev: v3.20.0 + hooks: + - id: pyupgrade + # JSON Schema Hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.2 + rev: 0.33.3 hooks: - id: check-github-workflows - id: check-github-actions @@ -113,10 +116,13 @@ repos: - repo: https://github.com/gruntwork-io/pre-commit rev: v0.1.30 hooks: - - id: terragrunt-hclfmt + - id: terragrunt-hcl-fmt + - id: tofu-fmt + - id: tofu-validate + - id: packer-validate - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.99.5 + rev: v1.100.0 hooks: - id: terraform_fmt args: @@ -125,7 +131,7 @@ repos: - id: terraform_tflint args: - --hook-config=--tf-path=tofu - exclude: (modules/integrations/splunk_cloud_data_manager|modules/infra/forge_subscription|modules/integrations/splunk_secrets/|modules/infra/eks_secrets/) + exclude: (modules/integrations/splunk_cloud_data_manager|modules/infra/forge_subscription|modules/integrations/splunk_secrets/) always_run: true - id: terraform_validate args: @@ -140,7 +146,7 @@ repos: args: - --hook-config=--tf-path=tofu - --args=--config=.terraform-docs.yml - exclude: (modules/integrations/splunk_cloud_data_manager|modules/infra/forge_subscription|modules/integrations/splunk_secrets/|modules/infra/eks_secrets/) + exclude: (modules/integrations/splunk_cloud_data_manager|modules/infra/forge_subscription|modules/integrations/splunk_secrets/) # Security Hooks - repo: https://github.com/gitleaks/gitleaks @@ -151,6 +157,6 @@ repos: # Ansible Hooks - repo: https://github.com/ansible-community/ansible-lint.git - rev: v25.7.0 + rev: v25.8.1 hooks: - id: ansible-lint diff --git a/modules/infra/eks_secrets/README.md b/modules/infra/eks_secrets/README.md deleted file mode 100644 index 03c54002..00000000 --- a/modules/infra/eks_secrets/README.md +++ /dev/null @@ -1,42 +0,0 @@ - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.9.1 | -| [aws](#requirement\_aws) | ~> 5.90 | -| [time](#requirement\_time) | ~> 0.13 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | 5.99.1 | -| [time](#provider\_time) | 0.13.1 | - -## Modules - -No modules. - -## Resources - -| Name | Type | -|------|------| -| [aws_secretsmanager_secret.cicd_secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | -| [aws_secretsmanager_secret_version.cicd_secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | -| [time_sleep.wait_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | -| [aws_secretsmanager_random_password.secret_seeds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_random_password) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [aws\_profile](#input\_aws\_profile) | AWS profile to use. | `string` | n/a | yes | -| [aws\_region](#input\_aws\_region) | Default AWS region. | `string` | n/a | yes | -| [default\_tags](#input\_default\_tags) | A map of tags to apply to resources. | `map(string)` | n/a | yes | -| [tags](#input\_tags) | A map of tags to apply to resources. | `map(string)` | n/a | yes | - -## Outputs - -No outputs. - diff --git a/modules/infra/eks_secrets/backend.tf b/modules/infra/eks_secrets/backend.tf deleted file mode 100644 index da35a754..00000000 --- a/modules/infra/eks_secrets/backend.tf +++ /dev/null @@ -1,4 +0,0 @@ -terraform { - # Intentionally empty. Re-use what's provided by the root module. - backend "s3" {} -} diff --git a/modules/infra/eks_secrets/main.tf b/modules/infra/eks_secrets/main.tf deleted file mode 100644 index 976393a3..00000000 --- a/modules/infra/eks_secrets/main.tf +++ /dev/null @@ -1,105 +0,0 @@ -locals { - cicd_secrets_prefix = "/cicd/common" - - secrets = [ - - { - name = "${local.cicd_secrets_prefix}/dockerhub_user" - description = "DockerHub o11y User" - recovery_days = 7 - }, - { - name = "${local.cicd_secrets_prefix}/dockerhub_token" - description = "DockerHub o11y Token" - recovery_days = 7 - }, - { - name = "${local.cicd_secrets_prefix}/dockerhub_email" - description = "DockerHub o11y Email" - recovery_days = 7 - }, - ] - - all_regions = toset(concat([var.aws_region], var.replica_regions)) -} - -# Psuedo-random seeds we use for initializing the secrets. If we don't do this, -# then the secret "exists", but has no value or initial version, and "tf apply" -# steps fail, requiring one to manually set the password outside of Terraform. -data "aws_secretsmanager_random_password" "secret_seeds" { - for_each = { - for key, val in local.secrets : val.name => val - } - - password_length = 16 -} - -resource "aws_kms_key" "regional" { - for_each = local.all_regions - provider = aws.by_region[each.value] - - description = "Customer managed CMK for EKS Secrets in ${each.key}" - enable_key_rotation = true - - tags = local.all_security_tags - tags_all = local.all_security_tags -} - -resource "aws_kms_alias" "regional_alias" { - for_each = aws_kms_key.regional - provider = aws.by_region[each.key] - - name = "alias/eks-cmk-${each.key}" - target_key_id = each.value.id -} - -# Actual object containing the secret. -resource "aws_secretsmanager_secret" "cicd_secrets" { - for_each = { - for key, val in local.secrets : val.name => val - } - - name = each.value.name - description = each.value.description - kms_key_id = aws_kms_key.regional[var.aws_region].arn - recovery_window_in_days = each.value.recovery_days - tags = local.all_security_tags - tags_all = local.all_security_tags - - dynamic "replica" { - for_each = var.replica_regions - content { - region = replica.value - kms_key_id = aws_kms_key.regional[replica.value].arn - } - } - -} - -# Force a delay between secret creation and seeding. We only need a few -# seconds, but if we don't do this, we get into a bad state requiring manual -# intervention and/or manual forced-deletion of secrets. -resource "time_sleep" "wait_60_seconds" { - depends_on = [ - aws_secretsmanager_secret.cicd_secrets, - ] - create_duration = "60s" -} - -# Only used for seeding purposes. Will not clobber/overwrite secrets afterward -# (i.e. if/when we set them manually via the AWS CLI or management console). -resource "aws_secretsmanager_secret_version" "cicd_secrets" { - depends_on = [time_sleep.wait_60_seconds] - for_each = { - for key, val in local.secrets : val.name => val - } - - secret_id = aws_secretsmanager_secret.cicd_secrets[each.key].id - secret_string = data.aws_secretsmanager_random_password.secret_seeds[each.key].random_password - - # Prevents this seed from being applied more than once (at initial "tf apply" - # time). - lifecycle { - ignore_changes = [secret_string, ] - } -} diff --git a/modules/infra/eks_secrets/providers.tf b/modules/infra/eks_secrets/providers.tf deleted file mode 100644 index 8e1c4c29..00000000 --- a/modules/infra/eks_secrets/providers.tf +++ /dev/null @@ -1,24 +0,0 @@ -# Re-use AWS settings from root module. -provider "aws" { - region = var.aws_region - profile = var.aws_profile - - # Required, as per security guidelines. - default_tags { - tags = var.default_tags - } -} - - -provider "aws" { - alias = "by_region" - # supported by opentofu >= 1.9.0 - for_each = toset(local.all_regions) - profile = var.aws_profile - region = each.key - - # Required, as per security guidelines. - default_tags { - tags = var.default_tags - } -} diff --git a/modules/infra/eks_secrets/tags.tf b/modules/infra/eks_secrets/tags.tf deleted file mode 100644 index a527e72f..00000000 --- a/modules/infra/eks_secrets/tags.tf +++ /dev/null @@ -1,4 +0,0 @@ -# Common tags we propagate project-wide. -locals { - all_security_tags = merge(var.default_tags, var.tags) -} diff --git a/modules/infra/eks_secrets/variables.tf b/modules/infra/eks_secrets/variables.tf deleted file mode 100644 index b79130da..00000000 --- a/modules/infra/eks_secrets/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "aws_profile" { - description = "AWS profile to use." - type = string -} - -variable "aws_region" { - type = string - description = "Default AWS region." - default = "us-east-1" -} - -variable "replica_regions" { - description = "List of regions to replicate the secret" - type = list(string) - default = [] -} - -variable "tags" { - type = map(string) - description = "A map of tags to apply to resources." -} - -variable "default_tags" { - type = map(string) - description = "A map of tags to apply to resources." -} diff --git a/modules/infra/eks_secrets/versions.tf b/modules/infra/eks_secrets/versions.tf deleted file mode 100644 index 62ac438a..00000000 --- a/modules/infra/eks_secrets/versions.tf +++ /dev/null @@ -1,16 +0,0 @@ -terraform { - # Provider versions. - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.90" - } - time = { - source = "hashicorp/time" - version = "~> 0.13" - } - } - - # OpenTofu version. - required_version = ">= 1.9.1" -} diff --git a/modules/integrations/splunk_aws_billing/lambda/common.py b/modules/integrations/splunk_aws_billing/lambda/common.py index 0ec488a0..53e3f90f 100644 --- a/modules/integrations/splunk_aws_billing/lambda/common.py +++ b/modules/integrations/splunk_aws_billing/lambda/common.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- - -# -*- coding: utf-8 -*- import gzip import json import os diff --git a/modules/integrations/splunk_aws_billing/lambda/handler_per_resource.py b/modules/integrations/splunk_aws_billing/lambda/handler_per_resource.py index 8d6e1894..ccca5216 100644 --- a/modules/integrations/splunk_aws_billing/lambda/handler_per_resource.py +++ b/modules/integrations/splunk_aws_billing/lambda/handler_per_resource.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import io import json from urllib.parse import unquote diff --git a/modules/integrations/splunk_aws_billing/lambda/handler_per_resource_process.py b/modules/integrations/splunk_aws_billing/lambda/handler_per_resource_process.py index e0e2b9a9..10cb0355 100644 --- a/modules/integrations/splunk_aws_billing/lambda/handler_per_resource_process.py +++ b/modules/integrations/splunk_aws_billing/lambda/handler_per_resource_process.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import calendar import io import json diff --git a/modules/integrations/splunk_aws_billing/lambda/handler_per_service.py b/modules/integrations/splunk_aws_billing/lambda/handler_per_service.py index 0e1a088a..142d69bc 100644 --- a/modules/integrations/splunk_aws_billing/lambda/handler_per_service.py +++ b/modules/integrations/splunk_aws_billing/lambda/handler_per_service.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import calendar import io import json diff --git a/modules/platform/forge_runners/lambda/github_app_runner_group.py b/modules/platform/forge_runners/lambda/github_app_runner_group.py index 5f7b2be6..fa04d9b8 100644 --- a/modules/platform/forge_runners/lambda/github_app_runner_group.py +++ b/modules/platform/forge_runners/lambda/github_app_runner_group.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import base64 # noqa: E402 import json # noqa: E402 import logging # noqa: E402 diff --git a/modules/platform/forge_runners/lambda/github_clean_global_lock.py b/modules/platform/forge_runners/lambda/github_clean_global_lock.py index 78d310e5..b7fdf430 100644 --- a/modules/platform/forge_runners/lambda/github_clean_global_lock.py +++ b/modules/platform/forge_runners/lambda/github_clean_global_lock.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import base64 # noqa: E402 import json # noqa: E402 import logging # noqa: E402 diff --git a/scripts/migrate-all-tenants.sh b/scripts/migrate-all-tenants.sh new file mode 100755 index 00000000..1a9ed3d9 --- /dev/null +++ b/scripts/migrate-all-tenants.sh @@ -0,0 +1,10 @@ +#!/bin/bash +SCRIPT="./migrate-tenant.sh" +TENANTS_DIR=$1 + +for tenant in "$TENANTS_DIR"/*; do + if [ -d "$tenant" ]; then + echo ">>> Migrating tenant: $(basename "$tenant")" + "$SCRIPT" --tf-dir "$tenant" + fi +done diff --git a/scripts/reinstall-eks-with-deps.sh b/scripts/reinstall-eks-with-deps.sh new file mode 100755 index 00000000..5a0c86a7 --- /dev/null +++ b/scripts/reinstall-eks-with-deps.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Function to apply modules and their dependencies +apply_with_deps() { + local orig_dir + orig_dir=$(pwd) + local deps + + # Get dependent modules (absolute paths) + deps=$(terragrunt render --format json | jq -r '.dependent_modules[]') + + echo ">>> Applying main module: $orig_dir" + terragrunt apply -auto-approve --non-interactive + + for dep in $deps; do + echo ">>> Applying dependency: $dep" + pushd "$dep" >/dev/null + terragrunt apply -auto-approve --non-interactive + popd >/dev/null + done +} + +# Function to destroy modules and their dependencies +destroy_with_deps() { + local orig_dir + orig_dir=$(pwd) + local deps + + # Get dependent modules (absolute paths) + deps=$(terragrunt render --format json | jq -r '.dependent_modules[]') + + # Destroy dependencies first (reverse order) + for dep in $deps; do + echo ">>> Destroying dependency: $dep" + pushd "$dep" >/dev/null + terragrunt destroy -auto-approve --non-interactive + popd >/dev/null + done + + echo ">>> Destroying current module: $orig_dir" + terragrunt destroy -auto-approve --non-interactive +} + +# Function to show usage +usage() { + echo "Usage: $0 {create|destroy}" + echo " create - Apply main module and its dependencies" + echo " destroy - Destroy dependencies and main module" + exit 1 +} + +# Main script logic +case "${1:-}" in +create | apply) + apply_with_deps + ;; +destroy) + destroy_with_deps + ;; +*) + usage + ;; +esac