DP-187

.github/workflows/build.yml

@@ -42,7 +42,7 @@ jobs:
           name: OpenAPI
           path: OpenAPI
 
-  docker:
+  package:
     runs-on: ubuntu-latest
     name: Build containers
     env:
@@ -52,6 +52,8 @@ jobs:
       CDP_PERSON_PORT: 8833
       CDP_FORMS_PORT: 8844
       CDP_DATA_SHARING_PORT: 8855
+    outputs:
+      version: ${{ steps.version.outputs.version }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -78,21 +80,48 @@ jobs:
         run: make down
         env:
           IMAGE_VERSION: ${{ steps.version.outputs.version }}
+      - name: Save Docker Images
+        run: |
+          IMAGES=$(docker images | awk '$1 ~ /cabinetoffice\/cdp-/ && $2 ~ /^'$IMAGE_VERSION'$/ { print $1":"$2}' | tr '\n' ' ')
+          echo "Images to be saved: $IMAGES"
+          docker save -o cdp-images.tar $IMAGES
+        env:
+          IMAGE_VERSION: ${{ steps.version.outputs.version }}
+      - name: Upload Docker Images as Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-images
+          path: cdp-images.tar
 
-  docs:
+  publish:
     runs-on: ubuntu-latest
-    name: Build documentation
+    name: Publish containers
+    needs: [test, package]
+
     steps:
       - uses: actions/checkout@v4
+
+      - name: Download Docker Images
+        uses: actions/download-artifact@v4
         with:
-          fetch-depth: 0
-      - name: Build
-        run: cd docs && make build
-      - name: Publish
-        if: github.ref == 'refs/heads/main' && github.repository_owner == 'cabinetoffice'
-        run: cd docs && make publish
-      - name: Upload Documentation
-        uses: actions/upload-artifact@v4
+          name: docker-images
+
+      - name: Load Docker Images
+        run: docker load -i cdp-images.tar
+
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v4
         with:
-          name: Documentation
-          path: docs/build/*
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Generate Image Version
+        id: image_version
+        run: |
+          IMAGE_VERSION=${{ needs.package.outputs.version }}
+          echo "IMAGE_VERSION=$IMAGE_VERSION" >> $GITHUB_ENV

.github/workflows/docs.yml

@@ -0,0 +1,26 @@
+name: Docs
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+
+  docs:
+    runs-on: ubuntu-latest
+    name: Build documentation
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build
+        run: cd docs && make build
+      - name: Publish
+        if: github.ref == 'refs/heads/main' && github.repository_owner == 'cabinetoffice'
+        run: cd docs && make publish
+      - name: Upload Documentation
+        uses: actions/upload-artifact@v4
+        with:
+          name: Documentation
+          path: docs/build/*

.mise.toml

@@ -1,4 +1,4 @@
 [tools]
 dotnet = "8.0.300"
-terraform = "1.8.2"
-terragrunt = "0.58.2"
+terraform = "1.9.2"
+terragrunt = "0.62.0"

Makefile

@@ -2,8 +2,6 @@
 AWS_ACCOUNT_ID=$$(aws sts get-caller-identity | jq -r '.Account')
 REPO_URL := $(AWS_ACCOUNT_ID).dkr.ecr.eu-west-2.amazonaws.com
 IMAGES := cdp-organisation-information-migrations cdp-data-sharing cdp-entity-verification cdp-forms cdp-organisation-app cdp-organisation cdp-person cdp-tenant cdp-authority
-TAGGED_IMAGES := $(addprefix cabinetoffice/,$(addsuffix :latest,$(IMAGES)))
-TAGGED_REPO_IMAGES := $(addprefix $(REPO_URL)/,$(TAGGED_IMAGES))
 DOCKER_COMPOSE_CMD=IMAGE_VERSION=$(IMAGE_VERSION) docker compose
 
 # Extracts targets and their comments
@@ -163,12 +161,16 @@ aws-push-authority-private-key: ## Push Authority's private key to the target AW
 		aws secretsmanager create-secret --name cdp-sirsi-authority-keys --secret-string "$$(jq -n --arg priv "$$(cat ./terragrunt/secrets/authority-private-key.pem)" '{PRIVATE: $$priv}')"; \
 	fi
 
-aws-push-to-ecr: build-docker ## Build, tag and push Docker images to ECR
-	$(foreach image,$(TAGGED_IMAGES),docker tag $(image) $(REPO_URL)/$(notdir $(basename $(image)));)
+aws-push-to-ecr: IMAGE_VERSION ?= latest
+aws-push-to-ecr: ## Tag latest built Docker images and push to ECR
+	$(foreach image,$(IMAGES),docker tag cabinetoffice/$(image):$(IMAGE_VERSION) $(REPO_URL)/$(image):$(IMAGE_VERSION);)
 	aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin $(REPO_URL)
-	$(foreach image,$(IMAGES),docker push $(REPO_URL)/$(image);)
+	$(foreach image,$(IMAGES),docker push $(REPO_URL)/$(image):$(IMAGE_VERSION);)
 .PHONY: aws-push-to-ecr
 
+aws-build-and-push-ecr: build-docker aws-push-to-ecr ## Build, tag and push Docker images to ECR
+.PHONY: aws-build-and-push-ecr
+
 version-commit: COMMIT_REF ?= HEAD
 version-commit: ## Determines the last commit hash
 	@git rev-parse --short "$(COMMIT_REF)"

terragrunt/components/common/networking/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//networking"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//networking" : null
 }
 
 include {

terragrunt/components/core/iam/terragrunt.hcl

@@ -20,6 +20,7 @@ locals {
 }
 
 inputs = {
+  account_ids         = local.global_vars.locals.account_ids
   tags                = local.tags
   terraform_operators = local.global_vars.locals.terraform_operators
   tfstate_bucket_name = local.global_vars.locals.tg.state_bucket

terragrunt/components/core/networking/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//core-networking"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//core-networking" : null
 }
 
 include {

terragrunt/components/core/security-groups/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//core-security-groups"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//core-security-groups" : null
 }
 
 include {

terragrunt/components/orchestrator/ecr/terragrunt.hcl

@@ -0,0 +1,28 @@
+terraform {
+  source = local.global_vars.locals.environment == "orchestrator" ? "../../../modules//orchestrator/ecr" : null
+}
+
+include {
+  path = find_in_parent_folders()
+}
+
+locals {
+
+  global_vars = read_terragrunt_config(find_in_parent_folders("terragrunt.hcl"))
+  orchestrator_vars   = read_terragrunt_config(find_in_parent_folders("orchestrator.hcl"))
+
+  tags = merge(
+    local.global_vars.inputs.tags,
+    local.orchestrator_vars.inputs.tags,
+    {
+      component = "orchestrator-ecr"
+    }
+  )
+
+}
+
+inputs = {
+  account_ids     = local.global_vars.locals.account_ids
+  service_configs = local.global_vars.locals.service_configs
+  tags            = local.tags
+}

terragrunt/components/orchestrator/iam/terragrunt.hcl

@@ -0,0 +1,27 @@
+terraform {
+  source = local.global_vars.locals.environment == "orchestrator" ? "../../../modules//orchestrator/iam" : null
+}
+
+include {
+  path = find_in_parent_folders()
+}
+
+locals {
+
+  global_vars = read_terragrunt_config(find_in_parent_folders("terragrunt.hcl"))
+  orchestrator_vars = read_terragrunt_config(find_in_parent_folders("orchestrator.hcl"))
+
+  tags = merge(
+    local.global_vars.inputs.tags,
+    local.orchestrator_vars.inputs.tags,
+    {
+      component = "orchestrator-iam"
+    }
+  )
+}
+
+inputs = {
+  account_ids         = local.global_vars.locals.account_ids
+  tags                = local.tags
+  terraform_operators = local.global_vars.locals.terraform_operators
+}

terragrunt/components/orchestrator/orchestrator.hcl

@@ -0,0 +1,9 @@
+locals {
+  tags = {
+    component_root = "orchestrator"
+  }
+}
+
+inputs = {
+  tags = local.tags
+}

terragrunt/components/service/api-gateway/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//api-gateway"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//api-gateway" : null
 }
 
 include {

terragrunt/components/service/database/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//database"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//database" : null
 }
 
 include {

terragrunt/components/service/ecs/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//ecs"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//ecs" : null
 }
 
 include {
@@ -76,6 +76,8 @@ dependency service_database {
 }
 
 inputs = {
+
+  account_ids     = local.global_vars.locals.account_ids
   service_configs = local.global_vars.locals.service_configs
   tags            = local.tags
 

terragrunt/components/service/telemetry/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "../../../modules//telemetry"
+  source = local.global_vars.locals.environment != "orchestrator" ? "../../../modules//telemetry" : null
 }
 
 include {
@@ -58,6 +58,7 @@ dependency service_ecs {
 }
 
 inputs = {
+  account_ids     = local.global_vars.locals.account_ids
   grafana_config  = local.global_vars.locals.tools_configs.grafana
   service_configs = local.global_vars.locals.service_configs
   tags            = local.tags

terragrunt/components/terragrunt.hcl

@@ -1,15 +1,38 @@
 locals {
 
+    account_ids = {
+        for name, env in local.environments : name => env.account_id
+    }
+
     cidr_b_development = 3
     cidr_b_integration = 4
+    cidr_b_orchestrator = 5
     cidr_b_production = 1
     cidr_b_staging = 2
 
     environment = get_env("TG_ENVIRONMENT", "development")
 
     environments = {
+        orchestrator = {
+            cidr_block             = "10.${local.cidr_b_orchestrator}.0.0/16"
+            account_id             = 891377225335
+            name                   = "orchestrator"
+            postgres_instance_type = "db.t4g.micro"
+            private_subnets        = [
+                "10.${local.cidr_b_orchestrator}.101.0/24",
+                "10.${local.cidr_b_orchestrator}.102.0/24",
+                "10.${local.cidr_b_orchestrator}.103.0/24"
+            ]
+            public_subnets = [
+                "10.${local.cidr_b_orchestrator}.1.0/24",
+                "10.${local.cidr_b_orchestrator}.2.0/24",
+                "10.${local.cidr_b_orchestrator}.3.0/24"
+            ]
+            top_level_domain = "findatender.codatt.net"
+        }
         development = {
             cidr_block             = "10.${local.cidr_b_development}.0.0/16"
+            account_id             = 471112892058
             name                   = "dev"
             postgres_instance_type = "db.t4g.micro"
             private_subnets        = [
@@ -26,6 +49,7 @@ locals {
         }
         staging = {
             cidr_block             = "10.${local.cidr_b_staging}.0.0/16"
+            account_id             = 905418042182
             name                   = "staging"
             postgres_instance_type = "db.t4g.micro"
             private_subnets        = [
@@ -42,6 +66,7 @@ locals {
         }
         integration = {
             cidr_block             = "10.${local.cidr_b_integration}.0.0/16"
+            account_id             = 767397666448
             name                   = "integration"
             postgres_instance_type = "db.t4g.micro"
             private_subnets        = [
@@ -58,6 +83,7 @@ locals {
         }
         production = {
             cidr_block             = "10.${local.cidr_b_production}.0.0/16"
+            account_id             = 471112843276
             name                   = "production"
             postgres_instance_type = "db.t4g.micro"
             private_subnets        = [

terragrunt/modules/core-iam/ci-datasource.tf

@@ -1,3 +1,6 @@
+# Note!
+# Resources in this file are shared with orchestrator/iam module
+
 data "aws_iam_policy_document" "terraform_assume" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -13,6 +16,14 @@ data "aws_iam_policy_document" "terraform_assume" {
   }
 }
 
+data "aws_iam_policy_document" "terraform_assume_orchestrator_role" {
+  statement {
+    effect    = "Allow"
+    actions   = ["sts:AssumeRole"]
+    resources = ["arn:aws:iam::${local.orchestrator_account_id}:role/cdp-sirsi-orchestrator-read-service-version"]
+  }
+}
+
 data "aws_iam_policy_document" "terraform" {
 
   statement {
@@ -202,8 +213,9 @@ data "aws_iam_policy_document" "terraform_global" {
 
   statement {
     actions = [
-      "ecr:CreateRepository",
+      "ecr:BatchGetImage",
       "ecr:GetAuthorizationToken",
+      "ecr:GetDownloadUrlForLayer",
       "ecs:Create*",
       "ecs:DeregisterTaskDefinition",
       "ecs:DescribeTaskDefinition",
@@ -268,7 +280,8 @@ data "aws_iam_policy_document" "terraform_product" {
     resources = [
       "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/cdp-sirsi-*",
       "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/cdp-sirsi-*",
-      "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/*"
+      "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-service-role/*",
+      "arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/cdp-sirsi-*"
     ]
     sid = "ManageProductIAMs"
   }
@@ -290,9 +303,7 @@ data "aws_iam_policy_document" "terraform_product" {
     actions = ["ec2:*"]
     effect  = "Allow"
     resources = [
-      "arn:aws:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*/cdp-sirsi-*",
-      "arn:aws:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:vpc/cdp-sirsi-*",
-      "arn:aws:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:elastic-ip/cdp-sirsi-*"
+      "arn:aws:ec2:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*/cdp-sirsi-*"
     ]
     sid = "ManageProductEC2"
   }