Skip to content

Commit

Permalink
Merge branch 'retire-airflow-dev-eks' of github.com:ministryofjustice…
Browse files Browse the repository at this point in the history
…/analytical-platform into retire-airflow-dev-eks
  • Loading branch information
BrianEllwood committed Jan 21, 2025
2 parents c5c3cd8 + 9864b87 commit 2657235
Show file tree
Hide file tree
Showing 21 changed files with 340 additions and 77 deletions.
37 changes: 0 additions & 37 deletions .github/path-filter/terraform.yml

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- name: Commit Changes
id: commit_changes
uses: planetscale/ghcommit-action@d4176bfacef926cc2db351eab20398dfc2f593b5 # v0.2.0
uses: planetscale/ghcommit-action@416d8ba8b8e04c703961e9a61f6b18a902c9428c # v0.2.3
with:
commit_message: "🤖 Update .github/dependabot.yml"
file_pattern: ".github/dependabot.yml"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/repository-openssf-scorecard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ jobs:

- name: Upload to CodeQL
id: upload_to_codeql
uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
uses: github/codeql-action/upload-sarif@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1
with:
sarif_file: results.sarif
9 changes: 4 additions & 5 deletions .github/workflows/reusable-workflow-terraform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
- name: Checkov
if: github.ref != 'refs/heads/main'
id: terraform_static_analysis_checkov
uses: bridgecrewio/checkov-action@2f247aefd2fdd1e1c8c03affa194ca9a230ccaac # v12.2943.0
uses: bridgecrewio/checkov-action@7e385e047f08facd4bc29345786743742edbee1f # v12.2944.0
with:
directory: ${{ env.working-directory }}
framework: terraform
Expand Down Expand Up @@ -357,18 +357,17 @@ jobs:
run: |
PR_NUMBER=$(git log -1 --pretty=%B | grep -oP '#\K\d+')
echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
echo "::set-output name=pr_number::$PR_NUMBER"
echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT
- name: Notify Slack on Failure
if: failure() && steps.terraform_apply.outcome == 'failure'
id: slack_notification
uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
with:
webhook: ${{ env.SLACK_WEBHOOK_URL }}
webhook-type: incoming-webhook
payload: |
{
"run_id": "${{ github.run_id }}",
"pr_number": "${{ steps.extract_pr.outputs.pr_number }}"
}
env:
SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
54 changes: 54 additions & 0 deletions .github/workflows/schedule-issue-jml-report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
---
name: 📥 Maintenance - Create JupyterLab Issue

on:
schedule:
- cron: "0 8 1 * *" # UTC time: “At 08:00 on first day-of-month”

permissions: {}

jobs:
create-maintenance-issue:
name: Create Maintenance Issue
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Create Maintenance Issue
id: create_maintenance_issue
run: |
if [[ "${CLOSE_PREVIOUS}" == true ]]; then
previous_issue_number=$(gh issue list \
--label "$LABELS" \
--json number \
--jq '.[0].number')
if [[ -n $previous_issue_number ]]; then
gh issue close "$previous_issue_number"
gh issue unpin "$previous_issue_number"
fi
fi
new_issue_url=$(gh issue create \
--title "$TITLE" \
--assignee "$ASSIGNEES" \
--label "$LABELS" \
--body "$BODY")
if [[ $PINNED == true ]]; then
gh issue pin "$new_issue_url"
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
TITLE: 📥 Maintenance - JML Report
ASSIGNEES:
LABELS: maintenance
BODY: |
### User Story
As a… engineer on the Analytical Platform
I need/want/expect to… keep the JML report mechanism up to date
So that… security / best practice etc
### Definition of Done
- [ ] Merge any open dependabot Pull Requests in the JML report [repository](https://github.com/ministryofjustice/analytical-platform-jml-report).
- [ ] Create new [release](https://github.com/ministryofjustice/analytical-platform-jml-report/releases).
- [ ] Update the [lambda function](https://github.com/ministryofjustice/analytical-platform/blob/main/terraform/aws/analytical-platform-data-production/joiners-movers-leavers/lambda-functions.tf) to consume the new release
PINNED: false
CLOSE_PREVIOUS: false
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ module "mojap_cadet_production" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions

source = "terraform-aws-modules/s3-bucket/aws"
version = "4.3.0"
version = "4.4.0"

bucket = "mojap-derived-tables"
grant = [{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module "development_s3" {
#checkov:skip=CKV2_AWS_67:Regular CMK key rotation is not required currently

source = "terraform-aws-modules/s3-bucket/aws"
version = "4.3.0"
version = "4.4.0"

bucket = "mojap-data-production-bold-egress-development"
force_destroy = true
Expand Down Expand Up @@ -88,7 +88,7 @@ module "production_s3" {
#checkov:skip=CKV2_AWS_67:Regular CMK key rotation is not required currently

source = "terraform-aws-modules/s3-bucket/aws"
version = "4.3.0"
version = "4.4.0"

bucket = "mojap-data-production-bold-egress-production"
force_destroy = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,3 @@ data "aws_iam_session_context" "session" {

arn = data.aws_caller_identity.session.arn
}

data "aws_secretsmanager_secret_version" "govuk_notify_api_key" {
secret_id = aws_secretsmanager_secret.govuk_notify_api_key.id
}

data "aws_secretsmanager_secret_version" "jml_email" {
secret_id = aws_secretsmanager_secret.jml_email.id
}
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
#tfsec:ignore:avd-aws-0066:no need for tracing
module "jml_extract_lambda" {
module "jml_report_lambda" {
#checkov:skip=CKV_TF_1:Module is from Terraform registry
source = "terraform-aws-modules/lambda/aws"
version = "7.20.0"

publish = true
create_package = false

function_name = "data_platform_jml_extract"
function_name = "analytical-platform-jml-report"
description = "Generates a JML report and sends it to JMLv4"
package_type = "Image"
memory_size = 512
timeout = 120
image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-jml-extract-lambda-ecr-repo:1.0.3"
image_uri = "509399598587.dkr.ecr.eu-west-2.amazonaws.com/analytical-platform-jml-report:1.4.4"

environment_variables = {
SECRET_ID = data.aws_secretsmanager_secret_version.govuk_notify_api_key.id
SECRET_ID = module.govuk_notify_api_key_secret.secret_id
LOG_GROUP_NAMES = "/aws/events/auth0/alpha-analytics-moj"
EMAIL_SECRET = data.aws_secretsmanager_secret_version.jml_email.id
EMAIL_SECRET = module.jml_email_secret.secret_id
TEMPLATE_ID = "de618989-db86-4d9a-aa55-4724d5485fa5"
}

Expand Down Expand Up @@ -49,8 +49,8 @@ module "jml_extract_lambda" {
"secretsmanager:ListSecrets"
]
resources = [
aws_secretsmanager_secret.govuk_notify_api_key.arn,
aws_secretsmanager_secret.jml_email.arn
module.govuk_notify_api_key_secret.secret_arn,
module.jml_email_secret.secret_arn
]
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,25 @@
#tfsec:ignore:avd-aws-0098 CMK not required currently
resource "aws_secretsmanager_secret" "govuk_notify_api_key" {
# checkov:skip=CKV2_AWS_57:These won't be rotated in the traditional manner
# checkov:skip=CKV_AWS_149:No KMS key needed as per above, these won't be rotated
name = "gov-uk-notify/production/api-key"
module "jml_email_secret" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
#checkov:skip=CKV_TF_2:Module registry does not support tags for versions

source = "terraform-aws-modules/secrets-manager/aws"
version = "1.3.1"

name = "jml-report/email"

ignore_secret_changes = true
secret_string = "CHANGEME"
}

#tfsec:ignore:avd-aws-0098 CMK not required currently
resource "aws_secretsmanager_secret" "jml_email" {
# checkov:skip=CKV2_AWS_57:These won't be rotated in the traditional manner
# checkov:skip=CKV_AWS_149:No KMS key needed as per above, these won't be rotated
name = "jml/email"
module "govuk_notify_api_key_secret" {
#checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
#checkov:skip=CKV_TF_2:Module registry does not support tags for versions

source = "terraform-aws-modules/secrets-manager/aws"
version = "1.3.1"

name = "gov-uk-notify/api-key"

ignore_secret_changes = true
secret_string = "CHANGEME"
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ provider "aws" {
}

provider "aws" {
region = "eu-west-1"
region = "eu-west-2"
assume_role {
role_arn = "arn:aws:iam::${var.account_ids["analytical-platform-data-production"]}:role/GlobalGitHubActionAdmin"
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
data "aws_caller_identity" "current" {}
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,11 @@ data "aws_iam_policy_document" "bedrock_integration" {
"bedrock:CreateFoundationModelAgreement",
"bedrock:DeleteFoundationModelAgreement",
"bedrock:ListFoundationModelAgreementOffers",
"bedrock:GetUseCaseForModelAccess"
"bedrock:GetUseCaseForModelAccess",
"bedrock:CreateModelInvocationJob",
"bedrock:GetModelInvocationJob",
"bedrock:ListModelInvocationJobs",
"bedrock:StopModelInvocationJob"
]
resources = ["*"]
condition {
Expand All @@ -72,6 +76,81 @@ resource "aws_iam_policy" "bedrock_integration" {
policy = data.aws_iam_policy_document.bedrock_integration.json
}

##################################################
# Bedrock Batch Inference
##################################################

data "aws_iam_policy_document" "bedrock_batch_inference" {
statement {
sid = "AllowBedrockAssumeRoleForBatchInference"
actions = ["sts:AssumeRole"]
condition {
test = "StringEquals"
variable = "aws:SourceAccount"
values = [data.aws_caller_identity.current.account_id]
}
condition {
test = "ArnLike"
variable = "aws:SourceArn"
values = ["arn:aws:bedrock:*:${data.aws_caller_identity.current.account_id}:model-invocation-job/*"]
}
effect = "Allow"
principals {
type = "Service"
identifiers = ["bedrock.amazonaws.com"]
}
}
}

resource "aws_iam_role" "bedrock_batch_inference" {
name = "bedrock-batch-inference-role"
description = "IAM role for AWS Bedrock to perform batch inference tasks as part of model invocation workflows."
assume_role_policy = data.aws_iam_policy_document.bedrock_batch_inference.json
}

resource "aws_iam_role_policy_attachment" "bedrock_batch_inference" {
role = aws_iam_role.bedrock_batch_inference.name
policy_arn = aws_iam_policy.bedrock_integration.arn
}

# Bedrock Batch Inference s3 access
data "aws_iam_policy_document" "bedrock_batch_inference_s3_access" {
statement {
sid = "BedrockBatchInferenceS3Access"
effect = "Allow"

actions = [
"s3:GetObject",
"s3:PutObject",
"s3:ListBucket",
]

resources = [
"arn:aws:s3:::*"
]

condition {
test = "StringEquals"
variable = "aws:ResourceAccount"
values = [
data.aws_caller_identity.current.account_id
]
}
}
}

resource "aws_iam_policy" "bedrock_batch_inference_s3_access" {
name = "bedrock-batch-inference-s3-access"
description = "S3 access policy for Bedrock batch inference."
policy = data.aws_iam_policy_document.bedrock_batch_inference_s3_access.json
}

resource "aws_iam_role_policy_attachment" "bedrock_batch_inference_s3_access" {
role = aws_iam_role.bedrock_batch_inference.name
policy_arn = aws_iam_policy.bedrock_batch_inference_s3_access.arn
}


#tfsec:ignore:aws-iam-no-policy-wildcards
data "aws_iam_policy_document" "textract_integration" {
#checkov:skip=CKV_AWS_111: This is a service policy
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,8 @@ data "aws_iam_policy_document" "control_panel_api" {
]
resources = [
"arn:aws:iam::${var.account_ids["analytical-platform-compute-development"]}:role/analytical-platform-control-panel",
"arn:aws:iam::${var.account_ids["analytical-platform-compute-test"]}:role/analytical-platform-control-panel"
"arn:aws:iam::${var.account_ids["analytical-platform-compute-test"]}:role/analytical-platform-control-panel",
"arn:aws:iam::${var.account_ids["parent-account"]}:role/AnalyticalPlatformIdentityCenter"
]
}
statement {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ account_ids = {
analytical-platform-production = "312423030077"
analytical-platform-compute-development = "381491960855"
analytical-platform-compute-test = "767397661611"
parent-account = "295814833350"
}

environment = "development"
Expand Down
Loading

0 comments on commit 2657235

Please sign in to comment.