From ed2a2386fc2a3b1a0e81d72dac7c14132ac9db6e Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Tue, 19 Nov 2024 07:52:54 -0800 Subject: [PATCH 01/19] [IT-3984] Change redirect for Schematic app (#1288) There is a new deployment for Schematic. Now we are ready to redirect the vanity dns names to the new deployment. depends on https://github.com/Sage-Bionetworks-IT/schematic-infra-v2/pull/20 --- org-formation/800-redirects/_tasks.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index c10645a6..cc3e09e0 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -149,7 +149,7 @@ SchematicDevAppDnsForward: # ID of the api.sagebionetworks.org zone (in sageit account) SourceHostedZoneId: !CopyValue [!Sub '${primaryRegion}-${resourcePrefix}-sagebio-api-zone-HostedZoneId'] # the value of the CNAME record - TargetHostName: !CopyValue ['schematic-dev-DockerFargateStack-LoadBalancerDNS', !Ref DnTDevAccount] + TargetHostName: !CopyValue ['schematic-dev-load-balancer-dns', !Ref DnTDevAccount] # forward schematic-staging.api.sagebionetworks.org to schematic-infra ALB # https://github.com/Sage-Bionetworks/schematic-infra @@ -169,6 +169,24 @@ SchematicStagingAppDnsForward: # the value of the CNAME record TargetHostName: !CopyValue ['schematic-stage-staging-DockerFargateStack-LoadBalancerDNS', !Ref DCAProdAccount] +# forward schematic-staging.api.sagebionetworks.org to schematic-infra ALB +# https://github.com/Sage-Bionetworks/schematic-infra +SchematicStageAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-schematic-stage-cname' + StackDescription: Setup a CNAME for schematic-infra stage ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "schematic-stage.api.sagebionetworks.org" + # ID of the api.sagebionetworks.org zone (in sageit account) + SourceHostedZoneId: !CopyValue [!Sub '${primaryRegion}-${resourcePrefix}-sagebio-api-zone-HostedZoneId'] + # the value of the CNAME record + TargetHostName: !CopyValue ['schematic-stage-load-balancer-dns', !Ref DCAProdAccount] + # forward schematic.api.sagebionetworks.org to schematic-infra ALB # https://github.com/Sage-Bionetworks/schematic-infra SchematicProdAppDnsForward: From 4b37bad79a9243c4f0a6022816724c2f2c71b579 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:24:05 -0700 Subject: [PATCH 02/19] Point staging url for schematic to multi-container deployment stack (#1290) **Problem:** 1. Resources are expecting the `staging` URL for schematic remains as is **Solution:** 1. Updating the source hostname to use `staging` and removing the other dns forwarding for the previous stack --- org-formation/800-redirects/_tasks.yaml | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index cc3e09e0..f19c4782 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -134,7 +134,7 @@ DcaProdAppDnsForward: # *.api.sagebionetworks.org # forward schematic-dev.api.sagebionetworks.org to schematic-infra ALB -# https://github.com/Sage-Bionetworks/schematic-infra +# https://github.com/Sage-Bionetworks-IT/schematic-infra-v2 SchematicDevAppDnsForward: Type: update-stacks Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml @@ -152,25 +152,7 @@ SchematicDevAppDnsForward: TargetHostName: !CopyValue ['schematic-dev-load-balancer-dns', !Ref DnTDevAccount] # forward schematic-staging.api.sagebionetworks.org to schematic-infra ALB -# https://github.com/Sage-Bionetworks/schematic-infra -SchematicStagingAppDnsForward: - Type: update-stacks - Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml - StackName: !Sub '${resourcePrefix}-schematic-staging-cname' - StackDescription: Setup a CNAME for schematic-infra staging ALB - DefaultOrganizationBindingRegion: !Ref primaryRegion - DefaultOrganizationBinding: - Account: !Ref SageITAccount - Parameters: - # the name of the CNAME record - SourceHostName: "schematic-staging.api.sagebionetworks.org" - # ID of the api.sagebionetworks.org zone (in sageit account) - SourceHostedZoneId: !CopyValue [!Sub '${primaryRegion}-${resourcePrefix}-sagebio-api-zone-HostedZoneId'] - # the value of the CNAME record - TargetHostName: !CopyValue ['schematic-stage-staging-DockerFargateStack-LoadBalancerDNS', !Ref DCAProdAccount] - -# forward schematic-staging.api.sagebionetworks.org to schematic-infra ALB -# https://github.com/Sage-Bionetworks/schematic-infra +# https://github.com/Sage-Bionetworks-IT/schematic-infra-v2 SchematicStageAppDnsForward: Type: update-stacks Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml @@ -181,7 +163,7 @@ SchematicStageAppDnsForward: Account: !Ref SageITAccount Parameters: # the name of the CNAME record - SourceHostName: "schematic-stage.api.sagebionetworks.org" + SourceHostName: "schematic-staging.api.sagebionetworks.org" # ID of the api.sagebionetworks.org zone (in sageit account) SourceHostedZoneId: !CopyValue [!Sub '${primaryRegion}-${resourcePrefix}-sagebio-api-zone-HostedZoneId'] # the value of the CNAME record From 14419b94a41d214ace54af08cac203cf6f3c3587 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:43:42 -0700 Subject: [PATCH 03/19] Point to schematic staging (Replace) (#1291) Deployment of https://github.com/Sage-Bionetworks-IT/organizations-infra/pull/1290/files failed: https://github.com/Sage-Bionetworks-IT/organizations-infra/actions/runs/11919331993/job/33218806052 Due to: ``` DEBG: Stack sagebase-schematic-stage-cname in account 797640923903 (us-east-1) update starting... (797640923903 = SageITAccount) ERROR: error updating CloudFormation stack sagebase-schematic-stage-cname in account 797640923903 (us-east-1). Resource is not in the state stackUpdateComplete (797640923903 = SageITAccount) ERROR: Resource DnsRecord failed because [Tried to create resource record set [name='schematic-staging.api.sagebionetworks.org.', type='CNAME'] but it already exists]. ERROR: Stack sagebase-schematic-stage-cname in account 797640923903 (us-east-1) update failed. reason: Resource is not in the state stackUpdateComplete (797640923903 = SageITAccount) Resource is not in the state stackUpdateComplete ``` I suspect because we're replacing a DNS record with a new stack. Instead, this change should point to the existing stack to update it's values with the new ALB. It should also destroy the "stage" stack DNS forwarding config. --- org-formation/800-redirects/_tasks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index f19c4782..b8a435c7 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -153,10 +153,10 @@ SchematicDevAppDnsForward: # forward schematic-staging.api.sagebionetworks.org to schematic-infra ALB # https://github.com/Sage-Bionetworks-IT/schematic-infra-v2 -SchematicStageAppDnsForward: +SchematicStagingAppDnsForward: Type: update-stacks Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml - StackName: !Sub '${resourcePrefix}-schematic-stage-cname' + StackName: !Sub '${resourcePrefix}-schematic-staging-cname' StackDescription: Setup a CNAME for schematic-infra stage ALB DefaultOrganizationBindingRegion: !Ref primaryRegion DefaultOrganizationBinding: From 45ba5ecd4161a08a220436bc0d52cce02b483c62 Mon Sep 17 00:00:00 2001 From: Xavier Schildwachter Date: Tue, 19 Nov 2024 17:14:30 -0800 Subject: [PATCH 04/19] Fix bucket name, remove obsolete ones (#1289) --- org-formation/650-identity-providers/_tasks.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/org-formation/650-identity-providers/_tasks.yaml b/org-formation/650-identity-providers/_tasks.yaml index d7affac4..f2d3418a 100644 --- a/org-formation/650-identity-providers/_tasks.yaml +++ b/org-formation/650-identity-providers/_tasks.yaml @@ -898,9 +898,7 @@ SynapseMonorepoBucketAccessPolicy: "Effect": "Allow", "Action": [ "s3:GetBucketLocation", "s3:ListBucket", "s3:ListBucketMultipartUploads" ], "Resource": [ - "arn:aws:s3:::prod.accounts.sagebionetworks.org", - "arn:aws:s3:::staging.accounts.sagebionetworks.org", - "arn:aws:s3:::dev.accounts.sagebionetworks.org", + "arn:aws:s3:::dev.accounts.synapse.org", "arn:aws:s3:::prod.accounts.synapse.org", "arn:aws:s3:::staging.accounts.synapse.org", "arn:aws:s3:::prod-adknowledgeportalsynapse-org-websitebucket-1wcys549ufmd", @@ -951,9 +949,7 @@ SynapseMonorepoFileAccessPolicy: "Effect": "Allow", "Action": [ "s3:PutObject", "s3:GetObject", "s3:DeleteObject", "s3:*Multipart*" ], "Resource": [ - "arn:aws:s3:::prod.accounts.sagebionetworks.org/*", - "arn:aws:s3:::staging.accounts.sagebionetworks.org/*", - "arn:aws:s3:::dev.accounts.sagebionetworks.org/*", + "arn:aws:s3:::dev.accounts.synapse.org/*", "arn:aws:s3:::prod.accounts.synapse.org/*", "arn:aws:s3:::staging.accounts.synapse.org/*", "arn:aws:s3:::prod-adknowledgeportalsynapse-org-websitebucket-1wcys549ufmd/*", @@ -1021,8 +1017,6 @@ SynapseMonorepoCloudfrontAccessPolicy: "arn:aws:cloudfront::797640923903:distribution/E10U4765KQQW5P", "arn:aws:cloudfront::797640923903:distribution/E1FILQHG8BTWIL", "arn:aws:cloudfront::797640923903:distribution/E14P60CJ0I6G7Y", - "arn:aws:cloudfront::797640923903:distribution/E2656IE63W1MXI", - "arn:aws:cloudfront::797640923903:distribution/EY52HOUGKDP1F", "arn:aws:cloudfront::797640923903:distribution/E14F656YEGR4P3", "arn:aws:cloudfront::797640923903:distribution/E1CB47ERU70VWV", "arn:aws:cloudfront::797640923903:distribution/E2K9BYXQN2MM76", From 5e5699ba6e6d84c3d9f6531b47c3cd7a89f0ab47 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Thu, 21 Nov 2024 08:58:51 -0800 Subject: [PATCH 05/19] [IT-4024] Setup a redirect to sagedpe.org apps (#1295) We had setup a redirect for dev.sagedpe.org as part of work for IT-3931 however it was done manually. This PR will do it with cloudformation and put it under CI control. --- org-formation/800-redirects/_tasks.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index b8a435c7..8c3d9a9f 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -258,3 +258,22 @@ SynapseDockerRegistryProdDnsForward: SourceHostedZoneId: ZHAU99KV4A1WU # the value of the CNAME record TargetHostName: !CopyValue ['registry-prod-DockerFargateStack-LoadBalancerDNS', !Ref SynapseProdAccount] + + +# forward dev.sagedpe.org to dev EKS stack ALB in org-sagebase-dnt-dev +# apps are setup with terraform at https://github.com/Sage-Bionetworks-Workflows/eks-stack +SageDpeDevAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-sagedpe-dev-cname' + StackDescription: Setup a CNAME for sagepde.org dev ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "dev.sagedpe.org" + # ID of the sagedpe.org zone (in sageit account) + SourceHostedZoneId: "Z04325181I2YIP983P1AD" + # the value of the CNAME record + TargetHostName: "ac5c848ac4ff54e2bb11dd87685375b0-1875694220.us-east-1.elb.amazonaws.com" From 580e79c2874bb89df3ae6a74c70d0c2943337812 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:21:27 -0700 Subject: [PATCH 06/19] Tear down DNS redirect for schematic-dev refactor (#1296) Since moving to the schematic multi-container deployment this stack is no longer needed and is being torn down --- org-formation/800-redirects/_tasks.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index 8c3d9a9f..eaaec907 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -187,24 +187,6 @@ SchematicProdAppDnsForward: # the value of the CNAME record TargetHostName: !CopyValue ['schematic-prod-DockerFargateStack-LoadBalancerDNS', !Ref DCAProdAccount] -# forward schematic-dev-refactor.api.sagebionetworks.org to schematic-infra ALB -# https://github.com/Sage-Bionetworks/schematic-infra -SchematicDevRefactorAppDnsForward: - Type: update-stacks - Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml - StackName: !Sub '${resourcePrefix}-schematic-dev-refactor-cname' - StackDescription: Setup a CNAME for schematic-infra dev-refactor ALB - DefaultOrganizationBindingRegion: !Ref primaryRegion - DefaultOrganizationBinding: - Account: !Ref SageITAccount - Parameters: - # the name of the CNAME record - SourceHostName: "schematic-dev-refactor.api.sagebionetworks.org" - # ID of the api.sagebionetworks.org zone (in sageit account) - SourceHostedZoneId: !CopyValue [!Sub '${primaryRegion}-${resourcePrefix}-sagebio-api-zone-HostedZoneId'] - # the value of the CNAME record - TargetHostName: !CopyValue ['schematic-dev-refactor-dev-refactor-DockerFargateStack-LoadBalancerDNS', !Ref DnTDevAccount] - # forward https://genie-bpc.app.sagebionetworks.org to genie-bpc-infra ALB # https://github.com/Sage-Bionetworks/genie-bpc-infra GenieBPCProdAppDnsForward: From dc5f6c97ac9ce980da12c1e199bb5b58ad1d0664 Mon Sep 17 00:00:00 2001 From: Joni Harker <506966+ConsoleCatzirl@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:43:31 -0800 Subject: [PATCH 07/19] [IT-3068] Bump version of s3-cost-report lambda (#1294) Add the account ID to the cost report email. --- .../config/prod/lambda-finops-s3-cost-report.yaml | 2 +- sceptre/strides/config/prod/lambda-finops-s3-cost-report.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sceptre/strides-ampad-workflows/config/prod/lambda-finops-s3-cost-report.yaml b/sceptre/strides-ampad-workflows/config/prod/lambda-finops-s3-cost-report.yaml index 4113d871..6fbdfd52 100644 --- a/sceptre/strides-ampad-workflows/config/prod/lambda-finops-s3-cost-report.yaml +++ b/sceptre/strides-ampad-workflows/config/prod/lambda-finops-s3-cost-report.yaml @@ -1,6 +1,6 @@ template: type: http - url: https://{{stack_group_config.admincentral_cf_bucket}}.s3.amazonaws.com/lambda-finops-s3-cost-report/1.0.0/lambda-finops-s3-cost-report.yaml + url: https://{{stack_group_config.admincentral_cf_bucket}}.s3.amazonaws.com/lambda-finops-s3-cost-report/1.0.1/lambda-finops-s3-cost-report.yaml stack_name: lambda-finops-s3-cost-report parameters: Sender: "aws.strides@sagebase.org" diff --git a/sceptre/strides/config/prod/lambda-finops-s3-cost-report.yaml b/sceptre/strides/config/prod/lambda-finops-s3-cost-report.yaml index 4113d871..6fbdfd52 100644 --- a/sceptre/strides/config/prod/lambda-finops-s3-cost-report.yaml +++ b/sceptre/strides/config/prod/lambda-finops-s3-cost-report.yaml @@ -1,6 +1,6 @@ template: type: http - url: https://{{stack_group_config.admincentral_cf_bucket}}.s3.amazonaws.com/lambda-finops-s3-cost-report/1.0.0/lambda-finops-s3-cost-report.yaml + url: https://{{stack_group_config.admincentral_cf_bucket}}.s3.amazonaws.com/lambda-finops-s3-cost-report/1.0.1/lambda-finops-s3-cost-report.yaml stack_name: lambda-finops-s3-cost-report parameters: Sender: "aws.strides@sagebase.org" From 1c287b51900a6d09cf36c74b1b33bb672858d741 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:55:37 -0800 Subject: [PATCH 08/19] [IT-3995] Setup github OIDC for agora-infra-v3 (#1293) Create github OIDC access to allow CI deployments from github actions. --- .../650-identity-providers/_tasks.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/org-formation/650-identity-providers/_tasks.yaml b/org-formation/650-identity-providers/_tasks.yaml index f2d3418a..88618c20 100644 --- a/org-formation/650-identity-providers/_tasks.yaml +++ b/org-formation/650-identity-providers/_tasks.yaml @@ -804,6 +804,29 @@ GithubOidcAgoraInfraDeploy: - !Ref AgoraProdAccount Region: us-east-1 +GithubOidcAgoraInfraV3: + Type: update-stacks + DependsOn: GithubOidcSageBionetworks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.7.6/templates/IAM/github-oidc-provider.j2 + StackName: !Sub ${resourcePrefix}-${appName}-agora-infra-v3 + Parameters: + ProviderArn: !CopyValue [ !Sub '${resourcePrefix}-${appName}-ProviderArn' ] + ProviderRoleName: !Sub ${resourcePrefix}-${appName}-agora-infra-v3 + MaxSessionDuration: 7200 + ManagedPolicyArns: + - "arn:aws:iam::aws:policy/AdministratorAccess" + - "arn:aws:iam::aws:policy/AWSKeyManagementServicePowerUser" + TemplatingContext: + GitHubOrg: "Sage-Bionetworks-IT" + Repositories: + - name: "agora-infra-v3" + branches: ["dev","stage","prod"] + DefaultOrganizationBinding: + Account: + - !Ref AgoraDevAccount + - !Ref AgoraProdAccount + Region: us-east-1 + GithubOidcAgoraEBDeploy: Type: update-stacks DependsOn: GithubOidcSageBionetworks From e4824a732d5ba0e1c4510b1c617ee7cd8e05f538 Mon Sep 17 00:00:00 2001 From: Xavier Schildwachter Date: Thu, 21 Nov 2024 12:10:29 -0800 Subject: [PATCH 09/19] IT-4016/IT-4017: Give PowerUser permissions and scale down later (#1292) * Give PowerUser and scale down later * Add deny-assume-role policy --- org-formation/700-aws-sso/_tasks.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/org-formation/700-aws-sso/_tasks.yaml b/org-formation/700-aws-sso/_tasks.yaml index 21b48e66..3041fa08 100644 --- a/org-formation/700-aws-sso/_tasks.yaml +++ b/org-formation/700-aws-sso/_tasks.yaml @@ -643,8 +643,18 @@ SsoLlmDeveloper: principalId: !Ref llmDeveloperGroup permissionSetName: 'LlmDeveloper' managedPolicies: - - 'arn:aws:iam::aws:policy/AmazonBedrockFullAccess' - - 'arn:aws:iam::aws:policy/AWSCloudFormationFullAccess' + - 'arn:aws:iam::aws:policy/PowerUserAccess' + inlinePolicy: >- + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "sts:AssumeRole", + "Resource": "*" + } + ] + } sessionDuration: 'PT12H' # Role for a user that can only access AWS Athena in the Synapse Dev account From 65abbdbd92de947fba0a386ebbdc8fdeafb9d7c4 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:57:06 -0700 Subject: [PATCH 10/19] [SCHEMATIC-211] Updates Schematic Prod DNS redirect for new deployment stack (#1297) * updates DNS redirect for new deployment stack --- org-formation/800-redirects/_tasks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index eaaec907..59d31b2a 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -170,7 +170,7 @@ SchematicStagingAppDnsForward: TargetHostName: !CopyValue ['schematic-stage-load-balancer-dns', !Ref DCAProdAccount] # forward schematic.api.sagebionetworks.org to schematic-infra ALB -# https://github.com/Sage-Bionetworks/schematic-infra +# https://github.com/Sage-Bionetworks/schematic-infra-v2 SchematicProdAppDnsForward: Type: update-stacks Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml @@ -185,7 +185,7 @@ SchematicProdAppDnsForward: # ID of the api.sagebionetworks.org zone (in sageit account) SourceHostedZoneId: !CopyValue [!Sub '${primaryRegion}-${resourcePrefix}-sagebio-api-zone-HostedZoneId'] # the value of the CNAME record - TargetHostName: !CopyValue ['schematic-prod-DockerFargateStack-LoadBalancerDNS', !Ref DCAProdAccount] + TargetHostName: !CopyValue ['schematic-prod-load-balancer-dns', !Ref DCAProdAccount] # forward https://genie-bpc.app.sagebionetworks.org to genie-bpc-infra ALB # https://github.com/Sage-Bionetworks/genie-bpc-infra From 0771a7ce68cab347c7ab21fd900cf0b2576d9977 Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:17:47 -0700 Subject: [PATCH 11/19] Set up fowarding for Sage DPE staging/prod domain --- org-formation/800-redirects/_tasks.yaml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index 59d31b2a..05bed490 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -259,3 +259,40 @@ SageDpeDevAppDnsForward: SourceHostedZoneId: "Z04325181I2YIP983P1AD" # the value of the CNAME record TargetHostName: "ac5c848ac4ff54e2bb11dd87685375b0-1875694220.us-east-1.elb.amazonaws.com" + +# forward staging.sagedpe.org to staging EKS stack ALB in org-sagebase-dpe-prod +# apps are setup with terraform at https://github.com/Sage-Bionetworks-Workflows/eks-stack +SageDpeStagingAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-sagedpe-staging-cname' + StackDescription: Setup a CNAME for sagepde.org staging ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "staging.sagedpe.org" + # ID of the sagedpe.org zone (in sageit account) + SourceHostedZoneId: "Z04325181I2YIP983P1AD" + # the value of the CNAME record + TargetHostName: "ae44aad490bd44942875e55a14963d7a-688764136.us-east-1.elb.amazonaws.com" + + +# forward prod.sagedpe.org to prod EKS stack ALB in org-sagebase-dpe-prod +# apps are setup with terraform at https://github.com/Sage-Bionetworks-Workflows/eks-stack +SageDpeProdAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-sagedpe-prod-cname' + StackDescription: Setup a CNAME for sagepde.org prod ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "prod.sagedpe.org" + # ID of the sagedpe.org zone (in sageit account) + SourceHostedZoneId: "Z04325181I2YIP983P1AD" + # the value of the CNAME record + TargetHostName: "aa14266f054574a309d8ec5a2fb2c77c-1977172949.us-east-1.elb.amazonaws.com" From aed64c80df74e105a2d0510e6facf5a6ff7c5654 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Fri, 22 Nov 2024 14:11:02 -0800 Subject: [PATCH 12/19] [IT-3995] Setup DNS redirect for ECS Agora deployment (#1299) Setup redirects for Agora deployment in ECS. We setup temporary names for now. It can be switch over to the vanity plates when we are to use in production. --- org-formation/800-redirects/_tasks.yaml | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index 05bed490..719a971f 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -296,3 +296,58 @@ SageDpeProdAppDnsForward: SourceHostedZoneId: "Z04325181I2YIP983P1AD" # the value of the CNAME record TargetHostName: "aa14266f054574a309d8ec5a2fb2c77c-1977172949.us-east-1.elb.amazonaws.com" + + +# forward agora.dev.adknowledgeportal.org to agora-infra-v3 ALB +# https://github.com/Sage-Bionetworks/agora-infra-v3 +AgoraDevAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-agora-dev-cname' + StackDescription: Setup a CNAME for agora-infra-v3 prod ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "agora.dev.adknowledgeportal.org" + # ID of the adknowledgeportal.org zone (in sageit account) + SourceHostedZoneId: "Z2DTJC6JTFRHBN" + # the value of the CNAME record + TargetHostName: !CopyValue ['agora-dev-load-balancer-dns', !Ref AgoraDevAccount] + +# forward agora.stage.adknowledgeportal.org to agora-infra-v3 ALB +# https://github.com/Sage-Bionetworks/agora-infra-v3 +AgoraStageAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-agora-stage-cname' + StackDescription: Setup a CNAME for agora-infra-v3 prod ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "agora.stage.adknowledgeportal.org" + # ID of the adknowledgeportal.org zone (in sageit account) + SourceHostedZoneId: "Z2DTJC6JTFRHBN" + # the value of the CNAME record + TargetHostName: !CopyValue ['agora-stage-load-balancer-dns', !Ref AgoraProdAccount] + +# forward agora.prod.adknowledgeportal.org to agora-infra-v3 ALB +# https://github.com/Sage-Bionetworks/agora-infra-v3 +AgoraProdAppDnsForward: + Type: update-stacks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.6.9/templates/R53/cname.yaml + StackName: !Sub '${resourcePrefix}-agora-prod-cname' + StackDescription: Setup a CNAME for agora-infra-v3 prod ALB + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + Account: !Ref SageITAccount + Parameters: + # the name of the CNAME record + SourceHostName: "agora.prod.adknowledgeportal.org" + # ID of the adknowledgeportal.org zone (in sageit account) + SourceHostedZoneId: "Z2DTJC6JTFRHBN" + # the value of the CNAME record + TargetHostName: !CopyValue ['agora-prod-load-balancer-dns', !Ref AgoraProdAccount] From 5a718b8b09983f6d1c7e0d305e50506b9b36e002 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:03:17 -0800 Subject: [PATCH 13/19] Agora redirect dns name change (#1300) The certificate used for agora is validated for adknowledgeportal.org domain not the dev, stage, and prod sub domains therefore we can't use sub domains unless we redo the certificate. for simplicity we rename to not use sub domains. --- org-formation/800-redirects/_tasks.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/org-formation/800-redirects/_tasks.yaml b/org-formation/800-redirects/_tasks.yaml index 719a971f..b7d59f4f 100644 --- a/org-formation/800-redirects/_tasks.yaml +++ b/org-formation/800-redirects/_tasks.yaml @@ -310,7 +310,7 @@ AgoraDevAppDnsForward: Account: !Ref SageITAccount Parameters: # the name of the CNAME record - SourceHostName: "agora.dev.adknowledgeportal.org" + SourceHostName: "newagora-dev.adknowledgeportal.org" # ID of the adknowledgeportal.org zone (in sageit account) SourceHostedZoneId: "Z2DTJC6JTFRHBN" # the value of the CNAME record @@ -328,7 +328,7 @@ AgoraStageAppDnsForward: Account: !Ref SageITAccount Parameters: # the name of the CNAME record - SourceHostName: "agora.stage.adknowledgeportal.org" + SourceHostName: "newagora-stage.adknowledgeportal.org" # ID of the adknowledgeportal.org zone (in sageit account) SourceHostedZoneId: "Z2DTJC6JTFRHBN" # the value of the CNAME record @@ -346,7 +346,7 @@ AgoraProdAppDnsForward: Account: !Ref SageITAccount Parameters: # the name of the CNAME record - SourceHostName: "agora.prod.adknowledgeportal.org" + SourceHostName: "newagora-prod.adknowledgeportal.org" # ID of the adknowledgeportal.org zone (in sageit account) SourceHostedZoneId: "Z2DTJC6JTFRHBN" # the value of the CNAME record From c996b589388caf854a27733f6be3d2234a78a2b5 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Mon, 2 Dec 2024 07:59:01 -0800 Subject: [PATCH 14/19] [IT-4025] Remove github OIDC for schematic (#1301) Schematic was updated with a new deployment "GithubOidcSageBionetworksItSchematicInfraV2", therefore we can remove the old deployment OIDC resource. --- .../650-identity-providers/_tasks.yaml | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/org-formation/650-identity-providers/_tasks.yaml b/org-formation/650-identity-providers/_tasks.yaml index 88618c20..f9f015e9 100644 --- a/org-formation/650-identity-providers/_tasks.yaml +++ b/org-formation/650-identity-providers/_tasks.yaml @@ -100,28 +100,6 @@ GithubOidcSageBionetworksDataCuratorInfra: - !Ref DCAProdAccount Region: us-east-1 -GithubOidcSageBionetworksSchematicInfra: - Type: update-stacks - DependsOn: GithubOidcSageBionetworks - Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.7.6/templates/IAM/github-oidc-provider.j2 - StackName: !Sub ${resourcePrefix}-${appName}-sage-bionetworks-schematic-infra - Parameters: - ProviderArn: !CopyValue [ !Sub '${resourcePrefix}-${appName}-ProviderArn' ] - ProviderRoleName: !Sub ${resourcePrefix}-${appName}-sage-bionetworks-schematic-infra - ManagedPolicyArns: - - "arn:aws:iam::aws:policy/AdministratorAccess" - - "arn:aws:iam::aws:policy/AWSKeyManagementServicePowerUser" - TemplatingContext: - GitHubOrg: "Sage-Bionetworks" - Repositories: - - name: "schematic-infra" - branches: ["*"] - DefaultOrganizationBinding: - Account: - - !Ref DnTDevAccount - - !Ref DCAProdAccount - Region: us-east-1 - GithubOidcSageBionetworksItSchematicInfraV2: Type: update-stacks DependsOn: GithubOidcSageBionetworks From 7cd62f7f6ef42179a3dd27d94a726de97054a087 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:12:20 -0800 Subject: [PATCH 15/19] [pre-commit.ci] pre-commit autoupdate (#1249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/awslabs/cfn-python-lint: v1.15.2 → v1.20.1](https://github.com/awslabs/cfn-python-lint/compare/v1.15.2...v1.20.1) - [github.com/sirosen/check-jsonschema: 0.29.3 → 0.30.0](https://github.com/sirosen/check-jsonschema/compare/0.29.3...0.30.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c76e2ed..6bfa2488 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: entry: git-secrets args: [--scan, --recursive] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: # On Windows, git will convert all CRLF to LF, but only after all hooks are done executing. # yamllint will fail before git has a chance to convert line endings, so it must be explicitly done before yamllint @@ -20,7 +20,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/awslabs/cfn-python-lint - rev: v1.15.2 + rev: v1.20.1 hooks: - id: cfn-python-lint files: templates/.*\.(json|yml|yaml)$ @@ -30,7 +30,7 @@ repos: hooks: - id: remove-tabs - repo: https://github.com/sirosen/check-jsonschema - rev: 0.29.3 + rev: 0.30.0 hooks: - id: check-github-workflows - id: check-github-actions From 462d10e35790c3ed29b47ea7554d7ddb7d43c745 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Wed, 4 Dec 2024 07:51:29 -0800 Subject: [PATCH 16/19] [IT-4003] Auto-update pre-commit hook versions monthly (#1303) Change the frequency that PRs to update pre-commit hook versions are auto-generated from weekly (the default) to monthly. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bfa2488..3ee0d256 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +ci: + autoupdate_schedule: monthly repos: - repo: https://github.com/awslabs/git-secrets rev: b9e96b3212fa06aea65964ff0d5cda84ce935f38 From 1a7eae98beca9aaf61536832109ec41c753e3381 Mon Sep 17 00:00:00 2001 From: Xavier Schildwachter Date: Thu, 5 Dec 2024 09:10:03 -0800 Subject: [PATCH 17/19] PLFM-8102: Use Admin Access (#1254) * Use cdk role * Park * Use inline policy to allow assuming cdk* roles * use admin access * combine synapse-related oidc --- .../650-identity-providers/_tasks.yaml | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/org-formation/650-identity-providers/_tasks.yaml b/org-formation/650-identity-providers/_tasks.yaml index f9f015e9..ae1326be 100644 --- a/org-formation/650-identity-providers/_tasks.yaml +++ b/org-formation/650-identity-providers/_tasks.yaml @@ -144,6 +144,30 @@ GithubOidcSageBionetworksSynapseDockerRegistry: - !Ref SynapseProdAccount Region: us-east-1 +GithubOidcSageBionetworksSynapse: + Type: update-stacks + DependsOn: GithubOidcSageBionetworks + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.7.6/templates/IAM/github-oidc-provider.j2 + StackName: !Sub ${resourcePrefix}-${appName}-sage-bionetworks-synapse + Parameters: + ProviderArn: !CopyValue [ !Sub '${resourcePrefix}-${appName}-ProviderArn' ] + ProviderRoleName: !Sub ${resourcePrefix}-${appName}-sage-bionetworks-synapse + ManagedPolicyArns: + - "arn:aws:iam::aws:policy/AdministratorAccess" + - "arn:aws:iam::aws:policy/AWSKeyManagementServicePowerUser" + TemplatingContext: + GitHubOrg: "Sage-Bionetworks" + Repositories: + - name: "synapse-docker-registry" + branches: ["*"] + - name: "nbconvert-webapp" + branches: ["master", "develop"] + DefaultOrganizationBinding: + Account: + - !Ref SynapseDevAccount + - !Ref SynapseProdAccount + Region: us-east-1 + GithubOidcSageBionetworksGenieBPCInfra: Type: update-stacks DependsOn: GithubOidcSageBionetworks @@ -850,32 +874,6 @@ GithubOidcOpenChallengesDeploy: - !Ref OpenChallengesProdAccount Region: us-east-1 -GithubOidcNbConvertDeploy: - Type: update-stacks - DependsOn: GithubOidcSageBionetworks - Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.7.6/templates/IAM/github-oidc-provider.j2 - StackName: !Sub ${resourcePrefix}-${appName}-nbconvert-deploy - Parameters: - ProviderArn: !CopyValue [ !Sub '${resourcePrefix}-${appName}-ProviderArn' ] - ProviderRoleName: !Sub ${resourcePrefix}-${appName}-nbconvert-deploy - MaxSessionDuration: 7200 - ManagedPolicyArns: - - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess" - - "arn:aws:iam::aws:policy/AWSLambda_FullAccess" - - "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" - - "arn:aws:iam::aws:policy/IAMFullAccess" - - "arn:aws:iam::aws:policy/AWSCloudFormationFullAccess" - TemplatingContext: - GitHubOrg: "Sage-Bionetworks" - Repositories: - - name: "nbconvert-webapp" - branches: ["master", "develop"] - DefaultOrganizationBinding: - Account: - - !Ref SynapseDevAccount - - !Ref SynapseProdAccount - Region: us-east-1 - ############################### Managed Policies ############################### # Managed policies used in github OIDC providers # Note: Managed policies can be used as work around for the AWS cloudformation From 41e656562591491ee524b7a29afc5882be072987 Mon Sep 17 00:00:00 2001 From: Khai Do <3697686+zaro0508@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:19:33 -0800 Subject: [PATCH 18/19] [IT-4031] Add openchallenges developer SSO access (#1304) Setup developer access to AWS org-sagebase-openchallenges-dev account. --- org-formation/700-aws-sso/_tasks.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/org-formation/700-aws-sso/_tasks.yaml b/org-formation/700-aws-sso/_tasks.yaml index 3041fa08..a3637a6a 100644 --- a/org-formation/700-aws-sso/_tasks.yaml +++ b/org-formation/700-aws-sso/_tasks.yaml @@ -313,6 +313,10 @@ Parameters: Type: String Default: '2448e4e8-50b1-70e5-def0-07e0f4fcd60e' + OpenchallengesDevDeveloperGroup: # JC aws-openchallenges-dev-developers + Type: String + Default: '44183438-a051-7070-f706-284ffd41907b' + OpenchallengesDevAdminGroup: # JC aws-openchallenges-dev-admins Type: String Default: 'e4388458-2011-7096-3f98-3a6eeb10e458' @@ -2164,6 +2168,23 @@ SsoItsandboxDeveloper: principalId: !Ref itsandboxDeveloperGroup permissionSetArn: !CopyValue [ !Sub '${resourcePrefix}-${appName}-developer-permission-set-arn' ] +SsoOpenchallengesDevDeveloper: + Type: update-stacks + DependsOn: SsoDeveloper + Template: https://raw.githubusercontent.com/Sage-Bionetworks/aws-infra/v0.3.8/templates/SSO/aws-sso.yaml + StackName: !Sub '${resourcePrefix}-${appName}-openchallenges-dev-developer' + StackDescription: 'SSO: Developer role used by openchallenges developer group' + DefaultOrganizationBindingRegion: !Ref primaryRegion + DefaultOrganizationBinding: + IncludeMasterAccount: true + OrganizationBindings: + TargetBinding: + Account: !Ref OpenChallengesDevAccount + Parameters: + instanceArn: !Ref instanceArn + principalId: !Ref OpenchallengesDevDeveloperGroup + permissionSetArn: !CopyValue [ !Sub '${resourcePrefix}-${appName}-developer-permission-set-arn' ] + SsoOpenchallengesDevAdmin: Type: update-stacks DependsOn: SsoAdministrator From 4ef56e35fc47bc319ed10db6f18c3d46c61e8748 Mon Sep 17 00:00:00 2001 From: Marco Marasca <8505576+marcomarasca@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:35:40 -0800 Subject: [PATCH 19/19] PLFM-8614: Add aoss permissions to deployer (#1307) --- sceptre/synapsedev/templates/SynapseCMK-template.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sceptre/synapsedev/templates/SynapseCMK-template.json b/sceptre/synapsedev/templates/SynapseCMK-template.json index 5dfd6035..4be552e6 100644 --- a/sceptre/synapsedev/templates/SynapseCMK-template.json +++ b/sceptre/synapsedev/templates/SynapseCMK-template.json @@ -247,6 +247,11 @@ ], "Resource": "*" }, + { + "Effect": "Allow", + "Action": [ "aoss:*" ], + "Resource": "*" + }, { "Effect": "Allow", "Action": [