diff --git a/iac/cdk.context.json b/iac/cdk.context.json index 6262754..9d7f815 100644 --- a/iac/cdk.context.json +++ b/iac/cdk.context.json @@ -36,5 +36,9 @@ ] } ] + }, + "hosted-zone:account=491085404175:domainName=zenml.mlops-club.org:region=us-west-2": { + "Id": "/hostedzone/Z06294602217HSCV3Z0OS", + "Name": "zenml.mlops-club.org." } } diff --git a/iac/src/iac/user-data.template.sh b/iac/src/iac/user-data.template.sh index d428b64..f7447fd 100644 --- a/iac/src/iac/user-data.template.sh +++ b/iac/src/iac/user-data.template.sh @@ -47,13 +47,23 @@ services: ports: - 3306:3306 environment: - - MYSQL_ROOT_PASSWORD=password + MYSQL_ROOT_PASSWORD: password + volumes: + - ./volumes/mysql/data/:/var/lib/mysql + - ./volumes/mysql/backups/:/backups + # TODO: mount the mysql state as a volume + zenml: image: zenmldocker/zenml-server ports: - "8080:8080" environment: - - ZENML_STORE_URL=mysql://root:password@mysql/zenml + ZENML_STORE_URL: mysql://root:password@mysql/zenml + ZENML_STORE_BACKUP_DIRECTORY: /backups + ZENML_STORE_BACKUP_STRATEGY: dump-file + ZENML_ENABLE_IMPLICIT_AUTH_METHODS: "true" + volumes: + - ./volumes/mysql/data/:/var/lib/mysql links: - mysql depends_on: @@ -61,6 +71,7 @@ services: # extra_hosts: # - "host.docker.internal:host-gateway" restart: on-failure + EOF ########################################## diff --git a/iac/src/iac/zenml_components_stack.py b/iac/src/iac/zenml_components_stack.py new file mode 100644 index 0000000..ca57266 --- /dev/null +++ b/iac/src/iac/zenml_components_stack.py @@ -0,0 +1,37 @@ +from pathlib import Path +from string import Template +from typing import Optional + +import aws_cdk as cdk +from aws_cdk import Stack +from aws_cdk import aws_cloudwatch as cloudwatch +from aws_cdk import aws_ec2 as ec2 +from aws_cdk import aws_ecr as ecr +from aws_cdk import aws_iam as iam +from aws_cdk import aws_route53 as route53 +from aws_cdk import aws_s3 as s3 +from constructs import Construct + +class ZenMLComponents(Stack): + """Stack responsible for creating the running minecraft server on AWS. + + :param scope: The scope of the stack. + :param construct_id: The ID of the stack. + :param minecraft_server_version: The semantic version of the ZenML server to install. + :param backup_service_ecr_repo_arn: The ARN of the ECR repository for the backup service. + :param backup_service_docker_image_uri: The URI of the Docker image in ECR for the backup service. + :param minecraft_server_backups_bucket_name: The name of the S3 bucket to store backups in. + :param ssh_key_pair_name: The name of the SSH key pair to use for the EC2 instance. + """ + + def __init__( + self, + scope: Construct, + construct_id: str, + ssh_key_pair_name: Optional[str] = None, + custom_top_level_domain_name: Optional[str] = None, + ec2_instance_type: Optional[str] = "t3.medium", + **kwargs, + ) -> None: + + ... \ No newline at end of file diff --git a/iac/src/iac/zenml_control_plane.py b/iac/src/iac/zenml_control_plane.py index 5dcfa03..6d33900 100644 --- a/iac/src/iac/zenml_control_plane.py +++ b/iac/src/iac/zenml_control_plane.py @@ -15,6 +15,8 @@ THIS_DIR = Path(__file__).parent USER_DATA_SH_TEMPLATE_FPATH = (THIS_DIR / "./user-data.template.sh").resolve() +BUCKET_NAME = "mlops-club-zeml-hackathon-bucket" +ECR_REPO_NAME = "zenml-hackathon-repo" class ZenMLControlPlaneStack(Stack): """Stack responsible for creating the running minecraft server on AWS. @@ -39,6 +41,18 @@ def __init__( ) -> None: super().__init__(scope, construct_id, **kwargs) + artifacts_bucket = s3.Bucket.from_bucket_name( + scope=self, + id="ArtifactsBucket", + bucket_name=BUCKET_NAME, + ) + + ecr_repo = ecr.Repository.from_repository_name( + scope=self, + id="ZenMLHackathonRepo", + repository_name=ECR_REPO_NAME, + ) + _vpc = ec2.Vpc.from_lookup(scope=self, id="DefaultVpc", is_default=True) # set up security group to allow inbound traffic on port 25565 for anyone @@ -65,7 +79,7 @@ def __init__( description="Allow all outbound traffic", ) - # create iam role for ec2 instance using AmazonSSMManagedInstanceCore + # create iam role for ec2 instance _iam_role = iam.Role( scope=self, id="ZenMLServerIamRole", @@ -74,6 +88,31 @@ def __init__( _iam_role.add_managed_policy( iam.ManagedPolicy.from_aws_managed_policy_name("AmazonSSMManagedInstanceCore") ) + + # Add S3 permissions + _iam_role.add_to_policy(iam.PolicyStatement( + actions=[ + "s3:ListAllMyBuckets", + "s3:GetBucketLocation", + "s3:ListBucket", + ], + resources=["arn:aws:s3:::*"] + )) + + # Add ECR permissions + _iam_role.add_to_policy(iam.PolicyStatement( + actions=[ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + ], + resources=["*"] + )) # fill in user data script _user_data_script = ec2.UserData.custom( @@ -96,6 +135,30 @@ def __init__( key_name=ssh_key_pair_name, ) + # Add the A Record for zenml.mlops-club.io + hosted_zone = route53.HostedZone.from_lookup( + self, "ZenMLHostedZone", domain_name="zenml.mlops-club.org" + ) + + route53.ARecord( + self, + "ZenMLARecord", + zone=hosted_zone, + target=route53.RecordTarget.from_ip_addresses(_ec2.instance_public_ip), + record_name="zenml.mlops-club.org" + ) + + # Add a new output for the domain + cdk.CfnOutput( + self, + id="ZenMLServerDomain", + value="zenml.mlops-club.org", + description="The domain name of the ZenML server", + ) + + ecr_repo.grant_pull_push(_iam_role) + artifacts_bucket.grant_read_write(_iam_role) + if custom_top_level_domain_name: a_record: route53.ARecord = add_custom_subdomain_to_ec2_ip( scope=self, diff --git a/setup-zenml.sh b/setup-zenml.sh index 54bb094..be7b0c6 100755 --- a/setup-zenml.sh +++ b/setup-zenml.sh @@ -66,10 +66,12 @@ zenml artifact-store register "s3-store" \ # ┃ bbee06f52f │ │ │ │ thon-bucket ┃ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +# /zenml-hackathon-repo +ECR_REPO_URI="491085404175.dkr.ecr.us-west-2.amazonaws.com" zenml container-registry register "ecr-docker-image-store" \ --flavor aws \ --connector ecr-connector \ - --uri="$ECR_REGISTRY_URI" + --uri=$ECR_REPO_URI # Successfully connected container registry `ecr-docker-image-store` to the following resources: # ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ @@ -82,7 +84,6 @@ zenml container-registry register "ecr-docker-image-store" \ zenml stack register "local-s3-ecr" \ --artifact-store s3-store \ --container_registry ecr-docker-image-store \ - --image_builder default \ --orchestrator default # Stack 'local-s3-ecr' successfully registered!