diff --git a/README.md b/README.md index 21d1688..560d9eb 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,29 @@ Everyone is very welcome to contribute to this project. You can contribute just by submitting bugs or suggesting improvements by opening an issue on GitHub. +## useful commands + +if you have a local buildbot you can check config before uploading it: + +```bash +buildbot checkconfig config/admin/admin.cfg +``` + +deploy new config: +```bash +npm run zip-config && cdk deploy --force BuildBotConfig-Personal +``` + +debug ecs container: +```bash +aws ecs execute-command --cluster XXX --task XXX --container buildbot-server --interactive --command "/bin/bash" +``` + +rebuild, redeploy everything: +```bash +npm run clean && npm run build && npm run zip-config && cdk deploy --all --force +``` + ## License diff --git a/configuration/admin/Dockerfile b/configuration/admin/Dockerfile index 7057c1e..a5e15fd 100644 --- a/configuration/admin/Dockerfile +++ b/configuration/admin/Dockerfile @@ -26,7 +26,7 @@ RUN buildbot create-master . COPY admin.cfg . COPY envfile . COPY entry.sh . -RUN mkdir userconfig +RUN mkdir userconfiguration RUN chown -R user: userconfiguration/ COPY user.cfg userconfiguration/ diff --git a/configuration/admin/admin.cfg b/configuration/admin/admin.cfg index bfcb453..3bccd27 100644 --- a/configuration/admin/admin.cfg +++ b/configuration/admin/admin.cfg @@ -14,11 +14,20 @@ c = BuildmasterConfig = {} init_script = '''#!/bin/bash exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +# add a debuguser +useradd -p '*' debuguser + +# https://wiki.yoctoproject.org/wiki/How_to_enable_KVM_for_Poky_qemu +sudo groupadd --system kvm +sudo gpasswd -a ubuntu kvm +sudo chown root:kvm /dev/kvm +sudo chmod 0660 /dev/kvm + sudo apt-get update sudo apt install buildbot-worker xfsprogs nfs-common curl -y # install additional packages from yocto https://docs.yoctoproject.org/brief-yoctoprojectqs/index.html -sudo apt install gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev xterm python3-subunit mesa-common-dev zstd liblz4-tool -y +sudo apt install gawk wget git diffstat unzip texinfo gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev xterm python3-subunit mesa-common-dev zstd liblz4-tool libtinfo5 mtools -y # mount sstate volume @@ -87,9 +96,13 @@ if 'BUILDBOT_MQ_URL' in os.environ: # could connect to your master with this protocol. # 'port' must match the value configured into the workers nvme1 +####### GIT POLLER DIR + +gitpoller_workdir = '/mount/data/gitpoller_workdir' + ####reload user.cfg factory reload_config_factory = util.BuildFactory() -reload_config_factory.addStep(steps.MasterShellCommand(command="git clone " + os.environ.get('CODECOMMIT_REPOSITORY_CLONE_URL_GRC', 'default') + " && rm -rf userconfiguration/* && cp buildbot-user-repo/* userconfiguration/ && rm -rf buildbot-user-repo && cat admin.cfg > master.cfg && cat userconfiguration/user.cfg >> master.cfg", env={'PATH': ["/home/user","${PATH}"]}, name="checkout changed configuration", description=" user.py is stored in a codecommit repo")) +reload_config_factory.addStep(steps.MasterShellCommand(command="git clone " + os.environ.get('CODECOMMIT_REPOSITORY_CLONE_URL_GRC', 'default') + " && rm -rf userconfig/* && cp buildbot-user-repo/* userconfig/ && rm -rf buildbot-user-repo && cat admin.cfg > master.cfg && cat userconfig/user.cfg >> master.cfg", env={'PATH': ["/home/user","${PATH}"]}, name="checkout changed configuration", description=" user.py is stored in a codecommit repo")) reload_config_factory.addStep(steps.MasterShellCommand(command="killall -SIGHUP python3", name="reload buildbot configuration", description="will send a SIGHUP to buildbot process")) c['workers'].append(worker.LocalWorker('bot1')) @@ -98,13 +111,16 @@ c['builders'].append(util.BuilderConfig(name="reload_config_factory",workernames c['change_source'].append(changes.GitPoller( os.environ.get('CODECOMMIT_REPOSITORY_CLONE_URL_GRC', 'default'), - workdir='gitpoller-workdir', branch='main', - pollinterval=60)) + workdir='gitpoller-workdir', + branches='main', + pollinterval=600, + pollRandomDelayMin=300, + pollRandomDelayMax=500)) c['schedulers'].append(schedulers.SingleBranchScheduler( name="tree_stable_reload_buildbot_configuration", change_filter=util.ChangeFilter(branch="main",repository=os.environ.get('CODECOMMIT_REPOSITORY_CLONE_URL_GRC', 'default')), - treeStableTimer=60*2, + treeStableTimer=10, builderNames=["reload_config_factory"])) ####### REPORTER TARGETS @@ -150,15 +166,66 @@ c['protocols'] = {'pb': {'port': 9989}} c['buildbotNetUsageData'] = None +######## AMI +# find the correct one for your region here: https://cloud-images.ubuntu.com/locator/ec2/ + + +# us-west-2 +# x86_64_ami = 'ami-0ee3d9a8776e8b99c' +# arm64_ami = 'ami-0817b9078575a7d59' + +# eu-central-1 +x86_64_ami = 'ami-088a4bdad130d7795' +arm64_ami = 'ami-07b9d4b65220d2cad' + + + +######## SSTATE +sstate_efs_dns_name = os.environ.get('BUILDBOT_WORKER_SSTATE_EFS_FS_ID', 'default')+".efs."+ os.environ.get('AWS_REGION', 'default') + ".amazonaws.com" + + +##### clean sstate ####################################################################################################### + +clean_sstate_factory = util.BuildFactory() +clean_sstate_factory.addStep(steps.ShellCommand(command='cd /sstate && rm -rfv *')) +workername = "clean_sstate_worker" +c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c5.metal', + ami=x86_64_ami, + build_wait_timeout=0, + keypair_name='worker-key', + security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),], + subnet_id=os.environ.get('BUILDBOT_WORKER_SUBNET', 'default'), + user_data=init_script.format(sstate_efs_dns_name, workername), + block_device_map= [ + { + "DeviceName": "/dev/xvdf", + "Ebs" : { + "VolumeType": "gp3", + "Iops": 16000, + "VolumeSize": 500, + "Throughput": 1000, + } + } + ], + )) +buildername = "clean_sstate_builder" +c['builders'].append( + util.BuilderConfig(name=buildername, + workernames=[workername], + factory=clean_sstate_factory)) +schedulername = "clean_sstate" +c['schedulers'].append( + schedulers.ForceScheduler( + name=schedulername, + builderNames=[buildername], + buttonName="force")) + #################################################################################### # # this should be in user.py # #################################################################################### -x86_64_ami = 'ami-0ee3d9a8776e8b99c' -arm64_ami = 'ami-0817b9078575a7d59' - ####### INSTANCE TEST factory2 = util.BuildFactory() @@ -171,8 +238,6 @@ factory2.addStep(steps.ShellCommand(command="./test.sh", name="test", descriptio x86_64_instance_types = ["m6i.metal", "c6a.8xlarge", "m6i.8xlarge","m6in.16xlarge","m6a.24xlarge","m6i.24xlarge","c6i.24xlarge","c6a.24xlarge"] arm_64_instance_types = ["c6g.metal", "c7g.metal", "c6g.16xlarge","c7g.16xlarge"] -sstate_efs_dns_name = os.environ.get('BUILDBOT_WORKER_SSTATE_EFS_FS_ID', 'default')+".efs."+ os.environ.get('AWS_REGION', 'default') + ".amazonaws.com" - for instance in x86_64_instance_types: c['workers'].append(worker.EC2LatentWorker(instance.replace(".", "_"), 'sekrit', instance, ami=x86_64_ami, @@ -229,17 +294,43 @@ for instance in x86_64_instance_types + arm_64_instance_types: ##### meta-aws release tests ####################################################################################################### meta_aws_supported_releases = ["master", "kirkstone", "dunfell", "mickledore"] -meta_aws_supported_archs = ["qemux86-64", "qemuarm64"] +meta_aws_supported_releases_next = ["master-next", "kirkstone-next", "dunfell-next", "mickledore-next"] +meta_aws_supported_x86_archs = ["qemux86-64"] +meta_aws_supported_arm_archs = ["qemuarm64"] + +c['change_source'].append(changes.GitPoller( + "https://github.com/aws4embeddedlinux/meta-aws.git", + workdir=gitpoller_workdir, + branches=meta_aws_supported_releases_next, + pollinterval=3600, + pollRandomDelayMin=300, + pollRandomDelayMax=3500)) + +c['change_source'].append(changes.GitPoller( + "https://github.com/yoctoproject/poky.git", + workdir=gitpoller_workdir, + branches=meta_aws_supported_releases, + pollinterval=3600, + pollRandomDelayMin=300, + pollRandomDelayMax=3500)) + +c['change_source'].append(changes.GitPoller( + "https://github.com/openembedded/meta-openembedded.git", + workdir=gitpoller_workdir, + branches=meta_aws_supported_releases, + pollinterval=3600, + pollRandomDelayMin=300, + pollRandomDelayMax=3500)) for release in meta_aws_supported_releases: - for arch in meta_aws_supported_archs: - meta_aws_release_factory = util.BuildFactory() - meta_aws_release_factory.addStep(steps.Git(repourl='https://github.com/aws4embeddedlinux/meta-aws-ci.git', mode='full', submodules=True)) - meta_aws_release_factory.addStep(steps.ShellCommand(command='cd release-tests/ && BB_ENV_PASSTHROUGH_ADDITIONS="SSTATE_DIR $BB_ENV_PASSTHROUGH_ADDITIONS" SSTATE_DIR="/sstate" ./meta-aws-release-tests.sh ' + release + ' ' + '"' + arch +'"')) + for arch in meta_aws_supported_x86_archs: + meta_aws_release_factory = util.BuildFactory() + meta_aws_release_factory.addStep(steps.ShellCommand(command='git clone https://github.com/aws4embeddedlinux/meta-aws-ci.git --depth=1 --single-branch')) + meta_aws_release_factory.addStep(steps.ShellCommand(command='cd meta-aws-ci/meta-aws-tests/ && BB_ENV_PASSTHROUGH_ADDITIONS="SSTATE_DIR $BB_ENV_PASSTHROUGH_ADDITIONS" SSTATE_DIR="/sstate" ./meta-aws-tests.sh --releases=' + release + ' --arch='+ arch)) workername = "meta_aws_release_worker_" + release + "_" + arch.replace("-", "_") - c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c7g.16xlarge', - ami=arm64_ami, + c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c5.metal', + ami=x86_64_ami, build_wait_timeout=0, keypair_name='worker-key', security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),], @@ -271,50 +362,135 @@ for release in meta_aws_supported_releases: builderNames=[buildername], buttonName="force")) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="check_meta_aws_" + release + "_" + arch.replace("-", "_"), + change_filter=util.ChangeFilter(branch=release+"-next",repository="https://github.com/aws4embeddedlinux/meta-aws.git"), + treeStableTimer=30*60, + builderNames=[buildername])) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="check_poky_" + release + "_" + arch.replace("-", "_"), + change_filter=util.ChangeFilter(branch=release,repository="https://github.com/yoctoproject/poky.git"), + treeStableTimer=30*60, + builderNames=[buildername])) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="check_oe_" + release + "_" + arch.replace("-", "_"), + change_filter=util.ChangeFilter(branch=release,repository="https://github.com/openembedded/meta-openembedded.git"), + treeStableTimer=30*60, + builderNames=[buildername])) + + for arch in meta_aws_supported_arm_archs: + meta_aws_release_factory = util.BuildFactory() + meta_aws_release_factory.addStep(steps.ShellCommand(command='git clone https://github.com/aws4embeddedlinux/meta-aws-ci.git --depth=1 --single-branch')) + meta_aws_release_factory.addStep(steps.ShellCommand(command='cd meta-aws-ci/meta-aws-tests/ && BB_ENV_PASSTHROUGH_ADDITIONS="SSTATE_DIR $BB_ENV_PASSTHROUGH_ADDITIONS" SSTATE_DIR="/sstate" ./meta-aws-tests.sh --releases=' + release + ' --arch='+ arch)) + + workername = "meta_aws_release_worker_" + release + "_" + arch.replace("-", "_") + c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c7g.metal', + ami=arm64_ami, + build_wait_timeout=0, + keypair_name='worker-key', + security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),], + subnet_id=os.environ.get('BUILDBOT_WORKER_SUBNET', 'default'), + user_data=init_script.format(sstate_efs_dns_name, workername), + block_device_map= [ + { + "DeviceName": "/dev/xvdf", + "Ebs" : { + "VolumeType": "gp3", + "Iops": 16000, + "VolumeSize": 500, + "Throughput": 1000, + } + } + ], + )) + + buildername = "meta_aws_release_builder_" + release + "_" + arch.replace("-", "_") + c['builders'].append( + util.BuilderConfig(name=buildername, + workernames=[workername], + factory=meta_aws_release_factory)) + + schedulername = "meta-aws_release_tests_" + release + "_" + arch.replace("-", "_") c['schedulers'].append( - schedulers.Nightly(name="weekly-meta_aws_release_builder_" + release + "_" + arch.replace("-", "_"), - builderNames=[buildername], - dayOfWeek=6, hour=0, minute=1)) + schedulers.ForceScheduler( + name=schedulername, + builderNames=[buildername], + buttonName="force")) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="check_meta_aws_" + release + "_" + arch.replace("-", "_"), + change_filter=util.ChangeFilter(branch=release+"-next",repository="https://github.com/aws4embeddedlinux/meta-aws.git"), + treeStableTimer=30*60, + builderNames=[buildername])) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="check_poky_" + release + "_" + arch.replace("-", "_"), + change_filter=util.ChangeFilter(branch=release,repository="https://github.com/yoctoproject/poky.git"), + treeStableTimer=30*60, + builderNames=[buildername])) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="check_oe_" + release + "_" + arch.replace("-", "_"), + change_filter=util.ChangeFilter(branch=release,repository="https://github.com/openembedded/meta-openembedded.git"), + treeStableTimer=30*60, + builderNames=[buildername])) ##### meta-aws-demos tests ####################################################################################################### -meta_aws_release_factory = util.BuildFactory() -meta_aws_release_factory.addStep(steps.Git(repourl='https://github.com/aws4embeddedlinux/meta-aws-demos.git', mode='full', submodules=True)) -meta_aws_release_factory.addStep(steps.ShellCommand(command=['/bin/bash', '-c', '''. init-build-env && BB_ENV_PASSTHROUGH_ADDITIONS="SSTATE_DIR $BB_ENV_PASSTHROUGH_ADDITIONS" SSTATE_DIR="/sstate" && for d in $(get_devices); do BUILD_DEVICE=$d bitbake aws-greengrass-test-image; done '''])) -workername = "meta_aws_demos_release_worker" -c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c7g.16xlarge', - ami=arm64_ami, - build_wait_timeout=0, - keypair_name='worker-key', - security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),], - subnet_id=os.environ.get('BUILDBOT_WORKER_SUBNET', 'default'), - user_data=init_script.format(sstate_efs_dns_name, workername), - block_device_map= [ - { - "DeviceName": "/dev/xvdf", - "Ebs" : { - "VolumeType": "gp3", - "Iops": 16000, - "VolumeSize": 500, - "Throughput": 1000, + +c['change_source'].append(changes.GitPoller( + "https://github.com/aws4embeddedlinux/meta-aws-demos.git", + workdir=gitpoller_workdir, + branches=True, + pollinterval=600, + pollRandomDelayMin=300, + pollRandomDelayMax=500)) + + +meta_aws_demos_supported_branches = ["master", "master-next"] + +for branch in meta_aws_demos_supported_branches: + meta_aws_release_factory = util.BuildFactory() + meta_aws_release_factory.addStep(steps.Git(repourl='https://github.com/aws4embeddedlinux/meta-aws-demos.git', mode='full', submodules=True, branch=branch)) + meta_aws_release_factory.addStep(steps.ShellCommand(command=['/bin/bash', '-c', '''. init-build-env && for d in $(get_devices); do BB_ENV_PASSTHROUGH_ADDITIONS="SSTATE_DIR $BB_ENV_PASSTHROUGH_ADDITIONS" SSTATE_DIR="/sstate" BUILD_DEVICE=$d bitbake aws-biga-image; done '''])) + workername = "meta_aws_demos_release_worker_"+branch + c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c5.metal', + ami=x86_64_ami, + build_wait_timeout=0, + keypair_name='worker-key', + security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),], + subnet_id=os.environ.get('BUILDBOT_WORKER_SUBNET', 'default'), + user_data=init_script.format(sstate_efs_dns_name, workername), + block_device_map= [ + { + "DeviceName": "/dev/xvdf", + "Ebs" : { + "VolumeType": "gp3", + "Iops": 16000, + "VolumeSize": 500, + "Throughput": 1000, + } } - } - ], - )) -buildername = "meta_aws_demos_release_builder" -c['builders'].append( - util.BuilderConfig(name=buildername, - workernames=[workername], - factory=meta_aws_release_factory)) -schedulername = "meta-aws_demos_release_tests" -c['schedulers'].append( - schedulers.ForceScheduler( - name=schedulername, - builderNames=[buildername], - buttonName="force")) -c['schedulers'].append( - schedulers.Nightly(name="weekly-meta_aws_demos_release_builder", - builderNames=[buildername], - dayOfWeek=6, hour=0, minute=1)) + ], + )) + buildername = "meta_aws_demos_release_builder_"+branch + c['builders'].append( + util.BuilderConfig(name=buildername, + workernames=[workername], + factory=meta_aws_release_factory)) + schedulername = "meta-aws_demos_release_tests_"+branch + c['schedulers'].append( + schedulers.ForceScheduler( + name=schedulername, + builderNames=[buildername], + buttonName="force")) + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="meta_aws_demos_release_builder_treestable_"+branch, + change_filter=util.ChangeFilter(branch=branch,repository="https://github.com/aws4embeddedlinux/meta-aws-demos.git"), + treeStableTimer=30*60, + builderNames=[buildername])) + + +# c['schedulers'].append( +# schedulers.Nightly(name="weekly-meta_aws_demos_release_builder_"+branch, +# builderNames=[buildername], hour=3, minute=1)) + #### user.cfg gets pasted at the end of this file #### diff --git a/lib/app.ts b/lib/app.ts index 560fce1..5e6b229 100644 --- a/lib/app.ts +++ b/lib/app.ts @@ -18,7 +18,7 @@ if (dev) { const env = { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }; - const terminationProtection = true; + const terminationProtection = false; /// App stacks const vpc = new Vpc(app, 'Vpc', { @@ -91,7 +91,7 @@ if (dev) { env, terminationProtection, bucket: buildBotConfig.bucket, - bucketKey: 'admin-configuration/config.zip', + bucketKey: 'admin-config/config.zip', service: buildBotServer.service, repo: buildBotImageRepo.repo, configrepo: buildBotConfig.configrepo, diff --git a/lib/buildbot/buildbot-server.ts b/lib/buildbot/buildbot-server.ts index 1272640..85aa4f9 100644 --- a/lib/buildbot/buildbot-server.ts +++ b/lib/buildbot/buildbot-server.ts @@ -70,17 +70,17 @@ export class BuildBotServer extends cdk.Stack { constructor(scope: cdk.App, id: string, props: BuildBotServerProps) { super(scope, id, { ...props }); - // Temporary SG to allow only Corp access to our insecure service. Later we will use OIDC. - const buildbotSg = new SecurityGroup(this, 'BuildBotCorpPrefix', { - vpc: props.vpc, - description: 'BuildBot Web Access From Corp.', - }); - - if (props.dns) { - buildbotSg.addIngressRule(Peer.anyIpv4(), Port.tcp(443)); - } else { - buildbotSg.addIngressRule(Peer.prefixList('pl-f8a64391'), Port.tcp(80)); - } + // // Temporary SG to allow only Corp access to our insecure service. Later we will use OIDC. + // const buildbotSg = new SecurityGroup(this, 'BuildBotCorpPrefix', { + // vpc: props.vpc, + // description: 'BuildBot Web Access From Corp.', + // }); + + // if (props.dns) { + // buildbotSg.addIngressRule(Peer.anyIpv4(), Port.tcp(443)); + // } else { + // buildbotSg.addIngressRule(Peer.prefixList('pl-f8a64391'), Port.tcp(80)); + // } // Set up a SG for worker nodes to use. this.workerSecurityGroup = new SecurityGroup(this, 'BuildBotWorkerSg', { @@ -103,8 +103,8 @@ export class BuildBotServer extends cdk.Stack { // Set up the ECS Server. const buildBotServerTask = new ecs.TaskDefinition(this, 'BuildBotServerTask', { compatibility: ecs.Compatibility.FARGATE, - cpu: '1024', - memoryMiB: '2048', + cpu: '4096', + memoryMiB: '8192', volumes: [ { name: 'DBMount', @@ -159,7 +159,7 @@ export class BuildBotServer extends cdk.Stack { const webLb = new ApplicationLoadBalancer(this, 'WebLB', { vpc: props.vpc, internetFacing: true, - securityGroup: buildbotSg, + // securityGroup: buildbotSg, }); const loadBalancerAccessLogs = new Bucket(this, 'LoadBalancerLogs', {}); @@ -183,11 +183,13 @@ export class BuildBotServer extends cdk.Stack { port: 8010, targets: [buildBotService], protocol: ApplicationProtocol.HTTP, + deregistrationDelay: Duration.seconds(5), }); webTargetGroup.configureHealthCheck({ path: '/api/v2', interval: Duration.seconds(150), + unhealthyThresholdCount: 10, }); // Allow our service to create and manage EC2 Workers.