Skip to content

Commit

Permalink
update buildbot, make filesystem bigger
Browse files Browse the repository at this point in the history
  • Loading branch information
@thomas-roos
thomas-roos committed Jul 10, 2024
1 parent 0136c53 commit 92bf201
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 87 deletions.
2 changes: 1 addition & 1 deletion configuration/admin/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ EXPOSE 9989
RUN python3 -m pip install --upgrade pip
RUN python3 -m pip install boto3 botocore
RUN python3 -m pip install git-remote-codecommit
RUN python3 -m pip install 'buildbot[bundle]==v3.11.2'
RUN python3 -m pip install buildbot[bundle]

ENV PATH="/home/user/.local/bin:${PATH}"

Expand Down
28 changes: 17 additions & 11 deletions configuration/admin/admin.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ c['builders'].append(util.BuilderConfig(name="reload_config_factory",workernames
c['change_source'].append(changes.GitPoller(
os.environ.get('CODECOMMIT_REPOSITORY_CLONE_URL_GRC', 'default'),
workdir='gitpoller-workdir',
branches='main',
branches=['main'],
pollAtLaunch=False,
pollInterval=1*60))

Expand Down Expand Up @@ -199,7 +199,7 @@ c['workers'].append(worker.EC2LatentWorker(workername, 'sekrit', 'c5.metal',
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"VolumeSize": 1000,
"Throughput": 1000,
"DeleteOnTermination": True,
}
Expand Down Expand Up @@ -252,7 +252,7 @@ for instance in x86_64_instance_types:
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"VolumeSize": 1000,
"Throughput": 1000,
"DeleteOnTermination": True,
}
Expand All @@ -276,7 +276,7 @@ for instance in arm_64_instance_types:
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"VolumeSize": 1000,
"Throughput": 1000,
"DeleteOnTermination": True,
}
Expand Down Expand Up @@ -304,21 +304,21 @@ c['change_source'].append(changes.GitPoller(
"https://github.com/aws4embeddedlinux/meta-aws.git",
workdir=gitpoller_workdir,
pollAtLaunch=False,
pollinterval=10*60,
pollInterval=10*60,
branches=meta_aws_supported_releases_next))

c['change_source'].append(changes.GitPoller(
"https://github.com/yoctoproject/poky.git",
workdir=gitpoller_workdir,
pollAtLaunch=False,
pollinterval=10*60,
pollInterval=10*60,
branches=meta_aws_supported_releases))

c['change_source'].append(changes.GitPoller(
"https://github.com/openembedded/meta-openembedded.git",
workdir=gitpoller_workdir,
pollAtLaunch=False,
pollinterval=10*60,
pollInterval=10*60,
branches=meta_aws_supported_releases))

for release in meta_aws_supported_releases:
Expand All @@ -342,7 +342,7 @@ for release in meta_aws_supported_releases:
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"VolumeSize": 1000,
"Throughput": 1000,
"DeleteOnTermination": True,
}
Expand Down Expand Up @@ -399,7 +399,7 @@ for release in meta_aws_supported_releases:
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"VolumeSize": 1000,
"Throughput": 1000,
"DeleteOnTermination": True,
}
Expand Down Expand Up @@ -453,7 +453,7 @@ for branch in meta_aws_demos_supported_branches:
set -euxo pipefail;
echo 'ACCEPT_FSL_EULA = \"1\"' >> conf/local.conf ;
echo 'BB_DANGLINGAPPENDS_WARNONLY = \"1\"' >> conf/local.conf ;
for i in \"agl-renesas aws-greengrass-test-image\" \"imx8m aws-greengrass-test-image\" \"phytec aws-greengrass-test-image\" \"qemu-arm64 aws-greengrass-test-image\" \"qemu-x86-64 aws-greengrass-test-image\" \"rpi4-64 aws-greengrass-test-image\" \"rpi4-32 aws-greengrass-test-image\" \"ec2-arm64 aws-greengrass-test-image\" \"ec2-x86-64 aws-greengrass-test-image\" \"ti-am572x-idk aws-greengrass-test-image\";
for i in \"agl-renesas aws-greengrass-test-image\" \"imx8m aws-greengrass-test-image\" \"phytec aws-greengrass-test-image\" \"qemu-arm64 aws-greengrass-test-image\" \"qemu-x86-64 aws-greengrass-test-image\" \"rpi5 aws-greengrass-test-image\" \"rpi4-32 aws-greengrass-test-image\" \"ec2-arm64 aws-greengrass-test-image\" \"ec2-x86-64 aws-greengrass-test-image\" \"ti-am572x-idk aws-greengrass-test-image\" \"ti-am572x-idk aws-greengrass-test-image\";
do set -- $i # Convert the "tuple" into the param args $1 $2... ;
echo DEVICE: $1 and IMAGE: $2;
BB_ENV_PASSTHROUGH_ADDITIONS="SSTATE_DIR $BB_ENV_PASSTHROUGH_ADDITIONS" SSTATE_DIR="/sstate" DEVICE=$1 IMAGE=$2 bitbake $2 -k;
Expand Down Expand Up @@ -506,7 +506,7 @@ for branch in meta_aws_demos_supported_branches:
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"VolumeSize": 1000,
"Throughput": 1000,
"DeleteOnTermination": True,
}
Expand Down Expand Up @@ -536,4 +536,10 @@ for branch in meta_aws_demos_supported_branches:
builderNames=[buildername], dayOfWeek=5, hour=3, minute=1))


#### define log preview number of lines
c['www']['ui_default_config'] = {
'LogPreview.loadlines': 100,
'LogPreview.maxlines': 300,
}

#### user.cfg gets pasted at the end of this file ####
94 changes: 47 additions & 47 deletions configuration/user-repo/user.cfg
View file
Original file line number Diff line number Diff line change
@@ -1,50 +1,50 @@
### user.cfg ###


#### core-image-minimal example ######################
c['change_source'].append(changes.GitPoller(
"https://github.com/yoctoproject/poky.git",
workdir='gitpoller-workdir', branch='master-next',
pollinterval=60))

c['schedulers'].append(schedulers.SingleBranchScheduler(
name="check_poky",
change_filter=util.ChangeFilter(branch="master-next",repository="https://github.com/yoctoproject/poky.git"),
treeStableTimer=60*2,
builderNames=["core_image_minimal_builder_c7g_16xlarge"]))

core_image_minimal_factory = util.BuildFactory()

core_image_minimal_factory.addStep(steps.Git(repourl='https://github.com/yoctoproject/poky.git'))
core_image_minimal_factory.addStep(steps.ShellCommand(command=['/bin/bash', '-c', '''source oe-init-build-env ; echo 'SSTATE_DIR = \"/sstate\"' > conf/site.conf ; bitbake core-image-minimal''']))

c['workers'].append(worker.EC2LatentWorker("core_image_minimal_worker_c7g_16xlarge", 'sekrit', 'c7g.16xlarge',
ami=arm64_ami,
build_wait_timeout=0,
keypair_name='worker-key',
security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),],
subnet_id=os.environ.get('BUILDBOT_WORKER_SUBNET', 'default'),
user_data=init_script.format(sstate_efs_dns_name, 'core_image_minimal_worker_c7g_16xlarge'),
block_device_map= [
{
"DeviceName": "/dev/xvdf",
"Ebs" : {
"VolumeType": "gp3",
"Iops": 16000,
"VolumeSize": 500,
"Throughput": 1000,
}
}
],
))

c['builders'].append(
util.BuilderConfig(name="core_image_minimal_builder_c7g_16xlarge",
workernames=["core_image_minimal_worker_c7g_16xlarge"],
factory=core_image_minimal_factory))

c['schedulers'].append(
schedulers.ForceScheduler(
name="core_image_minimal_c7g_16xlarge",
builderNames=["core_image_minimal_builder_c7g_16xlarge"],
buttonName="force"))
### core-image-minimal example ######################
#c['change_source'].append(changes.GitPoller(
# "https://github.com/yoctoproject/poky.git",
# workdir='gitpoller-workdir', branch='master-next',
# pollinterval=60))
#
#c['schedulers'].append(schedulers.SingleBranchScheduler(
# name="check_poky",
# change_filter=util.ChangeFilter(branch="master-next",repository="https://github.com/yoctoproject/poky.git"),
# treeStableTimer=60*2,
# builderNames=["core_image_minimal_builder_c7g_16xlarge"]))
#
#core_image_minimal_factory = util.BuildFactory()
#
#core_image_minimal_factory.addStep(steps.Git(repourl='https://github.com/yoctoproject/poky.git'))
#core_image_minimal_factory.addStep(steps.ShellCommand(command=['/bin/bash', '-c', '''source oe-init-build-env ; echo 'SSTATE_DIR = \"/sstate\"' > conf/site.conf ; bitbake core-image-minimal''']))
#
#c['workers'].append(worker.EC2LatentWorker("core_image_minimal_worker_c7g_16xlarge", 'sekrit', 'c7g.16xlarge',
# ami=arm64_ami,
# build_wait_timeout=0,
# keypair_name='worker-key',
# security_group_ids=[os.environ.get('BUILDBOT_WORKER_SECURITY_GROUP', 'default'),],
# subnet_id=os.environ.get('BUILDBOT_WORKER_SUBNET', 'default'),
# user_data=init_script.format(sstate_efs_dns_name, 'core_image_minimal_worker_c7g_16xlarge'),
# block_device_map= [
# {
# "DeviceName": "/dev/xvdf",
# "Ebs" : {
# "VolumeType": "gp3",
# "Iops": 16000,
# "VolumeSize": 500,
# "Throughput": 1000,
# }
# }
# ],
# ))
#
#c['builders'].append(
# util.BuilderConfig(name="core_image_minimal_builder_c7g_16xlarge",
# workernames=["core_image_minimal_worker_c7g_16xlarge"],
# factory=core_image_minimal_factory))
#
#c['schedulers'].append(
# schedulers.ForceScheduler(
# name="core_image_minimal_c7g_16xlarge",
# builderNames=["core_image_minimal_builder_c7g_16xlarge"],
# buttonName="force"))
52 changes: 26 additions & 26 deletions lib/app.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { App } from 'aws-cdk-lib';

import { CacheRepoStack, CachePipeline, CacheEcrStack, CacheEcsStack } from './cache-server';
// import { CacheRepoStack, CachePipeline, CacheEcrStack, CacheEcsStack } from './cache-server';
import { BuildBotConfig, BuildBotImageRepo, BuildBotPipeline, BuildBotServer, BuildBotFilesystem } from './buildbot';
import { DeveloperStages, FederateOIDC } from './constants';
import { Vpc, Cluster, ServiceDomain } from './stacks';
Expand Down Expand Up @@ -38,24 +38,24 @@ if (dev) {
terminationProtection,
});

const ssr = new CacheRepoStack(app, 'CacheRepo-Personal', {
env,
terminationProtection,
});

const ecr = new CacheEcrStack(app, 'CacheEcr-Personal', {
env,
terminationProtection,
});

const ecs = new CacheEcsStack(app, 'CacheEcs-Personal', {
env,
terminationProtection,
repository: ecr.repo,
cluster: cluster.cluster,
vpc: vpc.vpc,
namespace: serviceDomain.namespace,
});
// const ssr = new CacheRepoStack(app, 'CacheRepo-Personal', {
// env,
// terminationProtection,
// });

// const ecr = new CacheEcrStack(app, 'CacheEcr-Personal', {
// env,
// terminationProtection,
// });

// const ecs = new CacheEcsStack(app, 'CacheEcs-Personal', {
// env,
// terminationProtection,
// repository: ecr.repo,
// cluster: cluster.cluster,
// vpc: vpc.vpc,
// namespace: serviceDomain.namespace,
// });

const buildBotConfig = new BuildBotConfig(app, 'BuildBotConfig-Personal', {
env,
Expand Down Expand Up @@ -102,13 +102,13 @@ if (dev) {
workerSstateEfsFsID: buildBotServer.sstateFS.fileSystemId,
});

new CachePipeline(app, 'CachePipeline-Personal', {
env,
terminationProtection,
repository: ecr.repo,
cacheRepo: ssr.repo.repositoryName,
ecsCacheFargateService: ecs.service,
});
// new CachePipeline(app, 'CachePipeline-Personal', {
// env,
// terminationProtection,
// repository: ecr.repo,
// cacheRepo: ssr.repo.repositoryName,
// ecsCacheFargateService: ecs.service,
// });
}

// End
4 changes: 2 additions & 2 deletions lib/buildbot/buildbot-server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ export class BuildBotServer extends cdk.Stack {
// Set up the ECS Server.
const buildBotServerTask = new ecs.TaskDefinition(this, 'BuildBotServerTask', {
compatibility: ecs.Compatibility.FARGATE,
cpu: '4096',
memoryMiB: '8192',
cpu: '8192',
memoryMiB: '16384',
volumes: [
{
name: 'DBMount',
Expand Down

0 comments on commit 92bf201

Please sign in to comment.