Skip to content

Commit

Permalink
wip: windows and docker
Browse files Browse the repository at this point in the history 
Signed-off-by: Damien Duportal <[email protected]>
  • Loading branch information
@dduportal
dduportal committed Aug 21, 2022
1 parent 5d90662 commit dea5896
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 104 deletions.
191 changes: 104 additions & 87 deletions Jenkinsfile_k8s
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,34 @@
// The node to be used depends on the build type: this function executes the pipeline code block provided as "body"
// into the correct node type based on the provided arguments
def withPackerNode(String packer_template, String compute_type, String cpu_architecture, Closure body) {
// Build ARM64 CPU Docker images on a native machine (faster than using the local qemu)
if (cpu_architecture == 'arm64' && compute_type == 'docker') {
node('linux-arm64-docker') {
// New agent workspace specified as scripted requires an explicit checkout (compared to declarative)
checkout scm
echo "DEBUG: (withPackerNode) packer_template=${packer_template} compute_type=${compute_type}"
if (compute_type == 'docker') {
// Build ARM64 CPU Docker images on a native machine (faster than using the local qemu)
if (cpu_architecture == 'arm64') {
node('linux-arm64-docker') {
// New agent workspace specified as scripted requires an explicit checkout (compared to declarative)
checkout scm

// New agent means new packer project to initialize (plugins)
packerInitPlugins()
// New agent means new packer project to initialize (plugins)
packerInitPlugins()

return body.call()
return body.call()
}
} else if (packer_template == 'windows-2019') {
node('windows-amd64-docker') {
// New agent workspace specified as scripted requires an explicit checkout (compared to declarative)
checkout scm

// New agent means new packer project to initialize (plugins)
packerInitPlugins()

return body.call()
}
}
} else {
// No node allocation: keep the same default agent node (e.g. declarative top-level)
return body.call()
}

// No node allocation: keep the same default agent node (e.g. declarative top-level)
return body.call()
}

// Initialize the packer project by installing the plugins in $PACKER_HOME_DIR/ - ref. https://www.packer.io/docs/configure
Expand All @@ -24,9 +37,23 @@ def packerInitPlugins() {
// Authenticating to the GitHub API with an API token (auto-generated IAT, valid for 1 hour) provided to the environment variable PACKER_GITHUB_API_TOKEN
// to avoid hitting the rate limit. Ref. https://www.packer.io/docs/commands/init.
withCredentials([usernamePassword(credentialsId: 'github-app-infra',usernameVariable: 'UNUSED',passwordVariable: 'PACKER_GITHUB_API_TOKEN')]) {
// Cleanup any remnant of packer plugins on this agent
sh 'rm -rf /home/jenkins/.config /home/jenkins/.packer*'
sh 'packer init ./'
if (isUnix()) {
// Cleanup any remnant of packer plugins on this agent
sh 'rm -rf /home/jenkins/.config /home/jenkins/.packer*'
}
universalShell 'packer init ./'
}
}

def universalShell(String command) {
if (isUnix()) {
// To allow using ASDF shims
withEnv(["PATH+ASDF=/home/jenkins/.asdf/shims:/home/jenkins/.asdf/bin"]) {
sh command
}
} else {
windowsifiedCommand = command.replaceAll(/packer /, 'packer.exe ')
bat windowsifiedCommand
}
}

Expand Down Expand Up @@ -58,15 +85,11 @@ pipeline {
// Average build time is ~30 min. 1h30 hour timeout indicates that something is wrong and should fail
timeout(time: 90, unit: 'MINUTES')
}
environment {
// To allow using ASDF shims
PATH = "${env.PATH}:/home/jenkins/.asdf/shims:/home/jenkins/.asdf/bin"
}
stages {
stage('Side Tasks') {
environment {
DRYRUN = "${env.BRANCH_IS_PRIMARY ? 'false' : 'true'}"
}
// environment {
// DRYRUN = "${env.BRANCH_IS_PRIMARY ? 'false' : 'true'}"
// }
parallel {
stage('Packer Init') {
steps {
Expand All @@ -76,53 +99,53 @@ pipeline {
}
}
}
stage('GC on AWS us-east-2') {
agent {
kubernetes {
yaml podAgentDefinition
}
}
environment {
AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id')
AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key')
AWS_DEFAULT_REGION = 'us-east-2'
}
steps {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh './cleanup/aws.sh'
sh './cleanup/aws_images.sh'
}
}
}
stage('GC on Azure') {
agent {
kubernetes {
yaml podAgentDefinition
}
}
environment {
PACKER_AZURE = credentials('packer-azure-serviceprincipal')
}
steps {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
sh 'az login --service-principal -u "$PACKER_AZURE_CLIENT_ID" -p "$PACKER_AZURE_CLIENT_SECRET" -t "$PACKER_AZURE_TENANT_ID"'
sh 'az account set -s "$PACKER_AZURE_SUBSCRIPTION_ID"'
sh './cleanup/azure.sh'
}
}
}
stage('Updatecli') {
steps {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
script {
updatecli(action: 'diff', containerMemory: '1024Mi')
if (env.BRANCH_IS_PRIMARY) {
updatecli(action: 'apply', cronTriggerExpression: '@daily', containerMemory: '1024Mi')
}
}
}
}
}
// stage('GC on AWS us-east-2') {
// agent {
// kubernetes {
// yaml podAgentDefinition
// }
// }
// environment {
// AWS_ACCESS_KEY_ID = credentials('packer-aws-access-key-id')
// AWS_SECRET_ACCESS_KEY = credentials('packer-aws-secret-access-key')
// AWS_DEFAULT_REGION = 'us-east-2'
// }
// steps {
// catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// sh './cleanup/aws.sh'
// sh './cleanup/aws_images.sh'
// }
// }
// }
// stage('GC on Azure') {
// agent {
// kubernetes {
// yaml podAgentDefinition
// }
// }
// environment {
// PACKER_AZURE = credentials('packer-azure-serviceprincipal')
// }
// steps {
// catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// sh 'az login --service-principal -u "$PACKER_AZURE_CLIENT_ID" -p "$PACKER_AZURE_CLIENT_SECRET" -t "$PACKER_AZURE_TENANT_ID"'
// sh 'az account set -s "$PACKER_AZURE_SUBSCRIPTION_ID"'
// sh './cleanup/azure.sh'
// }
// }
// }
// stage('Updatecli') {
// steps {
// catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// script {
// updatecli(action: 'diff', containerMemory: '1024Mi')
// if (env.BRANCH_IS_PRIMARY) {
// updatecli(action: 'apply', cronTriggerExpression: '@daily', containerMemory: '1024Mi')
// }
// }
// }
// }
// }
}
}
stage('Packer Images') {
Expand All @@ -134,7 +157,8 @@ pipeline {
}
axis {
name 'agent_type'
values 'ubuntu-20.04', 'windows-2019', 'windows-2022'
//values 'ubuntu-20.04', 'windows-2019', 'windows-2022'
values 'windows-2019'
}
axis {
name 'compute_type'
Expand All @@ -143,7 +167,7 @@ pipeline {
}
}
excludes {
// Do NOT build ARM images in Azure
// Do NOT build ARM64 images in Azure (Resource Manager)
exclude {
axis {
name 'cpu_architecture'
Expand All @@ -154,32 +178,28 @@ pipeline {
values 'azure-arm'
}
}
// Only build Ubuntu images for arm64 CPU in AWS
// Do NOT build ARM64 images for Windows operating systems
exclude {
axis {
name 'cpu_architecture'
values 'arm64'
}
axis {
name 'agent_type'
notValues 'ubuntu-20.04'
}
axis {
name 'compute_type'
values 'amazon-ebs'
values 'windows-2022'
}
}
// No build on Windows or Docker, not yet implemented
exclude {
axis {
name 'agent_type'
values 'windows-2019'
name 'cpu_architecture'
values 'arm64'
}
axis {
name 'compute_type'
values 'docker'
name 'agent_type'
values 'windows-2019'
}
}
// Do not build Windows 2022 with Docker computing
exclude {
axis {
name 'agent_type'
Expand All @@ -197,8 +217,6 @@ pipeline {
// Defines the following environment variables: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID
// Ref. https://plugins.jenkins.io/azure-credentials/#plugin-content-declarative-pipeline
AZURE = credentials('packer-azure-serviceprincipal')
// // Split packer plugins/configuration for each matrix cell - ref. https://www.packer.io/docs/configure
// PACKER_PLUGIN_PATH = "${env.WORKSPACE}/plugins"
// Define Packer Input variables through environment variables prefixed with 'PKR_VAR_'
// Ref. https://www.packer.io/docs/templates/hcl_templates/variables#assigning-values-to-build-variables
PKR_VAR_build_type = "${env.TAG_NAME ? 'prod' : (env.BRANCH_IS_PRIMARY ? 'staging' : 'dev') }"
Expand All @@ -208,7 +226,6 @@ pipeline {
PKR_VAR_agent_os_version = "${env.agent_type.split('-')[1]}"
PKR_VAR_architecture = "${env.cpu_architecture}"
PKR_VAR_image_type = "${env.compute_type}"
PATH = "${WORKSPACE}/.bin:${env.PATH}" // Required if packer needs to be installed
}
stages {
stage('Build Template') {
Expand All @@ -225,11 +242,11 @@ pipeline {

withPackerNode(pkr_var_agent_os_type + '-' + pkr_var_agent_os_version , pkr_var_image_type, pkr_var_architecture) {
// Validate template (for all elements)
sh 'PACKER_LOG=1 packer validate ./'
universalShell 'packer validate .'

// Execute build only for this matrix cell's setup
retry(count: 2, conditions: [kubernetesAgent(handleNonKubernetes: true), nonresumable()]) {
sh 'packer build -timestamp-ui -force -only="${PKR_VAR_image_type}.${PKR_VAR_agent_os_type}" ./'
universalShell "packer build -timestamp-ui -force -only=${env.PKR_VAR_image_type}.${env.PKR_VAR_agent_os_type} ."
}
}
}
Expand All @@ -244,7 +261,7 @@ pipeline {
script {
echo "Pushing jenkinsciinfra/jenkins-agent-${agent_type}:${TAG_NAME} & jenkinsciinfra/jenkins-agent-${agent_type}:latest"
infra.withDockerPushCredentials {
sh 'docker push --all-tags jenkinsciinfra/jenkins-agent-${agent_type}'
universalShell "docker push --all-tags jenkinsciinfra/jenkins-agent-${env.agent_type}"
}
}
}
Expand Down
17 changes: 17 additions & 0 deletions build-jenkins-agent-ubuntu.pkr.hcl
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,23 @@
build {
source "docker.base" {
name = "ubuntu"

# Use official ubuntu image, with the tag set to the numeric version
image = "${var.agent_os_type}:${var.agent_os_version}"

# To improve audit and garbage collecting, we provide "labels" to the image
changes = [
"LABEL timestamp = ${local.now_unix_timestamp}",
"LABEL version = ${var.image_version}",
"LABEL scm_ref = ${var.scm_ref}",
"LABEL build_type = ${var.build_type}",
"ENV LANG=en_US.UTF-8",
"ENV LANGUAGE=en_US:en",
"ENV LC_ALL=en_US.UTF-8",
"ENV AGENT_WORKDIR=/home/jenkins/agent",
"WORKDIR /home/jenkins",
"ENTRYPOINT [\"/usr/local/bin/jenkins-agent\"]",
]
}

source "amazon-ebs.base" {
Expand Down
29 changes: 29 additions & 0 deletions build-jenkins-agent-windows.pkr.hcl
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,27 @@
build {
source "docker.base" {
name = "windows"

windows_container = true

# https://hub.docker.com/_/microsoft-windows-servercore
image = "mcr.microsoft.com/windows/servercore:ltsc2019"

# To improve audit and garbage collecting, we provide "labels" to the image
changes = [
"LABEL timestamp = ${local.now_unix_timestamp}",
"LABEL version = ${var.image_version}",
"LABEL scm_ref = ${var.scm_ref}",
"LABEL build_type = ${var.build_type}",
# https://github.com/jenkinsci/docker-agent/blob/1dd17e715fbebc7986154d5f54a0553d970dbf8d/11/windows/windowsservercore-ltsc2019/Dockerfile#L64
"ENV AGENT_WORKDIR=C:/Users/jenkins/agent",
# https://github.com/jenkinsci/docker-agent/blob/1dd17e715fbebc7986154d5f54a0553d970dbf8d/11/windows/windowsservercore-ltsc2019/Dockerfile#L84
"WORKDIR C:/Users/jenkins",
# https://github.com/jenkinsci/docker-inbound-agent/blob/708e03d72337cb6bc7debc1931ccb5019f82ecf6/11/windows/windowsservercore-ltsc2019/Dockerfile#L41
"ENTRYPOINT [\"powershell.exe\", \"-f\", \"C:/ProgramData/Jenkins/jenkins-agent.ps1\"]",
]
}

source "amazon-ebs.base" {
name = "windows"
communicator = "winrm"
Expand Down Expand Up @@ -75,4 +98,10 @@ build {
"if($env:AGENT_OS_VERSION = '2019') { C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SendWindowsIsReady.ps1 -Schedule; C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule; C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown;};"
]
}

post-processor "docker-tag" {
only = ["docker.windows"]
repository = "${var.docker_namespace}/${local.image_name}"
tags = [var.image_version, "latest"]
}
}
18 changes: 1 addition & 17 deletions sources.pkr.hcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,24 +68,8 @@ source "azure-arm" "base" {
}
}

# This source defines all the common settings for any Azure image (whatever Operating System)
# This source defines all the common settings for any Docker image (whatever Operating System)
source "docker" "base" {
image = "${var.agent_os_type}:${var.agent_os_version}"

# Persist image on local docker engine
commit = true

# To improve audit and garbage collecting, we provide "labels" to the image
changes = [
"LABEL timestamp = ${local.now_unix_timestamp}",
"LABEL version = ${var.image_version}",
"LABEL scm_ref = ${var.scm_ref}",
"LABEL build_type = ${var.build_type}",
"ENV LANG=en_US.UTF-8",
"ENV LANGUAGE=en_US:en",
"ENV LC_ALL=en_US.UTF-8",
"ENV AGENT_WORKDIR=/home/jenkins/agent",
"WORKDIR /home/jenkins",
"ENTRYPOINT [\"/usr/local/bin/jenkins-agent\"]",
]
}

0 comments on commit dea5896

Please sign in to comment.