Skip to content

Commit

Permalink
Docker container and lambda function for performing firmware builds
Browse files Browse the repository at this point in the history
Provides an entry point that builds and returns a combined LH + RH keyboard
firmware when provided a keymap via a POST body.

Wraps compilation with ccache, and includes a pre-warmed cache of the build in /tmp/ccache.
To maximize chance of a direct cache hit, changes the lambda driver to always build in /tmp/build.

some back of the envelope measurements (2012 xeon e3-1230v2, nixos)
clean build, no cache -> 21.308
clean build, cache -> 7.145
modified keymap, clean build, cache -> 12.127
  • Loading branch information
chrisandreae committed Jun 21, 2024
1 parent df2452a commit 99b2775
Show file tree
Hide file tree
Showing 13 changed files with 668 additions and 1 deletion.
110 changes: 110 additions & 0 deletions .github/workflows/build-container.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
name: Build Compiler Service Container

on:
push:
tags:
- "*"
pull_request_target:
branches:
- main

jobs:
build:
# This job must never be run on a PR from outside the same repository
if: github.repository == 'moergo-sc/zmk' && (github.event.pull_request == null || github.event.pull_request.head.repo.full_name == github.repository)
runs-on: ubuntu-latest
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
id-token: write
contents: read
env:
ECR_REPOSITORY: zmk-builder-lambda
VERSIONS_BUCKET: glove80firmwarepipelines-compilerversionsbucket44-zubaquiyjdam
UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV
REVISION_TAG: ${{ github.event.pull_request && github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ github.event.number }}
steps:
- uses: actions/checkout@v4
with:
repository: moergo-sc/zmk
ref: ${{ github.event.pull_request && github.event.pull_request.head.sha || github.sha }}
fetch-depth: 0
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder
aws-region: us-east-1
- name: Extract container name from branch name
shell: bash
run: |
if [ "$GITHUB_HEAD_REF" ]; then
branch_ref="$GITHUB_HEAD_REF"
type="pr"
tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}"
elif [[ "$GITHUB_REF" == refs/tags/* ]]; then
branch_ref="$GITHUB_REF"
type="tag"
tag="${GITHUB_REF#refs/tags/}"
else
echo "Not a pull request or release tag" >&2
exit 1
fi
# Replace / with . in container tag names
tag="${tag//\//.}"
echo "VERSION_BRANCH=${branch_ref}" >> $GITHUB_ENV
echo "VERSION_TYPE=${type}" >> $GITHUB_ENV
echo "VERSION_NAME=${tag}" >> $GITHUB_ENV
id: extract_name
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
- uses: cachix/install-nix-action@v27
with:
nix_path: nixpkgs=channel:nixos-22.05
- uses: cachix/cachix-action@v15
with:
name: moergo-glove80-zmk-dev
authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
- name: Build lambda image
run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A lambdaImage -o lambdaImage
- name: Import OCI image into docker-daemon
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
run: nix shell -f nix/pinned-nixpkgs.nix skopeo -c skopeo --insecure-policy copy oci:lambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$REVISION_TAG
- name: Push container image to Amazon ECR
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
run: docker push $REGISTRY/$ECR_REPOSITORY:$REVISION_TAG
- name: Create JSON metadata to represent the built container
env:
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
shell: bash
run: |
digest="$(docker inspect --format='{{index .RepoDigests 0}}' $REGISTRY/$ECR_REPOSITORY:$REVISION_TAG)"
digest="${digest##*@}"
api_version="$(cat lambda/api_version.txt)"
timestamp="$(date -u +"%Y%m%d.%H%M%S")"
if [ "$VERSION_TYPE" = "pr" ]; then
release_name="$VERSION_NAME.$timestamp"
else
release_name="$VERSION_NAME"
fi
jq -n '$ARGS.named' \
--arg name "$release_name" \
--arg version_name "$VERSION_NAME" \
--arg revision "$REVISION_TAG" \
--arg release_time "$timestamp" \
--arg branch "$VERSION_BRANCH" \
--arg digest "$digest" \
--arg api_version "$api_version" \
> "/tmp/$VERSION_NAME.json"
- name: Upload image metadata file to versions bucket
run: aws s3 cp "/tmp/$VERSION_NAME.json" "s3://$VERSIONS_BUCKET/images/$VERSION_NAME.json"
- name: Notify the build pipeline that the compile containers have updated
run: >-
aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION
--invocation-type Event
--cli-binary-format raw-in-base64-out
/dev/null
43 changes: 43 additions & 0 deletions .github/workflows/cleanup-container.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: Clean up PR Compiler Service Container

on:
pull_request:
types: [closed]
branches:
- main

jobs:
build:
if: github.repository == 'moergo-sc/zmk'
runs-on: ubuntu-latest
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
id-token: write
contents: read
env:
ECR_REPOSITORY: zmk-builder-lambda
VERSIONS_BUCKET: glove80firmwarepipelines-compilerversionsbucket44-zubaquiyjdam
UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV
PR_NUMBER: ${{ github.event.number }}
steps:
- name: Extract image tag name
shell: bash
run: |
tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}"
# Replace / with . in container tag names
tag="${tag//\//.}"
echo "VERSION_NAME=${tag}" >> $GITHUB_ENV
id: extract_name
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder
aws-region: us-east-1
- name: Delete the image metadata file from the versions s3 bucket
run: aws s3 rm s3://$VERSIONS_BUCKET/images/$VERSION_NAME.json
- name: Notify the build pipeline that the compile containers have updated
run: >-
aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION
--invocation-type Event
--cli-binary-format raw-in-base64-out
/dev/null
2 changes: 1 addition & 1 deletion .github/workflows/nix-build.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Build
name: Build Glove80 Firmware

on:
push:
Expand Down
3 changes: 3 additions & 0 deletions lambda/Gemfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
source 'https://rubygems.org'
gem 'aws_lambda_ric'

13 changes: 13 additions & 0 deletions lambda/Gemfile.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
GEM
remote: https://rubygems.org/
specs:
aws_lambda_ric (2.0.0)

PLATFORMS
ruby

DEPENDENCIES
aws_lambda_ric

BUNDLED WITH
2.1.4
1 change: 1 addition & 0 deletions lambda/api_version.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2
68 changes: 68 additions & 0 deletions lambda/app.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# frozen_string_literal: true

require 'stringio'
require 'digest'
require 'json'
require './compiler'

module LambdaFunction
# Handle a non-HTTP compile request, returning a JSON body of either the
# compiled result or an error.
class Handler
REVISION = ENV.fetch('REVISION', 'unknown')

def self.process(event:, context:)
return { type: 'keep_alive' } if event.has_key?('keep_alive')

parse_base64_param = ->(param, required: true) do
if event.include?(param)
Base64.strict_decode64(event.fetch(param))
elsif required
return error(status: 400, message: "Missing required argument: #{param}")
end
rescue ArgumentError
return error(status: 400, message: "Invalid Base64 in #{param} input")
end

keymap_data = parse_base64_param.('keymap')
kconfig_data = parse_base64_param.('kconfig', required: false)

# Including kconfig settings that affect the RHS require building both
# firmware images, doubling compile time. Clients should omit rhs_kconfig
# where possible.
rhs_kconfig_data = parse_base64_param.('rhs_kconfig', required: false)

result, log =
begin
log_compile(keymap_data, kconfig_data, rhs_kconfig_data)

Compiler.new.compile(keymap_data, kconfig_data, rhs_kconfig_data)
rescue Compiler::CompileError => e
return error(status: e.status, message: e.message, detail: e.log)
end

result = Base64.strict_encode64(result)

{ type: 'result', result: result, log: log, revision: REVISION }
rescue StandardError => e
error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message], exception: e)
end

def self.log_compile(keymap_data, kconfig_data, rhs_kconfig_data)
keymap = Digest::SHA1.base64digest(keymap_data)
kconfig = kconfig_data ? Digest::SHA1.base64digest(kconfig_data) : 'nil'
rhs_kconfig = rhs_kconfig_data ? Digest::SHA1.base64digest(rhs_kconfig_data) : 'nil'
puts("Compiling with keymap: #{keymap}; kconfig: #{kconfig}; rhs_kconfig: #{rhs_kconfig}")
end

def self.error(status:, message:, detail: nil, exception: nil)
reported_error = { type: 'error', status:, message:, detail:, revision: REVISION }

exception_detail = { class: exception.class, backtrace: exception.backtrace } if exception
logged_error = reported_error.merge(exception: exception_detail)
puts(JSON.dump(logged_error))

reported_error
end
end
end
131 changes: 131 additions & 0 deletions lambda/compiler.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# frozen_string_literal: true

require 'tmpdir'
require 'base64'
require 'json'
require 'open3'
require 'yaml'

class Compiler
class CompileError < RuntimeError
attr_reader :status, :log

def initialize(message, status: 400, log:)
super(message)
@status = status
@log = log
end
end

def compile(keymap_data, lhs_kconfig_data, rhs_kconfig_data)
if rhs_kconfig_data && !rhs_kconfig_data.empty?
lhs_result, lhs_output = compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: false)
rhs_result, rhs_output = compile_board('glove80_rh', keymap_data: nil, kconfig_data: rhs_kconfig_data, include_static_rhs: false)
[
lhs_result.concat(rhs_result),
["LHS Output:", *lhs_output, "RHS Output:", *rhs_output],
]
else
compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: true)
end
end

def compile_board(board, keymap_data:, kconfig_data:, include_static_rhs: false)
in_build_dir do
compile_command = ['compileZmk', '-b', board]

if keymap_data
dts_parse_errors = validate_devicetree!(keymap_data)
File.open('build.keymap', 'w') { |io| io.write(keymap_data) }
compile_command << '-k' << './build.keymap'
end

if kconfig_data
File.open('build.conf', 'w') { |io| io.write(kconfig_data) }
compile_command << '-c' << './build.conf'
end

if include_static_rhs
# Concatenate the pre-compiled glove80_rh image to the resulting uf2
compile_command << '-m'
end

compile_output = nil

IO.popen(compile_command, 'rb', err: [:child, :out]) do |io|
compile_output = io.read
end

compile_output = compile_output.split("\n")

unless $?.success?
status = $?.exitstatus
raise CompileError.new("Compile failed with exit status #{status}", log: compile_output)
end

unless File.exist?('zmk.uf2')
raise CompileError.new('Compile failed to produce result binary', status: 500, log: compile_output)
end

if dts_parse_errors
# DTS validation failed to parse the DTS, yet the Zephyr build
# nonetheless succeeded. We can't allow returning the result, since we
# were unable to check it for unsafe dts sections.
raise CompileError.new('Syntax error validating device-tree input', log: dts_parse_errors)
end

result = File.read('zmk.uf2')

[result, compile_output]
end
end

PERMITTED_DTS_SECTIONS = %w[
behaviors macros combos conditional_layers keymap underglow-indicators
].freeze

def validate_devicetree!(dtsi)
dts = "/dts-v1/;\n" + dtsi

stdout, stderr, status =
Open3.capture3({}, 'dts2yml', unsetenv_others: true, stdin_data: dts)

unless status.success?
# The error output from dtc is much harder to understand than Zephyr's
# errors, and the line numbers don't match up due to preprocessing. Rather
# than raising these now, return the error output in order that it's only
# used in the case that the Zephyr build doesn't itself error.
return stderr.split("\n")
end

data =
begin
YAML.safe_load(stdout)
rescue Psych::Exception => e
raise CompileError.new('Error parsing translated device-tree', status: 500, log: [e.message])
end

sections = data.flat_map(&:keys)
invalid_sections = sections - PERMITTED_DTS_SECTIONS

unless invalid_sections.empty?
raise CompileError.new(
"Device-tree included the non-permitted root sections: #{invalid_sections.inspect}", log: [])
end

nil
end

# Lambda is single-process per container, and we get substantial speedups
# from ccache by always building in the same path
BUILD_DIR = '/tmp/build'

def in_build_dir
FileUtils.remove_entry(BUILD_DIR, true)
Dir.mkdir(BUILD_DIR)
Dir.chdir(BUILD_DIR)
yield
ensure
FileUtils.remove_entry(BUILD_DIR, true) rescue nil
end
end
Loading

0 comments on commit 99b2775

Please sign in to comment.