Docker container and lambda function for performing firmware builds

Provides an entry point that builds and returns a combined LH + RH keyboard
firmware when provided a keymap via a POST body.

Wraps compilation with ccache, and includes a pre-warmed cache of the build in /tmp/ccache.
To maximize chance of a direct cache hit, changes the lambda driver to always build in /tmp/build.

some back of the envelope measurements (2012 xeon e3-1230v2, nixos)
clean build, no cache -> 21.308
clean build, cache -> 7.145
modified keymap, clean build, cache -> 12.127

.github/workflows/build-container.yml

@@ -0,0 +1,94 @@
+name: Build Compiler Service Container
+
+on:
+  push:
+    tags:
+      - "*"
+  pull_request_target:
+    branches:
+      - main
+
+jobs:
+  build:
+    # This job must never be run on a PR from outside the same repository
+    if: github.repository == 'moergo-sc/zmk' && (github.event.pull_request == null || github.event.pull_request.head.repo.full_name == github.repository)
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      ECR_REPOSITORY: zmk-builder-lambda
+      VERSIONS_BUCKET: glove80firmwarepipelines-compilerversionsbucket44-zubaquiyjdam
+      UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV
+      REVISION_TAG: ${{ github.event.pull_request && github.event.pull_request.head.sha || github.sha  }}
+      PR_NUMBER: ${{ github.event.number }}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          repository: moergo-sc/zmk
+          ref: ${{ github.event.pull_request && github.event.pull_request.head.sha || github.sha }}
+          fetch-depth: 0
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder
+          aws-region: us-east-1
+      - name: Extract container name from branch name
+        shell: bash
+        run: |
+          if [ "$GITHUB_HEAD_REF" ]; then
+            tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}"
+          elif [[ "$GITHUB_REF" == refs/tags/* ]]; then
+            tag="${GITHUB_REF#refs/tags/}"
+          else
+            echo "Not a pull request or release tag" >&2
+            exit 1
+          fi
+          # Replace / with . in container tag names
+          tag="${tag//\//.}"
+          echo "CONTAINER_NAME=${tag}" >> $GITHUB_ENV
+        id: extract_name
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+      - uses: cachix/install-nix-action@v20
+        with:
+          nix_path: nixpkgs=channel:nixos-22.05
+      - uses: cachix/cachix-action@v12
+        with:
+          name: moergo-glove80-zmk-dev
+          authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
+      - name: Build lambda image
+        run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A lambdaImage -o lambdaImage
+      - name: Import OCI image into docker-daemon
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: skopeo --insecure-policy copy oci:lambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$REVISION_TAG
+      - name: Push container image to Amazon ECR
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: docker push $REGISTRY/$ECR_REPOSITORY:$REVISION_TAG
+      - name: Create JSON metadata to represent the built container
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        shell: bash
+        run: |
+          digest="$(docker inspect --format='{{index .RepoDigests 0}}' $REGISTRY/$ECR_REPOSITORY:$REVISION_TAG)"
+          digest="${digest##*@}"
+          api_version="$(cat lambda/api_version.txt)"
+          jq -n '$ARGS.named' \
+            --arg name     "$CONTAINER_NAME" \
+            --arg revision "$REVISION_TAG" \
+            --arg branch   "$GITHUB_REF" \
+            --arg digest   "$digest" \
+            --arg api_version "$api_version" \
+            > "/tmp/$CONTAINER_NAME.json"
+      - name: Upload image metadata file to versions bucket
+        run: aws s3 cp "/tmp/$CONTAINER_NAME.json" "s3://$VERSIONS_BUCKET/images/$CONTAINER_NAME.json"
+      - name: Notify the build pipeline that the compile containers have updated
+        run: >-
+          aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION
+          --invocation-type Event
+          --cli-binary-format raw-in-base64-out
+          /dev/null

.github/workflows/cleanup-container.yml

@@ -0,0 +1,43 @@
+name: Clean up PR Compiler Service Container
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  build:
+    if: github.repository == 'moergo-sc/zmk'
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      ECR_REPOSITORY: zmk-builder-lambda
+      VERSIONS_BUCKET: glove80firmwarepipelines-compilerversionsbucket44-zubaquiyjdam
+      UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV
+      PR_NUMBER: ${{ github.event.number }}
+    steps:
+      - name: Extract image tag name
+        shell: bash
+        run: |
+          tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}"
+          # Replace / with . in container tag names
+          tag="${tag//\//.}"
+          echo "CONTAINER_NAME=${tag}" >> $GITHUB_ENV
+        id: extract_name
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder
+          aws-region: us-east-1
+      - name: Delete the image metadata file from the versions s3 bucket
+        run: aws s3 rm s3://$VERSIONS_BUCKET/images/$CONTAINER_NAME.json
+      - name: Notify the build pipeline that the compile containers have updated
+        run: >-
+          aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION
+          --invocation-type Event
+          --cli-binary-format raw-in-base64-out
+          /dev/null

.github/workflows/nix-build.yml

@@ -1,4 +1,4 @@
-name: Build
+name: Build Glove80 Firmware
 
 on:
   push:

lambda/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gem 'aws_lambda_ric'
+

lambda/Gemfile.lock

@@ -0,0 +1,13 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws_lambda_ric (2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws_lambda_ric
+
+BUNDLED WITH
+   2.1.4

lambda/api_version.txt

@@ -0,0 +1 @@
+2

lambda/app.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'digest'
+require 'json'
+require './compiler'
+
+module LambdaFunction
+  # Handle a non-HTTP compile request, returning a JSON body of either the
+  # compiled result or an error.
+  class Handler
+    REVISION = ENV.fetch('REVISION', 'unknown')
+
+    def self.process(event:, context:)
+      return { type: 'keep_alive' } if event.has_key?('keep_alive')
+
+      parse_base64_param = ->(param, required: true) do
+        if event.include?(param)
+          Base64.strict_decode64(event.fetch(param))
+        elsif required
+          return error(status: 400, message: "Missing required argument: #{param}")
+        end
+      rescue ArgumentError
+        return error(status: 400, message: "Invalid Base64 in #{param} input")
+      end
+
+      keymap_data  = parse_base64_param.('keymap')
+      kconfig_data = parse_base64_param.('kconfig', required: false)
+
+      # Including kconfig settings that affect the RHS require building both
+      # firmware images, doubling compile time. Clients should omit rhs_kconfig
+      # where possible.
+      rhs_kconfig_data = parse_base64_param.('rhs_kconfig', required: false)
+
+      result, log =
+        begin
+          log_compile(keymap_data, kconfig_data, rhs_kconfig_data)
+
+          Compiler.new.compile(keymap_data, kconfig_data, rhs_kconfig_data)
+        rescue Compiler::CompileError => e
+          return error(status: e.status, message: e.message, detail: e.log)
+        end
+
+      result = Base64.strict_encode64(result)
+
+      { type: 'result', result: result, log: log, revision: REVISION }
+    rescue StandardError => e
+      error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message], exception: e)
+    end
+
+    def self.log_compile(keymap_data, kconfig_data, rhs_kconfig_data)
+      keymap      = Digest::SHA1.base64digest(keymap_data)
+      kconfig     = kconfig_data ? Digest::SHA1.base64digest(kconfig_data) : 'nil'
+      rhs_kconfig = rhs_kconfig_data ? Digest::SHA1.base64digest(rhs_kconfig_data) : 'nil'
+      puts("Compiling with keymap: #{keymap}; kconfig: #{kconfig}; rhs_kconfig: #{rhs_kconfig}")
+    end
+
+    def self.error(status:, message:, detail: nil, exception: nil)
+      reported_error = { type: 'error', status:, message:, detail:, revision: REVISION }
+
+      exception_detail = { class: exception.class, backtrace: exception.backtrace } if exception
+      logged_error = reported_error.merge(exception: exception_detail)
+      puts(JSON.dump(logged_error))
+
+      reported_error
+    end
+  end
+end

lambda/compiler.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+require 'base64'
+require 'json'
+require 'open3'
+require 'yaml'
+
+class Compiler
+  class CompileError < RuntimeError
+    attr_reader :status, :log
+
+    def initialize(message, status: 400, log:)
+      super(message)
+      @status = status
+      @log = log
+    end
+  end
+
+  def compile(keymap_data, lhs_kconfig_data, rhs_kconfig_data)
+    if rhs_kconfig_data && !rhs_kconfig_data.empty?
+      lhs_result, lhs_output = compile_board('glove80_lh', keymap_data:,     kconfig_data: lhs_kconfig_data, include_static_rhs: false)
+      rhs_result, rhs_output = compile_board('glove80_rh', keymap_data: nil, kconfig_data: rhs_kconfig_data, include_static_rhs: false)
+      [
+        lhs_result.concat(rhs_result),
+        ["LHS Output:", *lhs_output, "RHS Output:", *rhs_output],
+      ]
+    else
+      compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: true)
+    end
+  end
+
+  def compile_board(board, keymap_data:, kconfig_data:, include_static_rhs: false)
+    in_build_dir do
+      compile_command = ['compileZmk', '-b', board]
+
+      if keymap_data
+        validate_devicetree!(keymap_data)
+        File.open('build.keymap', 'w') { |io| io.write(keymap_data) }
+        compile_command << '-k' << './build.keymap'
+      end
+
+      if kconfig_data
+        File.open('build.conf', 'w') { |io| io.write(kconfig_data) }
+        compile_command << '-c' << './build.conf'
+      end
+
+      if include_static_rhs
+        # Concatenate the pre-compiled glove80_rh image to the resulting uf2
+        compile_command << '-m'
+      end
+
+      compile_output = nil
+
+      IO.popen(compile_command, 'rb', err: [:child, :out]) do |io|
+        compile_output = io.read
+      end
+
+      compile_output = compile_output.split("\n")
+
+      unless $?.success?
+        status = $?.exitstatus
+        raise CompileError.new("Compile failed with exit status #{status}", log: compile_output)
+      end
+
+      unless File.exist?('zmk.uf2')
+        raise CompileError.new('Compile failed to produce result binary', status: 500, log: compile_output)
+      end
+
+      result = File.read('zmk.uf2')
+
+      [result, compile_output]
+    end
+  end
+
+  PERMITTED_DTS_SECTIONS = %w[
+    behaviors macros combos conditional_layers keymap underglow-indicators
+  ].freeze
+
+  def validate_devicetree!(dtsi)
+    dts = "/dts-v1/;\n" + dtsi
+
+    stdout, stderr, status =
+      Open3.capture3({}, 'dts2yml', unsetenv_others: true, stdin_data: dts)
+
+    unless status.success?
+      raise CompileError.new('Syntax error checking device-tree input', log: stderr.split("\n"))
+    end
+
+    data =
+      begin
+        YAML.safe_load(stdout)
+      rescue Psych::Exception => e
+        raise CompileError.new('Error parsing translated device-tree', status: 500, log: [e.message])
+      end
+
+    sections = data.flat_map(&:keys)
+    invalid_sections = sections - PERMITTED_DTS_SECTIONS
+
+    unless invalid_sections.empty?
+      raise CompileError.new(
+              "Device-tree included the non-permitted root sections: #{invalid_sections.inspect}", log: [])
+    end
+  end
+
+  # Lambda is single-process per container, and we get substantial speedups
+  # from ccache by always building in the same path
+  BUILD_DIR = '/tmp/build'
+
+  def in_build_dir
+    FileUtils.remove_entry(BUILD_DIR, true)
+    Dir.mkdir(BUILD_DIR)
+    Dir.chdir(BUILD_DIR)
+    yield
+  ensure
+    FileUtils.remove_entry(BUILD_DIR, true) rescue nil
+  end
+end

lambda/default.nix

@@ -0,0 +1,26 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs;
+
+let
+  bundleEnv = bundlerEnv {
+    name = "lambda-bundler-env";
+    ruby     = ruby_3_1;
+    gemfile  = ./Gemfile;
+    lockfile = ./Gemfile.lock;
+    gemset   = ./gemset.nix;
+  };
+
+  source = stdenv.mkDerivation {
+    name    = "lambda-builder";
+    version = "0.0.1";
+    src = ./.;
+    installPhase = ''
+      cp -r ./ $out
+    '';
+  };
+
+in
+{
+  inherit bundleEnv source;
+}

lambda/gemset.nix

@@ -0,0 +1,12 @@
+{
+  aws_lambda_ric = {
+    groups = ["default"];
+    platforms = [];
+    source = {
+      remotes = ["https://rubygems.org"];
+      sha256 = "19c4xlgnhgwf3n3z57z16nmr76jd2vihhshknm5zqip2g00awhi1";
+      type = "gem";
+    };
+    version = "2.0.0";
+  };
+}