diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 00000000000..7062812693f --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,94 @@ +name: Build Compiler Service Container + +on: + push: + tags: + - "*" + pull_request_target: + branches: + - main + +jobs: + build: + # This job must never be run on a PR from outside the same repository + if: github.repository == 'moergo-sc/zmk' && (github.event.pull_request == null || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + env: + ECR_REPOSITORY: zmk-builder-lambda + VERSIONS_BUCKET: glove80firmwarepipelines-compilerversionsbucket44-zubaquiyjdam + UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV + REVISION_TAG: ${{ github.event.pull_request && github.event.pull_request.head.sha || github.sha }} + PR_NUMBER: ${{ github.event.number }} + steps: + - uses: actions/checkout@v2.4.0 + with: + repository: moergo-sc/zmk + ref: ${{ github.event.pull_request && github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder + aws-region: us-east-1 + - name: Extract container name from branch name + shell: bash + run: | + if [ "$GITHUB_HEAD_REF" ]; then + tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}" + elif [[ "$GITHUB_REF" == refs/tags/* ]]; then + tag="${GITHUB_REF#refs/tags/}" + else + echo "Not a pull request or release tag" >&2 + exit 1 + fi + # Replace / with . in container tag names + tag="${tag//\//.}" + echo "CONTAINER_NAME=${tag}" >> $GITHUB_ENV + id: extract_name + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - uses: cachix/install-nix-action@v20 + with: + nix_path: nixpkgs=channel:nixos-22.05 + - uses: cachix/cachix-action@v12 + with: + name: moergo-glove80-zmk-dev + authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" + - name: Build lambda image + run: nix-build release.nix --arg revision "\"${REVISION_TAG}\"" -A lambdaImage -o lambdaImage + - name: Import OCI image into docker-daemon + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: skopeo --insecure-policy copy oci:lambdaImage docker-daemon:$REGISTRY/$ECR_REPOSITORY:$REVISION_TAG + - name: Push container image to Amazon ECR + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: docker push $REGISTRY/$ECR_REPOSITORY:$REVISION_TAG + - name: Create JSON metadata to represent the built container + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + shell: bash + run: | + digest="$(docker inspect --format='{{index .RepoDigests 0}}' $REGISTRY/$ECR_REPOSITORY:$REVISION_TAG)" + digest="${digest##*@}" + api_version="$(cat lambda/api_version.txt)" + jq -n '$ARGS.named' \ + --arg name "$CONTAINER_NAME" \ + --arg revision "$REVISION_TAG" \ + --arg branch "$GITHUB_REF" \ + --arg digest "$digest" \ + --arg api_version "$api_version" \ + > "/tmp/$CONTAINER_NAME.json" + - name: Upload image metadata file to versions bucket + run: aws s3 cp "/tmp/$CONTAINER_NAME.json" "s3://$VERSIONS_BUCKET/images/$CONTAINER_NAME.json" + - name: Notify the build pipeline that the compile containers have updated + run: >- + aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION + --invocation-type Event + --cli-binary-format raw-in-base64-out + /dev/null diff --git a/.github/workflows/cleanup-container.yml b/.github/workflows/cleanup-container.yml new file mode 100644 index 00000000000..627f93f034f --- /dev/null +++ b/.github/workflows/cleanup-container.yml @@ -0,0 +1,43 @@ +name: Clean up PR Compiler Service Container + +on: + pull_request: + types: [closed] + branches: + - main + +jobs: + build: + if: github.repository == 'moergo-sc/zmk' + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + env: + ECR_REPOSITORY: zmk-builder-lambda + VERSIONS_BUCKET: glove80firmwarepipelines-compilerversionsbucket44-zubaquiyjdam + UPDATE_COMPILER_VERSIONS_FUNCTION: arn:aws:lambda:us-east-1:431227615537:function:Glove80FirmwarePipelineSt-UpdateCompilerVersions2A-CNxPOHb4VSuV + PR_NUMBER: ${{ github.event.number }} + steps: + - name: Extract image tag name + shell: bash + run: | + tag="pr${PR_NUMBER}.${GITHUB_HEAD_REF}" + # Replace / with . in container tag names + tag="${tag//\//.}" + echo "CONTAINER_NAME=${tag}" >> $GITHUB_ENV + id: extract_name + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::431227615537:role/GithubCompilerLambdaBuilder + aws-region: us-east-1 + - name: Delete the image metadata file from the versions s3 bucket + run: aws s3 rm s3://$VERSIONS_BUCKET/images/$CONTAINER_NAME.json + - name: Notify the build pipeline that the compile containers have updated + run: >- + aws lambda invoke --function-name $UPDATE_COMPILER_VERSIONS_FUNCTION + --invocation-type Event + --cli-binary-format raw-in-base64-out + /dev/null diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index d864503167e..faaeabc220e 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -1,4 +1,4 @@ -name: Build +name: Build Glove80 Firmware on: push: diff --git a/lambda/Gemfile b/lambda/Gemfile new file mode 100644 index 00000000000..6b6bbf753ec --- /dev/null +++ b/lambda/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' +gem 'aws_lambda_ric' + diff --git a/lambda/Gemfile.lock b/lambda/Gemfile.lock new file mode 100644 index 00000000000..8b6c1f95c58 --- /dev/null +++ b/lambda/Gemfile.lock @@ -0,0 +1,13 @@ +GEM + remote: https://rubygems.org/ + specs: + aws_lambda_ric (2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + aws_lambda_ric + +BUNDLED WITH + 2.1.4 diff --git a/lambda/api_version.txt b/lambda/api_version.txt new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/lambda/api_version.txt @@ -0,0 +1 @@ +2 diff --git a/lambda/app.rb b/lambda/app.rb new file mode 100644 index 00000000000..8c3eb554cca --- /dev/null +++ b/lambda/app.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'stringio' +require 'digest' +require 'json' +require './compiler' + +module LambdaFunction + # Handle a non-HTTP compile request, returning a JSON body of either the + # compiled result or an error. + class Handler + REVISION = ENV.fetch('REVISION', 'unknown') + + def self.process(event:, context:) + return { type: 'keep_alive' } if event.has_key?('keep_alive') + + parse_base64_param = ->(param, required: true) do + if event.include?(param) + Base64.strict_decode64(event.fetch(param)) + elsif required + return error(status: 400, message: "Missing required argument: #{param}") + end + rescue ArgumentError + return error(status: 400, message: "Invalid Base64 in #{param} input") + end + + keymap_data = parse_base64_param.('keymap') + kconfig_data = parse_base64_param.('kconfig', required: false) + + # Including kconfig settings that affect the RHS require building both + # firmware images, doubling compile time. Clients should omit rhs_kconfig + # where possible. + rhs_kconfig_data = parse_base64_param.('rhs_kconfig', required: false) + + result, log = + begin + log_compile(keymap_data, kconfig_data, rhs_kconfig_data) + + Compiler.new.compile(keymap_data, kconfig_data, rhs_kconfig_data) + rescue Compiler::CompileError => e + return error(status: e.status, message: e.message, detail: e.log) + end + + result = Base64.strict_encode64(result) + + { type: 'result', result: result, log: log, revision: REVISION } + rescue StandardError => e + error(status: 500, message: "Unexpected error: #{e.class}", detail: [e.message], exception: e) + end + + def self.log_compile(keymap_data, kconfig_data, rhs_kconfig_data) + keymap = Digest::SHA1.base64digest(keymap_data) + kconfig = kconfig_data ? Digest::SHA1.base64digest(kconfig_data) : 'nil' + rhs_kconfig = rhs_kconfig_data ? Digest::SHA1.base64digest(rhs_kconfig_data) : 'nil' + puts("Compiling with keymap: #{keymap}; kconfig: #{kconfig}; rhs_kconfig: #{rhs_kconfig}") + end + + def self.error(status:, message:, detail: nil, exception: nil) + reported_error = { type: 'error', status:, message:, detail:, revision: REVISION } + + exception_detail = { class: exception.class, backtrace: exception.backtrace } if exception + logged_error = reported_error.merge(exception: exception_detail) + puts(JSON.dump(logged_error)) + + reported_error + end + end +end diff --git a/lambda/compiler.rb b/lambda/compiler.rb new file mode 100644 index 00000000000..1dba233afac --- /dev/null +++ b/lambda/compiler.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require 'tmpdir' +require 'json' +require 'base64' + +class Compiler + class CompileError < RuntimeError + attr_reader :status, :log + + def initialize(message, status: 400, log:) + super(message) + @status = status + @log = log + end + end + + def compile(keymap_data, lhs_kconfig_data, rhs_kconfig_data) + if rhs_kconfig_data && !rhs_kconfig_data.empty? + lhs_result, lhs_output = compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: false) + rhs_result, rhs_output = compile_board('glove80_rh', keymap_data: nil, kconfig_data: rhs_kconfig_data, include_static_rhs: false) + [ + lhs_result.concat(rhs_result), + ["LHS Output:", *lhs_output, "RHS Output:", *rhs_output], + ] + else + compile_board('glove80_lh', keymap_data:, kconfig_data: lhs_kconfig_data, include_static_rhs: true) + end + end + + def compile_board(board, keymap_data:, kconfig_data:, include_static_rhs: false) + in_build_dir do + compile_command = ['compileZmk', '-b', board] + + if keymap_data + File.open('build.keymap', 'w') { |io| io.write(keymap_data) } + compile_command << '-k' << './build.keymap' + end + + if kconfig_data + File.open('build.conf', 'w') { |io| io.write(kconfig_data) } + compile_command << '-c' << './build.conf' + end + + if include_static_rhs + # Concatenate the pre-compiled glove80_rh image to the resulting uf2 + compile_command << '-m' + end + + compile_output = nil + + IO.popen(compile_command, 'rb', err: [:child, :out]) do |io| + compile_output = io.read + end + + compile_output = compile_output.split("\n") + + unless $?.success? + status = $?.exitstatus + raise CompileError.new("Compile failed with exit status #{status}", log: compile_output) + end + + unless File.exist?('zmk.uf2') + raise CompileError.new('Compile failed to produce result binary', status: 500, log: compile_output) + end + + result = File.read('zmk.uf2') + + [result, compile_output] + end + end + + # Lambda is single-process per container, and we get substantial speedups + # from ccache by always building in the same path + BUILD_DIR = '/tmp/build' + + def in_build_dir + FileUtils.remove_entry(BUILD_DIR, true) + Dir.mkdir(BUILD_DIR) + Dir.chdir(BUILD_DIR) + yield + ensure + FileUtils.remove_entry(BUILD_DIR, true) rescue nil + end +end diff --git a/lambda/default.nix b/lambda/default.nix new file mode 100644 index 00000000000..6a34b0d1122 --- /dev/null +++ b/lambda/default.nix @@ -0,0 +1,26 @@ +{ pkgs ? import {} }: + +with pkgs; + +let + bundleEnv = bundlerEnv { + name = "lambda-bundler-env"; + ruby = ruby_3_1; + gemfile = ./Gemfile; + lockfile = ./Gemfile.lock; + gemset = ./gemset.nix; + }; + + source = stdenv.mkDerivation { + name = "lambda-builder"; + version = "0.0.1"; + src = ./.; + installPhase = '' + cp -r ./ $out + ''; + }; + +in +{ + inherit bundleEnv source; +} diff --git a/lambda/gemset.nix b/lambda/gemset.nix new file mode 100644 index 00000000000..6b2fd1a0207 --- /dev/null +++ b/lambda/gemset.nix @@ -0,0 +1,12 @@ +{ + aws_lambda_ric = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "19c4xlgnhgwf3n3z57z16nmr76jd2vihhshknm5zqip2g00awhi1"; + type = "gem"; + }; + version = "2.0.0"; + }; +} diff --git a/lambda/shell.nix b/lambda/shell.nix new file mode 100644 index 00000000000..2f1eca8bb75 --- /dev/null +++ b/lambda/shell.nix @@ -0,0 +1,9 @@ +{ pkgs ? (import {})}: + +let + lambda = import ./default.nix { inherit pkgs; }; +in +pkgs.stdenv.mkDerivation { + name = "lambda-shell"; + buildInputs = [lambda.bundleEnv.wrappedRuby]; +} diff --git a/nix/ccache.nix b/nix/ccache.nix new file mode 100644 index 00000000000..030153140e2 --- /dev/null +++ b/nix/ccache.nix @@ -0,0 +1,43 @@ +{ stdenv, lib, makeWrapper, ccache +, unwrappedCC ? stdenv.cc.cc, extraConfig ? "" }: + +# copied from ccache in nixpkgs, modified to glob over prefixes. Also doesn't +# pass lib. Why was it passing lib? +stdenv.mkDerivation { + name = "ccache-links"; + passthru = { + isClang = unwrappedCC.isClang or false; + isGNU = unwrappedCC.isGNU or false; + }; + nativeBuildInputs = [ makeWrapper ]; + buildCommand = '' + mkdir -p $out/bin + + wrap() { + local cname="$(basename $1)" + if [ -x "${unwrappedCC}/bin/$cname" ]; then + echo "Wrapping $1" + makeWrapper ${ccache}/bin/ccache $out/bin/$cname \ + --run ${lib.escapeShellArg extraConfig} \ + --add-flags ${unwrappedCC}/bin/$cname + fi + } + + wrapAll() { + for prog in "$@"; do + wrap "$prog" + done + } + + wrapAll ${unwrappedCC}/bin/{*cc,*c++,*gcc,*g++,*clang,*clang++} + + for executable in $(ls ${unwrappedCC}/bin); do + if [ ! -x "$out/bin/$executable" ]; then + ln -s ${unwrappedCC}/bin/$executable $out/bin/$executable + fi + done + for file in $(ls ${unwrappedCC} | grep -vw bin); do + ln -s ${unwrappedCC}/$file $out/$file + done + ''; +} diff --git a/nix/zmk.nix b/nix/zmk.nix index 7f6b897b911..aca06e24f42 100644 --- a/nix/zmk.nix +++ b/nix/zmk.nix @@ -4,6 +4,7 @@ , board ? "glove80_lh" , shield ? null , keymap ? null +, kconfig ? null }: @@ -66,7 +67,9 @@ stdenvNoCC.mkDerivation { # Transient state relPath == "build" || relPath == ".west" || # Fetched by west - relPath == "modules" || relPath == "tools" || relPath == "zephyr" + relPath == "modules" || relPath == "tools" || relPath == "zephyr" || + # Not part of ZMK + relPath == "lambda" || relPath == ".github" ); }; @@ -90,7 +93,8 @@ stdenvNoCC.mkDerivation { "-DZEPHYR_MODULES=${lib.concatStringsSep ";" zephyrModuleDeps}" ] ++ (lib.optional (shield != null) "-DSHIELD=${shield}") ++ - (lib.optional (keymap != null) "-DKEYMAP_FILE=${keymap}"); + (lib.optional (keymap != null) "-DKEYMAP_FILE=${keymap}") ++ + (lib.optional (kconfig != null) "-DCONF_FILE=${kconfig}"); nativeBuildInputs = [ cmake ninja python dtc gcc-arm-embedded ]; buildInputs = [ zephyr ]; diff --git a/release.nix b/release.nix new file mode 100644 index 00000000000..3ed1cb5ef8d --- /dev/null +++ b/release.nix @@ -0,0 +1,197 @@ +{ pkgs ? (import ./nix/pinned-nixpkgs.nix {}), revision ? "HEAD" }: + +let + lib = pkgs.lib; + zmkPkgs = (import ./default.nix { inherit pkgs; }); + lambda = (import ./lambda { inherit pkgs; }); + ccacheWrapper = pkgs.callPackage ./nix/ccache.nix {}; + + nix-utils = pkgs.fetchFromGitHub { + owner = "iknow"; + repo = "nix-utils"; + rev = "c13c7a23836c8705452f051d19fc4dff05533b53"; + sha256 = "0ax7hld5jf132ksdasp80z34dlv75ir0ringzjs15mimrkw8zcac"; + }; + + ociTools = pkgs.callPackage "${nix-utils}/oci" {}; + + inherit (zmkPkgs) zmk zephyr; + + accounts = { + users.deploy = { + uid = 999; + group = "deploy"; + home = "/home/deploy"; + shell = "/bin/sh"; + }; + groups.deploy.gid = 999; + }; + + baseLayer = { + name = "base-layer"; + path = [ pkgs.busybox ]; + entries = ociTools.makeFilesystem { + inherit accounts; + tmp = true; + usrBinEnv = "${pkgs.busybox}/bin/env"; + binSh = "${pkgs.busybox}/bin/sh"; + }; + }; + + depsLayer = { + name = "deps-layer"; + path = [ pkgs.ccache ]; + includes = zmk.buildInputs ++ zmk.nativeBuildInputs ++ zmk.zephyrModuleDeps; + }; + + zmkCompileScript = let + zmk' = zmk.override { + gcc-arm-embedded = ccacheWrapper.override { + unwrappedCC = pkgs.gcc-arm-embedded; + }; + }; + zmk_glove80_rh = zmk.override { board = "glove80_rh"; }; + realpath_coreutils = if pkgs.stdenv.isDarwin then pkgs.coreutils else pkgs.busybox; + in pkgs.writeShellScriptBin "compileZmk" '' + set -eo pipefail + + function usage() { + echo "Usage: compileZmk [-m] [-k keymap_file] [-c kconfig_file] [-b board]" + } + + function checkPath() { + if [ -z "$1" ]; then + return 0 + elif [ ! -f "$1" ]; then + echo "Error: Missing $2 file" >&2 + usage >&2 + exit 1 + fi + + ${realpath_coreutils}/bin/realpath "$1" + } + + keymap="${zmk.src}/app/boards/arm/glove80/glove80.keymap" + kconfig="" + board="glove80_lh" + merge_rhs="" + + while getopts "hk:c:d:b:m" opt; do + case "$opt" in + h|\?) + usage >&2 + exit 1 + ;; + k) + keymap="$OPTARG" + ;; + c) + kconfig="$OPTARG" + ;; + b) + board="$OPTARG" + ;; + m) + merge_rhs=t + ;; + esac + done + + if [ "$board" = "glove80_rh" -a -n "$merge_rhs" ]; then + echo "Cannot merge static RHS with built RHS" >&2 + exit 2 + fi + + keymap="$(checkPath "$keymap" keymap)" + kconfig="$(checkPath "$kconfig" Kconfig)" + + export PATH=${lib.makeBinPath (with pkgs; zmk'.nativeBuildInputs ++ [ ccache ])}:$PATH + export CMAKE_PREFIX_PATH=${zephyr} + + export CCACHE_BASEDIR=$PWD + export CCACHE_NOHASHDIR=t + export CCACHE_COMPILERCHECK=none + + if [ -n "$DEBUG" ]; then ccache -z; fi + + cmake -G Ninja -S ${zmk'.src}/app ${lib.escapeShellArgs zmk'.cmakeFlags} "-DUSER_CACHE_DIR=/tmp/.cache" "-DKEYMAP_FILE=$keymap" "-DCONF_FILE=$kconfig" "-DBOARD=$board" "-DBUILD_VERSION=${revision}" + + ninja + + if [ -n "$DEBUG" ]; then ccache -s; fi + + if [ -n "$merge_rhs" ]; then + cat zephyr/zmk.uf2 ${zmk_glove80_rh}/zmk.uf2 > zmk.uf2 + else + mv zephyr/zmk.uf2 zmk.uf2 + fi + ''; + + ccacheCache = pkgs.runCommandNoCC "ccache-cache" { + nativeBuildInputs = [ zmkCompileScript ]; + } '' + export CCACHE_DIR=$out + + mkdir /tmp/build + cd /tmp/build + + compileZmk -b glove80_lh -k ${zmk.src}/app/boards/arm/glove80/glove80.keymap + + rm -fr /tmp/build + mkdir /tmp/build + cd /tmp/build + + compileZmk -b glove80_rh -k ${zmk.src}/app/boards/arm/glove80/glove80.keymap + ''; + + entrypoint = pkgs.writeShellScriptBin "entrypoint" '' + set -euo pipefail + + if [ ! -d "$CCACHE_DIR" ]; then + cp -r ${ccacheCache} "$CCACHE_DIR" + chmod -R u=rwX,go=u-w "$CCACHE_DIR" + fi + + if [ ! -d /tmp/build ]; then + mkdir /tmp/build + fi + + exec "$@" + ''; + + startLambda = pkgs.writeShellScriptBin "startLambda" '' + set -euo pipefail + export PATH=${lib.makeBinPath [ zmkCompileScript ]}:$PATH + cd ${lambda.source} + ${lambda.bundleEnv}/bin/bundle exec aws_lambda_ric "app.LambdaFunction::Handler.process" + ''; + + simulateLambda = pkgs.writeShellScriptBin "simulateLambda" '' + ${pkgs.aws-lambda-rie}/bin/aws-lambda-rie ${startLambda}/bin/startLambda + ''; + + lambdaImage = + let + appLayer = { + name = "app-layer"; + path = [ startLambda zmkCompileScript ]; + }; + in + ociTools.makeSimpleImage { + name = "zmk-builder-lambda"; + layers = [ baseLayer depsLayer appLayer ]; + config = { + User = "deploy"; + WorkingDir = "/tmp"; + Entrypoint = [ "${entrypoint}/bin/entrypoint" ]; + Cmd = [ "startLambda" ]; + Env = [ "CCACHE_DIR=/tmp/ccache" "REVISION=${revision}" ]; + }; + }; +in { + inherit lambdaImage zmkCompileScript ccacheCache; + directLambdaImage = lambdaImage; + + # nix shell -f release.nix simulateLambda -c simulateLambda + inherit simulateLambda; +}