Azure images

.reuse/dep5

@@ -2,4 +2,4 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 
 Copyright: 2023 Technology Innovation Institute (TII)
 License: Apache-2.0
-Files: *.lock *.png *.svg *.csv *.yaml
+Files: *.lock *.png *.svg *.csv *.yaml *.pub

docs/adapting-to-new-environments.md

@@ -164,7 +164,6 @@ $ cat hosts/mytarget/configuration.nix
   # Define the services you want to run on your target, as well as the users
   # who can access the target with ssh:
   imports = [
-    inputs.nix-serve-ng.nixosModules.default
     inputs.sops-nix.nixosModules.sops
     inputs.disko.nixosModules.disko
     ../generic-disk-config.nix

flake.nix

@@ -6,7 +6,7 @@
 
   inputs = {
     # Nixpkgs
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05";
+    nixpkgs.url = "github:nixos/nixpkgs/master";
     # Allows us to structure the flake with the NixOS module system
     flake-parts.url = "github:hercules-ci/flake-parts";
     flake-root.url = "github:srid/flake-root";
@@ -19,7 +19,8 @@
     # Binary cache with nix-serve-ng
     nix-serve-ng = {
       url = "github:aristanetworks/nix-serve-ng";
-      inputs.nixpkgs.follows = "nixpkgs";
+      # Broken with 23.11, base32 misses text >=2.0 && <2.1
+      # inputs.nixpkgs.follows = "nixpkgs";
     };
     # Disko for disk partitioning
     disko = {

hosts/azure-common-2.nix

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Profile to import for Azure VMs. Imports azure-common.nix from nixpkgs,
+# and configures cloud-init.
+{modulesPath, ...}: {
+  imports = [
+    "${modulesPath}/virtualisation/azure-config.nix"
+  ];
+
+  # enable cloud-init, so instance metadata is set accordingly and we can use
+  # cloud-config for ssh key management.
+  services.cloud-init.enable = true;
+
+  # Use systemd-networkd for network configuration.
+  services.cloud-init.network.enable = true;
+  networking.useDHCP = false;
+  networking.useNetworkd = true;
+  # FUTUREWORK: Ideally, we'd keep systemd-resolved disabled too,
+  # but the way nixpkgs configures cloud-init prevents it from picking up DNS
+  # settings from elsewhere.
+  # services.resolved.enable = false;
+}

hosts/azure-scratch-store-common.nix

@@ -0,0 +1,86 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  pkgs,
+  utils,
+  ...
+}: {
+  # Disable explicit resource disk handling in waagent.
+  # We want to take control over it in initrd already.
+  virtualisation.azure.agent.mountResourceDisk = false;
+
+  boot.initrd.systemd = {
+    # This requires systemd-in-initrd.
+    enable = true;
+
+    # We need the wipefs binary available in the initrd
+    extraBin = {
+      "wipefs" = "${pkgs.util-linux}/bin/wipefs";
+    };
+
+    # The resource disk comes pre-formatted with NTFS, not ext4.
+    # Wipe the superblock if it's NTFS (and only then, to not wipe on every reboot).
+    # Once we get `filesystems`-syntax to work again, we could delegate the mkfs
+    # part to systemd-makefs (and make this `wantedBy` and `before` that makefs
+    # unit).
+    services.wipe-resource-disk = {
+      description = "Wipe resource disk before makefs";
+      requires = ["${utils.escapeSystemdPath "dev/disk/azure/resource-part1"}.device"];
+      after = ["${utils.escapeSystemdPath "dev/disk/azure/resource-part1"}.device"];
+      wantedBy = ["${utils.escapeSystemdPath "sysroot/mnt/resource"}.mount"];
+      before = ["${utils.escapeSystemdPath "sysroot/mnt/resource"}.mount"];
+
+      script = ''
+        if [[ $(wipefs --output=TYPE -p /dev/disk/azure/resource-part1) == "ntfs" ]]; then
+          echo "wiping resource disk (was ntfs)"
+          wipefs -a /dev/disk/azure/resource-part1
+          mkfs.ext4 /dev/disk/azure/resource-part1
+        else
+          echo "skip wiping resource disk (not ntfs)"
+        fi
+      '';
+    };
+
+    # Once /sysroot/mnt/resource is mounted, ensure the two .rw-store/
+    # {work,store} directories that overlayfs is using are present.
+    # The kernel doesn't create them on its own and fails the mount if they're
+    # not present, so we set `wantedBy` and `before` to the .mount unit.
+    services.setup-resource-disk = {
+      description = "Setup resource disk after it's mounted";
+      unitConfig.RequiresMountsFor = "/sysroot/mnt/resource";
+      wantedBy = ["${utils.escapeSystemdPath "sysroot/nix/store"}.mount"];
+      before = ["${utils.escapeSystemdPath "sysroot/nix/store"}.mount"];
+
+      script = ''
+        mkdir -p /sysroot/mnt/resource/.rw-store/{work,store}
+      '';
+    };
+
+    # These describe the mountpoints inside the initrd
+    # (/sysroot/mnt/resource, /sysroot/nix/store).
+    # In the future, this should be moved to `filesystems`-syntax, so we can
+    # make use of systemd-makefs and can write some things more concisely.
+    mounts = [
+      {
+        where = "/sysroot/mnt/resource";
+        what = "/dev/disk/azure/resource-part1";
+        type = "ext4";
+      }
+      # describe the overlay mount
+      {
+        where = "/sysroot/nix/store";
+        what = "overlay";
+        type = "overlay";
+        options = "lowerdir=/sysroot/nix/store,upperdir=/sysroot/mnt/resource/.rw-store/store,workdir=/sysroot/mnt/resource/.rw-store/work";
+        wantedBy = ["initrd-fs.target"];
+        before = ["initrd-fs.target"];
+        requires = ["setup-resource-disk.service"];
+        after = ["setup-resource-disk.service"];
+        unitConfig.RequiresMountsFor = ["/sysroot" "/sysroot/mnt/resource"];
+      }
+    ];
+  };
+  # load the overlay kernel module
+  boot.initrd.kernelModules = ["overlay"];
+}

hosts/binary-cache/configuration.nix

@@ -0,0 +1,103 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  self,
+  config,
+  pkgs,
+  lib,
+  ...
+}: {
+  imports = [
+    ../azure-common-2.nix
+    ../azure-scratch-store-common.nix
+    self.nixosModules.service-openssh
+  ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  # Configure /var/lib/caddy in /etc/fstab.
+  # Due to an implicit RequiresMountsFor=$state-dir, systemd
+  # will block starting the service until this mounted.
+  fileSystems."/var/lib/caddy" = {
+    device = "/dev/disk/by-lun/10";
+    fsType = "ext4";
+    options = [
+      "x-systemd.makefs"
+      "x-systemd.growfs"
+    ];
+  };
+
+  # Run a read-only HTTP webserver proxying to the "binary-cache-v1" storage
+  # container at a unix socket.
+  # This relies on IAM to grant access to the storage container.
+  systemd.services.rclone-http = {
+    after = ["network.target"];
+    requires = ["network.target"];
+    wantedBy = ["multi-user.target"];
+    serviceConfig = {
+      Type = "notify";
+      Restart = "always";
+      RestartSec = 2;
+      DynamicUser = true;
+      RuntimeDirectory = "rclone-http";
+      ExecStart =
+        "${pkgs.rclone}/bin/rclone "
+        + "serve http "
+        + "--azureblob-env-auth "
+        + "--read-only "
+        + "--addr unix://%t/rclone-http/socket "
+        + ":azureblob:binary-cache-v1";
+      # On successful startup, grant caddy write permissions to the socket.
+      ExecStartPost = "${pkgs.acl.bin}/bin/setfacl -m u:caddy:rw %t/rclone-http/socket";
+      EnvironmentFile = "/var/lib/rclone-http/env";
+    };
+  };
+
+  # Expose the rclone-http unix socket over a HTTPS, limiting to certain
+  # keys only, disallowing listing too.
+  # TODO: use https://caddyserver.com/docs/caddyfile-tutorial#environment-variables for domain
+  services.caddy = {
+    enable = true;
+    configFile = pkgs.writeTextDir "Caddyfile" ''
+      # Disable the admin API, we don't want to reconfigure Caddy at runtime.
+      {
+        admin off
+      }
+
+      # Proxy a subset of requests to rclone.
+      https://{$SITE_ADDRESS} {
+        handle /nix-cache-info {
+          reverse_proxy unix///run/rclone-http/socket
+        }
+        handle /*.narinfo {
+          reverse_proxy unix///run/rclone-http/socket
+        }
+        handle /nar/*.nar {
+          reverse_proxy unix///run/rclone-http/socket
+        }
+        handle /nar/*.nar.* {
+          reverse_proxy unix///run/rclone-http/socket
+        }
+      }
+    '';
+  };
+
+  # workaround for https://github.com/NixOS/nixpkgs/issues/272532
+  # FUTUREWORK: rebase once https://github.com/NixOS/nixpkgs/pull/272617 landed
+  services.caddy.enableReload = false;
+  systemd.services.caddy.serviceConfig.ExecStart = lib.mkForce [
+    ""
+    "${pkgs.caddy}/bin/caddy run --environ --config ${config.services.caddy.configFile}/Caddyfile"
+  ];
+  systemd.services.caddy.serviceConfig.EnvironmentFile = "/run/caddy.env";
+
+  # Wait for cloud-init mounting before we start caddy.
+  systemd.services.caddy.after = ["cloud-init.service"];
+  systemd.services.caddy.requires = ["cloud-init.service"];
+
+  # Expose the HTTPS port. No need for HTTP, as caddy can use TLS-ALPN-01.
+  networking.firewall.allowedTCPPorts = [443];
+
+  system.stateVersion = "23.05";
+}

hosts/binarycache/configuration.nix

@@ -13,7 +13,6 @@
 
   imports = lib.flatten [
     (with inputs; [
-      nix-serve-ng.nixosModules.default
       sops-nix.nixosModules.sops
       disko.nixosModules.disko
     ])