Start using terraform infra with nixos configs

Prepare ghaf-infra so that the infra created with terraform can be used with flake nixos configurations. This includes the following changes: - Remove testhost terraform configuration since it's not used in ghaf-infra. - Use static internal IP addresses in terraform VM configurations. - Configure public cloudapp.azure.com domain names for terraform VMs by using the 'domain_name_label' configuration option. - Use the cloudapp.azure.com domain names in task.py alias hostnames where appropriate. - Use the azarm VM as a remote aarch64 builder for the ghafhydra (in addition to awsarm). In addition, this commit changes some common garbage collection options. Signed-off-by: Henri Rosten <[email protected]>
tiiuae · Nov 15, 2023 · dc58805 · dc58805
1 parent ff38a56
commit dc58805
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 40 deletions.
diff --git a/hosts/common.nix b/hosts/common.nix
@@ -44,9 +44,9 @@ in {
       # perform a garbage-collection until max-free bytes are available or there
       # is no more garbage.
       min-free = asGB 20;
-      max-free = asGB 100;
-      # check the free disk space every 10 seconds
-      min-free-check-interval = 10;
+      max-free = asGB 200;
+      # check the free disk space every 5 seconds
+      min-free-check-interval = 5;
     };
     # Garbage collection
     gc.automatic = true;

diff --git a/hosts/default.nix b/hosts/default.nix
@@ -23,10 +23,11 @@
     # make self and inputs available in nixos modules
     specialArgs = {inherit self inputs;};
   in {
-    build01 = lib.nixosSystem {
-      inherit specialArgs;
-      modules = [self.nixosModules.host-build01];
-    };
+    # Currently not used for anything:
+    # build01 = lib.nixosSystem {
+    #   inherit specialArgs;
+    #   modules = [self.nixosModules.host-build01];
+    # };
     ghafhydra = lib.nixosSystem {
       inherit specialArgs;
       modules = [self.nixosModules.host-ghafhydra];

diff --git a/hosts/ghafhydra/default.nix b/hosts/ghafhydra/default.nix
@@ -47,7 +47,7 @@
   security.acme.acceptTerms = true;
   services.nginx = {
     virtualHosts = {
-      "ghafhydra.swedencentral.cloudapp.azure.com" = {
+      "ghafhydra.northeurope.cloudapp.azure.com" = {
         forceSSL = true;
         enableACME = true;
         locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
@@ -58,10 +58,10 @@
   # TODO: demo with static IP:
   networking.useDHCP = false;
   networking.nameservers = ["1.1.1.1" "8.8.8.8"];
-  networking.defaultGateway = "10.3.0.1";
+  networking.defaultGateway = "10.0.2.1";
   networking.interfaces.eth0.ipv4.addresses = [
     {
-      address = "10.3.0.4";
+      address = "10.0.2.4";
       prefixLength = 24;
     }
   ];

diff --git a/services/hydra/default.nix b/services/hydra/default.nix
@@ -23,17 +23,16 @@
       localhost x86_64-linux - 8 2 kvm,benchmark,big-parallel,nixos-test - -
     '';
   };
-  build01Machine = pkgs.writeTextFile {
-    name = "build-build01Machine";
-    # TODO: get rid of static IP config:
+  azarmMachine = pkgs.writeTextFile {
+    name = "build-azarmMachine";
     text = ''
-      ssh://nix@10.3.0.5 x86_64-linux ${config.sops.secrets.id_buildfarm.path} 8 2 kvm,benchmark,big-parallel,nixos-test - -
+      ssh://nix@10.0.2.10 aarch64-linux ${config.sops.secrets.id_buildfarm.path} 8 2 kvm,benchmark,big-parallel,nixos-test - -
     '';
   };
   awsarmMachine = pkgs.writeTextFile {
     name = "build-awsarmMachine";
     text = ''
-      ssh://[email protected] aarch64-linux ${config.sops.secrets.id_buildfarm.path} 8 2 kvm,benchmark,big-parallel,nixos-test - -
+      ssh://[email protected] aarch64-linux ${config.sops.secrets.id_buildfarm.path} 16 4 kvm,benchmark,big-parallel,nixos-test - -
     '';
   };
   createJobsetsScript = pkgs.stdenv.mkDerivation {
@@ -49,12 +48,11 @@
 in {
   programs.ssh.knownHosts = {
     # Add builder machines' public ids to ssh known_hosts
-    build01 = {
-      # TODO: get rid of static IP config:
-      hostNames = ["10.3.0.5"];
-      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID+hx/Ff8U123lI8wMYvmVYn5M3Cv4m+XQxxNYFgJGTo";
+    azarm = {
+      hostNames = ["10.0.2.10"];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWojItdbPn0dNbGKfCSJv+duYsb+xzJ6hPWOu+TZ4rm";
     };
-    armbuild01 = {
+    awsarm = {
       hostNames = ["awsarm.vedenemo.dev"];
       publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3f7tAAO3Fc+8BqemsBQc/Yl/NmRfyhzr5SFOSKqrv0";
     };
@@ -73,7 +71,7 @@ in {
 
     buildMachinesFiles = [
       "${localMachine}"
-      "${build01Machine}"
+      "${azarmMachine}"
       "${awsarmMachine}"
     ];
 

diff --git a/tasks.py b/tasks.py
@@ -68,13 +68,14 @@ class TargetHost:
 
 
 # Below dictionary defines the set of ghaf-infra configuration aliases:
-#  - Name (e.g. 'build01-dev) defines the alias name for each target.
+#  - Name (e.g. 'ghafhydra-dev) defines the alias name for each target.
 #  - TargetHost.hostname: host name or IP address of the target.
 #  - TargetHost.nixosconfig: name of the target nixosConfiguration.
 TARGETS = OrderedDict(
     {
-        "build01-dev": TargetHost(hostname="51.12.57.124", nixosconfig="build01"),
-        "ghafhydra-dev": TargetHost(hostname="51.12.56.79", nixosconfig="ghafhydra"),
+        "ghafhydra-dev": TargetHost(
+            hostname="ghafhydra.northeurope.cloudapp.azure.com", nixosconfig="ghafhydra"
+        ),
         "binarycache-ficolo": TargetHost(
             hostname="172.18.20.109", nixosconfig="binarycache"
         ),
@@ -245,7 +246,6 @@ def deploy(_c: Any, alias: str) -> None:
     inv deploy --alias ghafhydra-dev
     """
     h = get_deploy_host(alias)
-    command = "sudo nixos-rebuild"
     res = h.run_local(
         ["nix", "flake", "archive", "--to", f"ssh://{h.host}", "--json"],
         stdout=subprocess.PIPE,
@@ -255,6 +255,7 @@ def deploy(_c: Any, alias: str) -> None:
     LOG.debug("data['path']: %s", path)
     flags = "--option accept-flake-config true"
     nixosconfig = _get_target(alias).nixosconfig
+    command = "sudo nixos-rebuild"
     h.run(f"{command} switch {flags} --flake {path}#{nixosconfig}")
 
 

diff --git a/terraform/azure-ghaf-infra.tf b/terraform/azure-ghaf-infra.tf
@@ -65,38 +65,40 @@ resource "azurerm_network_security_group" "ghaf_infra_tf_nsg" {
 
 ################################################################################
 
-# testhost
+# ghafhydra:
 
 # Public IP
-resource "azurerm_public_ip" "testhost_public_ip" {
-  name                = "testhost-public-ip"
+resource "azurerm_public_ip" "ghafhydra_public_ip" {
+  name                = "ghafhydra-public-ip"
+  domain_name_label   = "ghafhydra"
   location            = azurerm_resource_group.ghaf_infra_tf_dev.location
   resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
   allocation_method   = "Static"
 }
 # Network interface
-resource "azurerm_network_interface" "testhost_ni" {
-  name                = "testhost-nic"
+resource "azurerm_network_interface" "ghafhydra_ni" {
+  name                = "ghafhydra-nic"
   location            = azurerm_resource_group.ghaf_infra_tf_dev.location
   resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
   ip_configuration {
-    name                          = "testhost_nic_configuration"
+    name                          = "ghafhydra_nic_configuration"
     subnet_id                     = azurerm_subnet.ghaf_infra_tf_subnet.id
-    private_ip_address_allocation = "Dynamic"
-    public_ip_address_id          = azurerm_public_ip.testhost_public_ip.id
+    private_ip_address_allocation = "Static"
+    private_ip_address            = "10.0.2.4"
+    public_ip_address_id          = azurerm_public_ip.ghafhydra_public_ip.id
   }
 }
-# Example Linux Virtual Machine (testhost)
-resource "azurerm_linux_virtual_machine" "testhost_vm" {
-  name                = "testhost"
+# Ghafhydra VM
+resource "azurerm_linux_virtual_machine" "ghafhydra_vm" {
+  name                = "ghafhydra"
   location            = azurerm_resource_group.ghaf_infra_tf_dev.location
   resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
   network_interface_ids = [
-    azurerm_network_interface.testhost_ni.id
+    azurerm_network_interface.ghafhydra_ni.id
   ]
   size = "Standard_B8ms"
   os_disk {
-    name                 = "testhost-disk"
+    name                 = "ghafhydra-disk"
     caching              = "ReadWrite"
     storage_account_type = "Premium_LRS"
     disk_size_gb         = 512
@@ -119,11 +121,17 @@ resource "azurerm_linux_virtual_machine" "testhost_vm" {
 
 ################################################################################
 
-# azarm
+# azarm:
+# aarch64-linux builder - Ubuntu host with nix package manager.
+# Why not NixOS? The reason is: we have not managed to get nixos-anywhere
+# working with azure arm VMs.
+# Since the host is not NixOS, all the host configuration is done on
+# terraform apply using the configuration script at scripts/ubuntu-builder.sh
 
 # Public IP
 resource "azurerm_public_ip" "azarm_public_ip" {
   name                = "azarm-public-ip"
+  domain_name_label   = "azarm"
   location            = azurerm_resource_group.ghaf_infra_tf_dev.location
   resource_group_name = azurerm_resource_group.ghaf_infra_tf_dev.name
   allocation_method   = "Static"
@@ -136,7 +144,8 @@ resource "azurerm_network_interface" "azarm_ni" {
   ip_configuration {
     name                          = "azarm_nic_configuration"
     subnet_id                     = azurerm_subnet.ghaf_infra_tf_subnet.id
-    private_ip_address_allocation = "Dynamic"
+    private_ip_address_allocation = "Static"
+    private_ip_address            = "10.0.2.10"
     public_ip_address_id          = azurerm_public_ip.azarm_public_ip.id
   }
 }