Make monitoring public with basic auth

Signed-off-by: Joonas Rautiola <[email protected]>
tiiuae · Nov 13, 2024 · d9a98c3 · d9a98c3
1 parent 98fae78
commit d9a98c3
Show file tree

Hide file tree

Showing 13 changed files with 166 additions and 95 deletions.
diff --git a/hosts/binarycache/configuration.nix b/hosts/binarycache/configuration.nix
@@ -25,7 +25,6 @@
       service-openssh
       service-binary-cache
       service-nginx
-      service-monitoring
       user-jrautiola
       user-cazfi
       user-hrosten

diff --git a/hosts/builders/build3/configuration.nix b/hosts/builders/build3/configuration.nix
@@ -7,9 +7,6 @@
   ...
 }:
 {
-  sops.defaultSopsFile = ./secrets.yaml;
-  sops.secrets.ssh_private_key.owner = "root";
-
   imports =
     [
       ../ficolo.nix
@@ -25,7 +22,15 @@
       user-remote-build # Remove when all jenkins builds moved to build4
     ]);
 
-  # build3 specific configuration
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets.ssh_private_key.owner = "root";
+  };
+
+  services.monitoring = {
+    metrics.enable = true;
+    logs.enable = true;
+  };
 
   networking.hostName = "build3";
 

diff --git a/hosts/builders/build4/configuration.nix b/hosts/builders/build4/configuration.nix
@@ -17,4 +17,8 @@
 
   networking.hostName = "build4";
 
+  services.monitoring = {
+    metrics.enable = true;
+    logs.enable = true;
+  };
 }
diff --git a/hosts/builders/ficolo.nix b/hosts/builders/ficolo.nix
@@ -16,7 +16,6 @@
       common
       ficolo-common
       service-openssh
-      service-monitoring
       user-cazfi
       user-hrosten
       user-jrautiola
@@ -32,11 +31,6 @@
     cpu.intel.updateMicrocode = true;
   };
 
-  services.monitoring = {
-    metrics.enable = true;
-    logs.enable = true;
-  };
-
   boot = {
     initrd.availableKernelModules = [
       "ahci"

diff --git a/hosts/builders/hetzarm/configuration.nix b/hosts/builders/hetzarm/configuration.nix
@@ -4,11 +4,10 @@
   self,
   inputs,
   lib,
+  config,
   ...
 }:
 {
-  sops.defaultSopsFile = ./secrets.yaml;
-
   imports =
     [
       ./disk-config.nix
@@ -31,6 +30,13 @@
       user-remote-build
     ]);
 
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets = {
+      loki_password.owner = "promtail";
+    };
+  };
+
   nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";
   hardware.enableRedistributableFirmware = true;
 
@@ -39,7 +45,17 @@
     useDHCP = true;
   };
 
-  services.monitoring.metrics.enable = true;
+  services.monitoring = {
+    metrics = {
+      enable = true;
+      ssh = true;
+    };
+    logs = {
+      enable = true;
+      lokiAddress = "https://monitoring.vedenemo.dev";
+      auth.password_file = config.sops.secrets.loki_password.path;
+    };
+  };
 
   boot = {
     initrd.availableKernelModules = [
@@ -54,22 +70,12 @@
     };
   };
 
-  users.users = {
-    # sshified user for monitoring server to log in as
-    sshified = {
-      isNormalUser = true;
-      openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEKd30t0EFmMyULGlecaUX6puIAF4IjynZUo+X9k8h69 monitoring"
-      ];
-    };
-
-    # build3 can use this as remote builder
-    build3 = {
-      isNormalUser = true;
-      openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPf56a3ISY64w0Y0BmoLu+RyTIWQrXG6ugla6if9RteT build3"
-      ];
-    };
+  # build3 can use this as remote builder
+  users.users.build3 = {
+    isNormalUser = true;
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPf56a3ISY64w0Y0BmoLu+RyTIWQrXG6ugla6if9RteT build3"
+    ];
   };
 
   nix.settings.trusted-users = [

diff --git a/hosts/builders/hetzarm/secrets.yaml b/hosts/builders/hetzarm/secrets.yaml
@@ -1,4 +1,5 @@
 ssh_host_ed25519_key: ENC[AES256_GCM,data:UTXKJoJTMT1Wk4UIC6wAaUs5vm3c1FWuTNqzOYkU1l1t57rrFLj3I9ZogClvWMdxZ/Y1rWtUX8845IKYoUEyKjenJg1OATjSB2kMJeF0WKVUtEsIuSstpvSIFk2H65+ABLrzNXiPJhqiUaL+3I7DvVDAWXNnt7dLKsmoxnmG7f9psVJVd6tR1Wx4orBeZjJHRO3YPq4uk7xy/CrJg45UzYZAbNnVw/pKPN19oRXwWYVRRJirEWrj+ECjbDNudtgwRfcIBJtrWGuAxdjiPlFxl7HWucHqs3M+5ziJyycTbM/zG+QEIdhv5bAIk+tsP0tcrKbAWlaXBU1JypOe9N2EBCCRI9VaGwsMFmabgtgo6jU8GvT2qxukOgnBDkLclBmBIhNvgBDDmLObZCgmdhA+wgsy3pDuvBJlwmj0sdKCFxyRENPxoowD2jcibxJI79evUoaLMDayLCGRfxkZgiCcbIG+3dPaprT8vs20YijFbAEKQiiTYohs6ZttIx22jBBdHbwOAwh4I4ywZVAmWMtNWYh7+E0LtriZQGnKJYuPnTTaUFiRRsUY98EtzhNmo8Kt,iv:U7tpw+Ce4DuGhnO2RIUjsgpJ6I0j0vmdqI4zk8RoHec=,tag:nyxaLHy20C6uMcxZLVUWiw==,type:str]
+loki_password: ENC[AES256_GCM,data:bVecIUiiFo2Epa0/fo3Js8gHOg==,iv:c5XaFv9Elr+DBT+8X4YLbJvvRPVaq3oED7Jtpxngucg=,tag:ZewOdGXzpuTn99UukoP3eA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -23,8 +24,8 @@ sops:
             eFBKZkQxN1FnblRRekxKa2dFM0NMQ1kK4gaqw9P+R2kYOrKQFLiDzNIJqIqeJLXi
             aek/otKL5wG7Nvj3aMtyl4TrZCXLqZ8Jk9qzXQ9th5j/tklJXTG6uA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-05-08T16:06:07Z"
-    mac: ENC[AES256_GCM,data:CClwWEZr3YW5sVD952FJTMGCqI7/r7cZy/VavKCvjI1nj8zc+sidkhRWcnphi6flnqR+2fo5dbINGI1qldFrbR+N2ncW6llvu6yLk9T2QH/thkDcfYNRnpyt+ykFBpzy83HWxzj416CaXQqR3Cc8Z0Nff8xhVGdZzTTTMfO1eE4=,iv:GFQGJRG+HS0jcsyYTc3dVf22HpQSfRfBCswGHllUqXc=,tag:axJUMtBYSh58izaWc+uTIg==,type:str]
+    lastmodified: "2024-11-11T09:27:30Z"
+    mac: ENC[AES256_GCM,data:dJ4sixRMFeoZl25M+rykO4VVh2K29eECvDNPWl8otg10JusAen8j+i1uCWOus0ny4+6q4g0FEai+9BT1JSrv8y32AzJsz12dm+PSypQFG7RqVdzLC03LxSn0bseoJ6pLTxDfuxq2MsGybxjmkWQrjriotbpPMtvXy/emy7w7F68=,iv:p6HSeUKo4pdOzClY/fBg/g9CqjDyx5wDqMz/RqXloIY=,tag:luY/k6J9tdRxfbUXSzG1eA==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1
diff --git a/hosts/ficolo-common.nix b/hosts/ficolo-common.nix
@@ -1,10 +1,18 @@
 # SPDX-FileCopyrightText: 2022-2024 TII (SSRC) and the Ghaf contributors
 # SPDX-License-Identifier: Apache-2.0
 #
+{ self, lib, ... }:
 {
+  imports = [ self.nixosModules.service-monitoring ];
+
   # Use ci-server as primary DNS and pfsense as secondary
   networking.nameservers = [
     "172.18.20.100"
     "172.18.20.1"
   ];
+
+  services.monitoring = {
+    metrics.openFirewall = true;
+    logs.lokiAddress = lib.mkDefault "http://172.18.20.108";
+  };
 }
diff --git a/hosts/ghaf-log/configuration.nix b/hosts/ghaf-log/configuration.nix
@@ -9,8 +9,6 @@
   ...
 }:
 {
-  sops.defaultSopsFile = ./secrets.yaml;
-
   imports =
     [
       ./disk-config.nix
@@ -35,12 +33,17 @@
       user-vunnyso
     ]);
 
-  sops.secrets = {
-    # basic auth credentials generated with htpasswd
-    loki_basic_auth.owner = "nginx";
-    # github oauth app credentials
-    github_client_id.owner = "grafana";
-    github_client_secret.owner = "grafana";
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets = {
+      # basic auth credentials generated with htpasswd
+      loki_basic_auth.owner = "nginx";
+      # github oauth app credentials
+      github_client_id.owner = "grafana";
+      github_client_secret.owner = "grafana";
+      # vedenemo monitoring
+      vedenemo_loki_password.owner = "promtail";
+    };
   };
 
   nixpkgs.hostPlatform = "x86_64-linux";
@@ -51,14 +54,6 @@
     useDHCP = true;
   };
 
-  # sshified user for monitoring server to log in as
-  users.users.sshified = {
-    isNormalUser = true;
-    openssh.authorizedKeys.keys = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEKd30t0EFmMyULGlecaUX6puIAF4IjynZUo+X9k8h69 monitoring"
-    ];
-  };
-
   boot = {
     # use predictable network interface names (eth0)
     kernelParams = [ "net.ifnames=0" ];
@@ -71,7 +66,17 @@
   # this server has been reinstalled with 24.05
   system.stateVersion = lib.mkForce "24.05";
 
-  services.monitoring.metrics.enable = true;
+  services.monitoring = {
+    metrics = {
+      enable = true;
+      ssh = true;
+    };
+    logs = {
+      enable = true;
+      lokiAddress = "https://monitoring.vedenemo.dev";
+      auth.password_file = config.sops.secrets.vedenemo_loki_password.path;
+    };
+  };
 
   # Grafana
   services.grafana = {

diff --git a/hosts/ghaf-log/secrets.yaml b/hosts/ghaf-log/secrets.yaml
@@ -2,6 +2,7 @@ ssh_host_ed25519_key: ENC[AES256_GCM,data:2nCYQcgjV6SLUZIulp3FcL4rRwXW83RqyozdtN
 loki_basic_auth: ENC[AES256_GCM,data:WjRlJGwp1OYIfZYDd1tvAoW+yxko49NX4onrKY21tWJ20PTiEEfxpXJO,iv:eCXRu7tGyuZheQSz4k94nyQWRr0sndQOS5RPM7a5ZVE=,tag:Kw2WFYN2X5jbJfgL7e6K+A==,type:str]
 github_client_id: ENC[AES256_GCM,data:KhQtPeuMATu122vSb9kUrtICcWI=,iv:+8X1B31h8tqW5nHsKIxsy/Eh49faMfYC0CVcbqyhVes=,tag:7XODRd6NRo5vk5vIdOZ79Q==,type:str]
 github_client_secret: ENC[AES256_GCM,data:AY/1NgKgNOw8SrZb+T1C1vZCK9CV/lMEykY6lTPeycXW4vp4d0putQ==,iv:AqofLZQ1AG/FCHIo3jRDp3Xih1Envq6yugDDHlqD9/o=,tag:PPPdyQZZ1eRDFlXwVJcoYQ==,type:str]
+vedenemo_loki_password: ENC[AES256_GCM,data:8OFLIBdNo2IaB4UiC6N2OljR8g==,iv:ZmKwPGozJOzh3T5gxFUTmis0zsZrXVpcr6D2ZR1hPoU=,tag:ujuncHYbCMq0OYapV1NWhQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -26,8 +27,8 @@ sops:
             RE8rU2lmakJRenVqemhjWUpxeUttRzgKFqd0iVZgbhib0J4eLaCg07xmTJ8uAk6G
             XuEHIrV/3T34BttUo7boc/48caRjfvATYG3JWDotqJNyfDfAerGjgA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-08-15T07:33:54Z"
-    mac: ENC[AES256_GCM,data:Bt7VSB6HdVxh8ziTTMAdSW+eMrK6aS47gxPDAXrXqOTLsmo2Ckg5eWgwG7g+adPZIxGL64o+9BCd7mU7hKFCHMBliv+0LhoAa4H+f+YpIyeX3kh2aaloMTrjXvMokOMptlKj+XuHKKv8x4dyqoAMrGUIg2b8QRXAz1vAIdeEeUo=,iv:A1pFbB18P+lZ0selOAxwP/JSKup4WZchPWBJfI3+ycM=,tag:MzgqrnS3atvpsZg0HXcN8g==,type:str]
+    lastmodified: "2024-11-11T09:29:49Z"
+    mac: ENC[AES256_GCM,data:4ZL4ZHcKU9jV187WKSB6cusvnSilWs2pjxXDUZA8hFx/moAMfJhw7lqfnqvE5pIX5B0nnraKbzPM5Bl2YVZj7GOzQgihH7e/xqFgugSsa4XHKyYvxWjgJFibp7u/sfJTRMhRbegoOg1Pwogk/TyK18T6O6tNVVQDxMkNUU0uUnc=,iv:3hCHPtT+QUh7QCkILed2plrTzYAPtYD2dFks/KQAL9w=,tag:dPKuoSy2YZP8VvuH+saDhA==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1
diff --git a/hosts/himalia/configuration.nix b/hosts/himalia/configuration.nix
@@ -20,15 +20,16 @@
       qemu-common
       ficolo-common
       service-openssh
-      service-nginx
-      service-monitoring
       user-jrautiola
       user-cazfi
       user-karim
       user-barna
       user-mika
     ]);
+
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  networking.hostName = "himalia";
+
   # List packages installed in system profile
   environment.systemPackages = with pkgs; [
     git
@@ -43,33 +44,12 @@
       ]
     ))
   ];
+
   # docker daemon running
   virtualisation.docker.enable = true;
 
-  networking = {
-    hostName = "himalia";
-  };
-
-  security.acme = {
-    acceptTerms = true;
-    defaults.email = "[email protected]";
-  };
-
   services.monitoring = {
     metrics.enable = true;
     logs.enable = true;
   };
-
-  services.nginx = {
-    virtualHosts = {
-      "himalia.vedenemo.dev" = {
-        enableACME = true;
-        forceSSL = true;
-        default = true;
-        locations."/" = {
-          proxyPass = "http://127.0.0.1:3015";
-        };
-      };
-    };
-  };
 }