From b2aa2c2eae3060978ef1c6a38bd89c0f0bbeb6b2 Mon Sep 17 00:00:00 2001 From: Nico D'Cotta <45274424+cottand@users.noreply.github.com> Date: Fri, 15 Nov 2024 18:20:02 +0000 Subject: [PATCH] document histograms, add doh upstream test (#73) Also fixes #74 --- .github/workflows/test-nix.yml | 13 ++++++++-- api.go | 6 +++-- doc/src/Configuration.md | 1 + doc/src/Prometheus-Metrics.md | 18 +++++++++++-- flake.nix | 1 + nixos-tests/doh-upstream.nix | 46 +++++++++++++++++++++++++++++++++ nixos-tests/systemctl-start.nix | 2 +- 7 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 nixos-tests/doh-upstream.nix diff --git a/.github/workflows/test-nix.yml b/.github/workflows/test-nix.yml index 8f00c62..c8dc22f 100644 --- a/.github/workflows/test-nix.yml +++ b/.github/workflows/test-nix.yml @@ -15,9 +15,18 @@ jobs: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v22 with: - nix_path: nixpkgs=channel:nixos-23.05 + nix_path: nixpkgs=channel:nixos-24.05 github_access_token: ${{ secrets.GITHUB_TOKEN }} - + + - name: Enable KVM group perms (for NixOS tests) + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + sudo apt-get update + sudo apt-get install -y libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils qemu qemu-system-x86 + sudo usermod -a -G kvm,libvirt $USER + - run: nix build . --show-trace - run: nix flake check diff --git a/api.go b/api.go index ed6d6bc..eaa7e06 100644 --- a/api.go +++ b/api.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "time" "github.com/gin-gonic/gin" "gopkg.in/gin-contrib/cors.v1" @@ -48,8 +49,9 @@ func StartAPIServer(config *Config, } server := &http.Server{ - Addr: config.API, - Handler: router, + Addr: config.API, + Handler: router, + ReadHeaderTimeout: time.Duration(config.Timeout) * time.Second, } router.Use(cors.Default()) diff --git a/doc/src/Configuration.md b/doc/src/Configuration.md index d7d33e1..d289477 100644 --- a/doc/src/Configuration.md +++ b/doc/src/Configuration.md @@ -109,6 +109,7 @@ customdnsrecords = [ path = "/metrics" # see https://cottand.github.io/leng/Prometheus-Metrics.html highCardinalityEnabled = false + histogramsEnabled = false resetPeriodMinutes = 60 [DnsOverHttpServer] diff --git a/doc/src/Prometheus-Metrics.md b/doc/src/Prometheus-Metrics.md index be9429b..5e50cda 100644 --- a/doc/src/Prometheus-Metrics.md +++ b/doc/src/Prometheus-Metrics.md @@ -22,7 +22,7 @@ size of the `/metrics` response will grow to be so big the metrics stop being updated. While resetting the counters periodically can help (and you can tweak that with the config `Metrics.resetPeriodMinutes`) -but you might still see issues depending on your traffic. +you might still see issues depending on your traffic. You can read [this SO post](https://stackoverflow.com/questions/46373442/how-dangerous-are-high-cardinality-labels-in-prometheus) to learn more. @@ -38,4 +38,18 @@ with the following config: enabled = true path = "/metrics" highCardinalityEnabled = true -``` \ No newline at end of file +``` + +## Histogram metrics + +Histogram metrics are not unbounded and usually will not be as high-cardinality as the metrics discussed above, +but you should still expect them to have some impact on leng's the memory footprint. + +You can enable them with: + +```toml +[Metrics] +enabled = true +path = "/metrics" +histogramsEnabled = true +``` diff --git a/flake.nix b/flake.nix index ba33034..f0e4c32 100644 --- a/flake.nix +++ b/flake.nix @@ -73,6 +73,7 @@ metrics-api = pkgs.callPackage ./nixos-tests/metrics-api.nix { inherit self; }; systemctl-start = pkgs.callPackage ./nixos-tests/systemctl-start.nix { inherit self; }; custom-dns = pkgs.callPackage ./nixos-tests/custom-dns.nix { inherit self; }; + doh-upstream = pkgs.callPackage ./nixos-tests/doh-upstream.nix { inherit self; }; }; })) diff --git a/nixos-tests/doh-upstream.nix b/nixos-tests/doh-upstream.nix new file mode 100644 index 0000000..040f4ea --- /dev/null +++ b/nixos-tests/doh-upstream.nix @@ -0,0 +1,46 @@ +{ self, pkgs, home-manager, ... }: +let + nixpkgs = self.inputs.nixpkgs; +in +(nixpkgs.lib.nixos.runTest { + hostPkgs = pkgs; + defaults.documentation.enable = false; + node.specialArgs = { inherit self; }; + + name = "leng-custom-dns"; + + nodes = { + server = { config, pkgs, ... }: { + imports = [ self.nixosModules.default ]; + # Open the default port for `postgrest` in the firewall + networking.firewall.allowedUDPPorts = [ 53 ]; + networking.firewall.allowedTCPPorts = [ 80 ]; + + services.leng.enable = true; + services.leng.configuration = { + blocking.sourcesStore = "/tmp"; + customdnsrecords = []; + upstream.DoH = ""; + DnsOverHttpServer.enabled = true; + }; + }; + + client = { pkgs, ... }: { + environment.systemPackages = [ pkgs.dig pkgs.curl ]; + }; + }; + + testScript = + '' + start_all() + + server.wait_for_unit("leng", timeout=10) + server.wait_for_open_port(80, timeout=10) + + client.succeed( + 'curl -vH "accept: application/dns-json" "http://server/dns-query?dns=AAABAAABAAAAAAAAA3d3dwdleGFtcGxlA2NvbQAAAQAB"', + timeout=10, + ) + ''; + +}).config.result diff --git a/nixos-tests/systemctl-start.nix b/nixos-tests/systemctl-start.nix index 3a759c2..cba88c1 100644 --- a/nixos-tests/systemctl-start.nix +++ b/nixos-tests/systemctl-start.nix @@ -15,7 +15,7 @@ in services.leng.enable = true; services.leng.configuration = { - blocking.sourcesStore="/tmp"; + blocking.sourcesStore = "/tmp"; }; };