diff --git a/flake.lock b/flake.lock index 7341bbda..bd3dfaee 100644 --- a/flake.lock +++ b/flake.lock @@ -88,16 +88,16 @@ "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1746509278, - "narHash": "sha256-/pcYYOKgD2+1Zl0jV+3S4h3GSp/3TSfokrqu0rDAijo=", + "lastModified": 1750447588, + "narHash": "sha256-NtklUmPsd5/AmQz4MSRgEtZJPRL2l/tihh0/ErMEZvE=", "owner": "johnalotoski", "repo": "blockperf", - "rev": "cfc8155b9c14fedd569f879f2b3cf7ead91d47be", + "rev": "305f783c4f119bab8ef25a96c9d6809cb48484c5", "type": "github" }, "original": { "owner": "johnalotoski", - "ref": "preview-network", + "ref": "new-tracing", "repo": "blockperf", "type": "github" } @@ -269,6 +269,22 @@ "type": "github" } }, + "cardano-db-sync-service-ng": { + "flake": false, + "locked": { + "lastModified": 1749798397, + "narHash": "sha256-3tzP/6KrFVoFAtguO/Yd7yRPXbldBSjWsrdO1sc9zA0=", + "owner": "IntersectMBO", + "repo": "cardano-db-sync", + "rev": "da764296943a1de65a5085810bf70f55e6bda289", + "type": "github" + }, + "original": { + "owner": "IntersectMBO", + "repo": "cardano-db-sync", + "type": "github" + } + }, "cardano-metadata-service": { "flake": false, "locked": { @@ -316,33 +332,33 @@ "cardano-node-service": { "flake": false, "locked": { - "lastModified": 1746450573, - "narHash": "sha256-Szhph7YncIbQ+/QmqzSW61WLConBa8XIA6K/3A8mP00=", - "owner": "input-output-hk", + "lastModified": 1750466044, + "narHash": "sha256-FO/8R28q9I1ZYjQWA+nIQqrdQQlRRowUK6MaZp/9TOI=", + "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "0983ac29304aadac74a5604eeefa76cfbcc91611", + "rev": "7f1b2c0a79814a87ad8bebbd06db2747d82043ea", "type": "github" }, "original": { - "owner": "input-output-hk", + "owner": "IntersectMBO", + "ref": "jl/new-tracing-service", "repo": "cardano-node", - "rev": "0983ac29304aadac74a5604eeefa76cfbcc91611", "type": "github" } }, "cardano-node-service-ng": { "flake": false, "locked": { - "lastModified": 1750182374, - "narHash": "sha256-fLNUecLWwZTvJuyekr53mb3fcC+tvCu6UBJuJ4vcYHI=", + "lastModified": 1750466044, + "narHash": "sha256-FO/8R28q9I1ZYjQWA+nIQqrdQQlRRowUK6MaZp/9TOI=", "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "7e045ab501e99130a57f363f0964bb4f241c6550", + "rev": "7f1b2c0a79814a87ad8bebbd06db2747d82043ea", "type": "github" }, "original": { "owner": "IntersectMBO", - "ref": "10.5.0", + "ref": "jl/new-tracing-service", "repo": "cardano-node", "type": "github" } @@ -373,11 +389,15 @@ "cardano-db-sync-schema": "cardano-db-sync-schema", "cardano-db-sync-schema-ng": "cardano-db-sync-schema-ng", "cardano-db-sync-service": "cardano-db-sync-service", + "cardano-db-sync-service-ng": "cardano-db-sync-service-ng", "cardano-metadata-service": "cardano-metadata-service", "cardano-node-service": "cardano-node-service", "cardano-node-service-ng": "cardano-node-service-ng", "cardano-ogmios-service": "cardano-ogmios-service", + "cardano-submit-api-service": "cardano-submit-api-service", + "cardano-submit-api-service-ng": "cardano-submit-api-service-ng", "cardano-tracer-service": "cardano-tracer-service", + "cardano-tracer-service-ng": "cardano-tracer-service-ng", "cardano-wallet-service": "cardano-wallet-service", "colmena": "colmena", "flake-parts": "flake-parts", @@ -395,16 +415,16 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1750797952, - "narHash": "sha256-HbtXmIO1MUtdbeCaFbPst0jga1d8JUhhHmhAWbl4V2w=", + "lastModified": 1751054006, + "narHash": "sha256-alwTDrehBP3gfBqt2yHYxdIIdhxaogiD214MdFEWg8Q=", "owner": "input-output-hk", "repo": "cardano-parts", - "rev": "8684614a2bf7f250013579b1ee4ce7633aa1034c", + "rev": "22aa4f481758031dbe63ad16cf3f2fecf2e26fb4", "type": "github" }, "original": { "owner": "input-output-hk", - "ref": "v2025-06-24", + "ref": "jl/new-tracing", "repo": "cardano-parts", "type": "github" } @@ -425,19 +445,70 @@ "type": "github" } }, + "cardano-submit-api-service": { + "flake": false, + "locked": { + "lastModified": 1745954148, + "narHash": "sha256-Moth/2BOCWDdgXrOX6fv2rykqe4zR9POjC+g4dlUqqc=", + "owner": "IntersectMBO", + "repo": "cardano-node", + "rev": "420c94fbb075146c6ec7fba78c5b0482fafe72dd", + "type": "github" + }, + "original": { + "owner": "IntersectMBO", + "ref": "10.4.1", + "repo": "cardano-node", + "type": "github" + } + }, + "cardano-submit-api-service-ng": { + "flake": false, + "locked": { + "lastModified": 1750182374, + "narHash": "sha256-fLNUecLWwZTvJuyekr53mb3fcC+tvCu6UBJuJ4vcYHI=", + "owner": "IntersectMBO", + "repo": "cardano-node", + "rev": "7e045ab501e99130a57f363f0964bb4f241c6550", + "type": "github" + }, + "original": { + "owner": "IntersectMBO", + "ref": "10.5.0", + "repo": "cardano-node", + "type": "github" + } + }, "cardano-tracer-service": { "flake": false, "locked": { - "lastModified": 1717175013, - "narHash": "sha256-b89kXIvJNRwz60fsRtZ7M1islKcVcl2+cSo7IkZu7ko=", + "lastModified": 1750466044, + "narHash": "sha256-FO/8R28q9I1ZYjQWA+nIQqrdQQlRRowUK6MaZp/9TOI=", + "owner": "IntersectMBO", + "repo": "cardano-node", + "rev": "7f1b2c0a79814a87ad8bebbd06db2747d82043ea", + "type": "github" + }, + "original": { + "owner": "IntersectMBO", + "ref": "jl/new-tracing-service", + "repo": "cardano-node", + "type": "github" + } + }, + "cardano-tracer-service-ng": { + "flake": false, + "locked": { + "lastModified": 1750274327, + "narHash": "sha256-eJble5dz62eE0VNmHJhoYJn/xowHSbvlkcJtfwO8QH0=", "owner": "IntersectMBO", "repo": "cardano-node", - "rev": "ebb8be83e16b4e2a62e70bd8575013271e4b1886", + "rev": "7e86a3633f52e160ea3f81721642688903505782", "type": "github" }, "original": { "owner": "IntersectMBO", - "ref": "jl/tracer-service", + "ref": "jl/new-tracing-service", "repo": "cardano-node", "type": "github" } @@ -1008,15 +1079,16 @@ "sodium": "sodium_2" }, "locked": { - "lastModified": 1747796208, - "narHash": "sha256-ZUBNrv+lO0CWhnN0R0ZiG/eIB+W5pnyAGsxf/lozECo=", + "lastModified": 1750118009, + "narHash": "sha256-AgpwV2pcM5844elTb9yhlOlrwF/Jwxm7J2JjJTHQEII=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "281f635b01f8fe843baefe010a694f67041c0848", + "rev": "e7a454189581106a17ff2f1acf990208557bb120", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "jl/new-tracing", "repo": "iohk-nix", "type": "github" } @@ -1029,15 +1101,16 @@ "sodium": "sodium_3" }, "locked": { - "lastModified": 1750025513, - "narHash": "sha256-WUNoYIZvU9moc5ccwJcF22r+bUJXO5dWoRyLPs8bJic=", + "lastModified": 1750118009, + "narHash": "sha256-AgpwV2pcM5844elTb9yhlOlrwF/Jwxm7J2JjJTHQEII=", "owner": "input-output-hk", "repo": "iohk-nix", - "rev": "f63aa2a49720526900fb5943db4123b5b8dcc534", + "rev": "e7a454189581106a17ff2f1acf990208557bb120", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "jl/new-tracing", "repo": "iohk-nix", "type": "github" } @@ -1149,11 +1222,11 @@ "nixpkgs-regression": "nixpkgs-regression_2" }, "locked": { - "lastModified": 1748154947, - "narHash": "sha256-rCpANMHFIlafta6J/G0ILRd+WNSnzv/lzi40Y8f1AR8=", + "lastModified": 1751027644, + "narHash": "sha256-f3ar0uez/+tc+PFtOTyjo7Vfa2xsLxl6K3vZt2Bs4so=", "owner": "nixos", "repo": "nix", - "rev": "d761dad79c79af17aa476a29749bd9d69747548f", + "rev": "8f6c5d088ad49df471168b769ae44f4ea0b915f3", "type": "github" }, "original": { @@ -1451,32 +1524,32 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1684171562, - "narHash": "sha256-BMUWjVWAUdyMWKk0ATMC9H0Bv4qAV/TXwwPUvTiC5IQ=", + "lastModified": 1740683541, + "narHash": "sha256-t1NxrPg8UJCZQ039zSMYB5//XluEdwACTDIxDK31R1I=", "owner": "nixos", "repo": "nixpkgs", - "rev": "55af203d468a6f5032a519cba4f41acf5a74b638", + "rev": "2bc8d3dcc2eb6f7d23590b52042e274eda02318e", "type": "github" }, "original": { "owner": "nixos", - "ref": "release-22.11", + "ref": "release-24.11", "repo": "nixpkgs", "type": "github" } }, "nixpkgs_5": { "locked": { - "lastModified": 1684171562, - "narHash": "sha256-BMUWjVWAUdyMWKk0ATMC9H0Bv4qAV/TXwwPUvTiC5IQ=", + "lastModified": 1740683541, + "narHash": "sha256-t1NxrPg8UJCZQ039zSMYB5//XluEdwACTDIxDK31R1I=", "owner": "nixos", "repo": "nixpkgs", - "rev": "55af203d468a6f5032a519cba4f41acf5a74b638", + "rev": "2bc8d3dcc2eb6f7d23590b52042e274eda02318e", "type": "github" }, "original": { "owner": "nixos", - "ref": "release-22.11", + "ref": "release-24.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index fb4cfbc0..ccc10baf 100644 --- a/flake.nix +++ b/flake.nix @@ -5,11 +5,11 @@ nixpkgs.follows = "cardano-parts/nixpkgs"; nixpkgs-unstable.follows = "cardano-parts/nixpkgs-unstable"; flake-parts.follows = "cardano-parts/flake-parts"; - cardano-parts.url = "github:input-output-hk/cardano-parts/v2025-06-24"; - # cardano-parts.url = "path:/home/jlotoski/work/iohk/cardano-parts-wt/v2025-06-24"; + cardano-parts.url = "github:input-output-hk/cardano-parts/jl/new-tracing"; + # cardano-parts.url = "path:/home/jlotoski/work/iohk/cardano-parts-wt/jl/new-tracing"; # Local pins for additional customization: - # cardanoTest.url = "github:IntersectMBO/cardano-node/mgalazyn/feature/make-peer-snapshot-file-path-relative-to-topology-file"; + # cardanoFix.url = "github:IntersectMBO/cardano-node/new-tracing/fix-metrics"; # PParams api testing cardano-node-pparams-api.url = "github:johnalotoski/cardano-node-pparams-api"; diff --git a/flake/cluster.nix b/flake/cluster.nix index 83227bc7..a6fdb6ab 100644 --- a/flake/cluster.nix +++ b/flake/cluster.nix @@ -63,17 +63,17 @@ with flake.lib; { // optionalAttrs isNg { lib.cardanoLib = flake.config.flake.cardano-parts.pkgs.special.cardanoLibNg; - # Until upstream parts ng has capkgs version, use local flake pins pkgs = { cardano-cli = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-cli-ng); + cardano-db-sync-pkgs = flake.config.flake.cardano-parts.pkgs.special.cardano-db-sync-pkgs-ng; cardano-db-sync = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-db-sync-ng); cardano-db-tool = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-db-tool-ng); - cardano-db-sync-pkgs = flake.config.flake.cardano-parts.pkgs.special.cardano-db-sync-pkgs-ng; cardano-faucet = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-faucet-ng); + cardano-node-pkgs = flake.config.flake.cardano-parts.pkgs.special.cardano-node-pkgs-ng; cardano-node = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-node-ng); cardano-smash = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-smash-ng); cardano-submit-api = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-submit-api-ng); - cardano-node-pkgs = flake.config.flake.cardano-parts.pkgs.special.cardano-node-pkgs-ng; + cardano-tracer = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.cardano-tracer-ng); mithril-client-cli = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.mithril-client-cli-ng); mithril-signer = system: flake.withSystem system ({config, ...}: config.cardano-parts.pkgs.mithril-signer-ng); }; diff --git a/flake/colmena.nix b/flake/colmena.nix index 31fa0a48..b33a3769 100644 --- a/flake/colmena.nix +++ b/flake/colmena.nix @@ -82,8 +82,9 @@ in # Cardano-node modules for group deployment node = { imports = [ - # Base cardano-node service - config.flake.cardano-parts.cluster.groups.default.meta.cardano-node-service-ng + # Base cardano-node and tracer service + config.flake.cardano-parts.cluster.groups.default.meta.cardano-node-service + config.flake.cardano-parts.cluster.groups.default.meta.cardano-tracer-service # Config for cardano-node group deployments inputs.cardano-parts.nixosModules.profile-cardano-node-group @@ -219,6 +220,7 @@ in dbsync = { imports = [ config.flake.cardano-parts.cluster.groups.default.meta.cardano-node-service + config.flake.cardano-parts.cluster.groups.default.meta.cardano-tracer-service config.flake.cardano-parts.cluster.groups.default.meta.cardano-db-sync-service inputs.cardano-parts.nixosModules.profile-cardano-db-sync inputs.cardano-parts.nixosModules.profile-cardano-node-group @@ -235,6 +237,23 @@ in ]; }; + # While new tracing PRs are WIP, the new node service is required + # dbsync-pre = { + # imports = [ + # config.flake.cardano-parts.cluster.groups.default.meta.cardano-node-service-ng + # config.flake.cardano-parts.cluster.groups.default.meta.cardano-tracer-service-ng + # config.flake.cardano-parts.cluster.groups.default.meta.cardano-db-sync-service + # inputs.cardano-parts.nixosModules.profile-cardano-db-sync + # inputs.cardano-parts.nixosModules.profile-cardano-node-group + # inputs.cardano-parts.nixosModules.profile-cardano-custom-metrics + # inputs.cardano-parts.nixosModules.profile-cardano-postgres + # { + # services.cardano-node.shareNodeSocket = true; + # services.cardano-postgres.enablePsqlrc = true; + # } + # ]; + # }; + # ogmios = { # imports = [ # config.flake.cardano-parts.cluster.groups.default.meta.cardano-ogmios-service @@ -257,6 +276,7 @@ in # }; mithrilRelease = {imports = [nixosModules.mithril-release-pin];}; + mithrilSignerDisable = {services.mithril-signer.enable = false;}; dbsyncPub = { pkgs, @@ -361,22 +381,6 @@ in preprodRelMig = mkWorldRelayMig 30000; previewRelMig = mkWorldRelayMig 30002; - # newMetrics = { - # imports = [ - # ( - # # Existing tracer service requires a pkgs with commonLib defined in the cardano-node repo flake overlay. - # # We'll import it through flake-compat so we don't need a full flake input just for obtaining commonLib. - # import - # config.flake.cardano-parts.cluster.groups.default.meta.cardano-tracer-service - # (import - # "${config.flake.cardano-parts.cluster.groups.default.meta.cardano-node-service}/../../default.nix" {system = "x86_64-linux";}) - # .legacyPackages - # .x86_64-linux - # ) - # inputs.cardano-parts.nixosModules.profile-cardano-node-new-tracing - # ]; - # }; - # logRejected = { # services = { # cardano-node.extraNodeConfig = { @@ -449,78 +453,6 @@ in # }; # }; - traceTxs = { - services.cardano-node.extraNodeConfig = { - TraceLocalTxSubmissionProtocol = true; - TraceLocalTxSubmissionServer = true; - TraceTxSubmissionProtocol = true; - TraceTxInbound = true; - TraceTxOutbound = true; - }; - }; - - # maxVerbosity = {services.cardano-node.extraNodeConfig.TracingVerbosity = "MaximalVerbosity";}; - # - # mempoolDisable = { - # services.cardano-node.extraNodeConfig.TraceMempool = false; - # }; - # - # p2p and legacy network debugging code - # netDebug = { - # services.cardano-node = { - # useNewTopology = false; - # extraNodeConfig = { - # TraceMux = true; - # TraceConnectionManagerTransitions = true; - # DebugPeerSelectionInitiator = true; - # DebugPeerSelectionInitiatorResponder = true; - # options.mapSeverity = { - # "cardano.node.DebugPeerSelectionInitiatorResponder" = "Debug"; - # "cardano.node.ChainSyncProtocol" = "Error"; - # "cardano.node.ConnectionManager" = "Debug"; - # "cardano.node.ConnectionManagerTransition" = "Debug"; - # "cardano.node.DebugPeerSelection" = "Debug"; - # "cardano.node.Handshake" = "Debug"; - # "cardano.node.InboundGovernor" = "Debug"; - # "cardano.node.Mux" = "Info"; - # "cardano.node.PeerSelectionActions" = "Debug"; - # "cardano.node.PeerSelection" = "Info"; - # "cardano.node.resources" = "Notice"; - # }; - # }; - # }; - # }; - # - minLog = { - services.cardano-node.extraNodeConfig = { - # Let's make sure we can at least see the blockHeight in logs and metrics - TraceChainDb = true; - - # And then shut everything else off - TraceAcceptPolicy = false; - TraceConnectionManager = false; - TraceDiffusionInitialization = false; - TraceDNSResolver = false; - TraceDNSSubscription = false; - TraceErrorPolicy = false; - TraceForge = false; - TraceHandshake = false; - TraceInboundGovernor = false; - TraceIpSubscription = false; - TraceLedgerPeers = false; - TraceLocalConnectionManager = false; - TraceLocalErrorPolicy = false; - TraceLocalHandshake = false; - TraceLocalRootPeers = false; - TraceMempool = false; - TracePeerSelectionActions = false; - TracePeerSelectionCounters = false; - TracePeerSelection = false; - TracePublicRootPeers = false; - TraceServer = false; - }; - }; - # Optimize tcp sysctl and route params for long distance transmission. # Apply to one relay per pool group. # Ref: https://forum.cardano.org/t/problem-with-increasing-blocksize-or-processing-requirements/140044 @@ -668,8 +600,6 @@ in # ]; # }; # - # gcLogging = {services.cardano-node.extraNodeConfig.options.mapBackends."cardano.node.resources" = ["EKGViewBK" "KatipBK"];}; - # # Example of node pinning to a custom version; see also the relevant flake inputs. # dbsync921 = { # imports = [ @@ -694,6 +624,83 @@ in # cardano-parts.perNode.meta.hostsList = # filter (name: hasPrefix prefix name) (attrNames nixosConfigurations); # }; + + # Legacy tracing system module code samples: + legacyT = {services.cardano-node.useLegacyTracing = true;}; + # + # traceTxs = { + # services.cardano-node.extraNodeConfig = { + # TraceLocalTxSubmissionProtocol = true; + # TraceLocalTxSubmissionServer = true; + # TraceTxSubmissionProtocol = true; + # TraceTxInbound = true; + # TraceTxOutbound = true; + # }; + # }; + # + # maxVerbosity = {services.cardano-node.extraNodeConfig.TracingVerbosity = "MaximalVerbosity";}; + # + # mempoolDisable = { + # services.cardano-node.extraNodeConfig.TraceMempool = false; + # }; + # + # p2p and legacy network debugging code + # netDebug = { + # services.cardano-node = { + # useNewTopology = false; + # extraNodeConfig = { + # TraceMux = true; + # TraceConnectionManagerTransitions = true; + # DebugPeerSelectionInitiator = true; + # DebugPeerSelectionInitiatorResponder = true; + # options.mapSeverity = { + # "cardano.node.DebugPeerSelectionInitiatorResponder" = "Debug"; + # "cardano.node.ChainSyncProtocol" = "Error"; + # "cardano.node.ConnectionManager" = "Debug"; + # "cardano.node.ConnectionManagerTransition" = "Debug"; + # "cardano.node.DebugPeerSelection" = "Debug"; + # "cardano.node.Handshake" = "Debug"; + # "cardano.node.InboundGovernor" = "Debug"; + # "cardano.node.Mux" = "Info"; + # "cardano.node.PeerSelectionActions" = "Debug"; + # "cardano.node.PeerSelection" = "Info"; + # "cardano.node.resources" = "Notice"; + # }; + # }; + # }; + # }; + # + # minLog = { + # services.cardano-node.extraNodeConfig = { + # # Let's make sure we can at least see the blockHeight in logs and metrics + # TraceChainDb = true; + # # And then shut everything else off + # TraceAcceptPolicy = false; + # TraceConnectionManager = false; + # TraceDiffusionInitialization = false; + # TraceDNSResolver = false; + # TraceDNSSubscription = false; + # TraceErrorPolicy = false; + # TraceForge = false; + # TraceHandshake = false; + # TraceInboundGovernor = false; + # TraceIpSubscription = false; + # TraceLedgerPeers = false; + # TraceLocalConnectionManager = false; + # TraceLocalErrorPolicy = false; + # TraceLocalHandshake = false; + # TraceLocalRootPeers = false; + # TraceMempool = false; + # TracePeerSelectionActions = false; + # TracePeerSelectionCounters = false; + # TracePeerSelection = false; + # TracePublicRootPeers = false; + # TraceServer = false; + # }; + # }; + # + # gcLogging = {services.cardano-node.extraNodeConfig.options.mapBackends."cardano.node.resources" = ["EKGViewBK" "KatipBK"];}; + # in { meta = { nixpkgs = import inputs.nixpkgs { @@ -741,11 +748,11 @@ in preprod1-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod1") node rel preprodRelMig mithrilRelay (declMSigner "preprod1-bp-a-1")];}; preprod1-rel-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod1") node rel preprodRelMig];}; preprod1-rel-c-1 = {imports = [us-east-2 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod1") node rel preprodRelMig tcpTxOpt];}; - preprod1-dbsync-a-1 = {imports = [eu-central-1 r6a-xlarge (ebs 200) (group "preprod1") dbsync pre smash preprodSmash];}; + preprod1-dbsync-a-1 = {imports = [eu-central-1 r6a-xlarge (ebs 200) (group "preprod1") dbsync smash preprodSmash];}; preprod1-faucet-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod1") node faucet preprodFaucet];}; - preprod2-bp-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod2") node bp mithrilRelease (declMRel "preprod2-rel-b-1")];}; - preprod2-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod2") node rel preprodRelMig];}; + preprod2-bp-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod2") node bp legacyT mithrilRelease (declMRel "preprod2-rel-b-1")];}; + preprod2-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod2") node rel legacyT preprodRelMig];}; preprod2-rel-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod2") node rel preprodRelMig mithrilRelay (declMSigner "preprod2-bp-b-1")];}; preprod2-rel-c-1 = {imports = [us-east-2 r6a-large (ebs 80) (nodeRamPct 70) (group "preprod2") node rel preprodRelMig tcpTxOpt];}; @@ -760,42 +767,16 @@ in preview1-bp-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node bp mithrilRelease (declMRel "preview1-rel-a-1")];}; # preview1-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node rel maxVerbosity previewRelMig mithrilRelay (declMSigner "preview1-bp-a-1")];}; preview1-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node rel previewRelMig mithrilRelay (declMSigner "preview1-bp-a-1")];}; - preview1-rel-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node minLog rel previewRelMig];}; + preview1-rel-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node rel previewRelMig];}; preview1-rel-c-1 = {imports = [us-east-2 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node rel previewRelMig tcpTxOpt];}; - preview1-dbsync-a-1 = {imports = [eu-central-1 r6a-large (ebs 250) (group "preview1") dbsync pre smash previewSmash];}; + preview1-dbsync-a-1 = {imports = [eu-central-1 r6a-large (ebs 250) (group "preview1") dbsync smash previewSmash];}; preview1-faucet-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview1") node faucet previewFaucet];}; # Smallest d variant for testing - preview1-test-a-1 = { - imports = [ - eu-central-1 - m5ad-large - # i7ie-2xlarge - (ebs 80) - (nodeRamPct 70) - (group "preview1") - node - # pparamsApi - # tcpTxOpt - bp - pre - # config.flake.cardano-parts.cluster.groups.default.meta.cardano-tracer-service-ng - { - services.mithril-signer.enable = false; - # services.cardano-node = { - # useLegacyTracing = false; - # ngTracer = true; - # # profiling = "space-cost"; - # }; - # services.cardano-tracer = { - # # profiling = "space-cost"; - # }; - } - ]; - }; + preview1-test-a-1 = {imports = [eu-central-1 m5ad-large (ebs 80) (nodeRamPct 70) (group "preview1") node bp pre mithrilSignerDisable pparamsApi tcpTxOpt];}; - preview2-bp-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview2") node bp pre mithrilRelease (declMRel "preview2-rel-b-1")];}; - preview2-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview2") node traceTxs rel pre previewRelMig];}; + preview2-bp-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview2") node bp pre legacyT mithrilRelease (declMRel "preview2-rel-b-1")];}; + preview2-rel-a-1 = {imports = [eu-central-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview2") node rel pre legacyT previewRelMig];}; preview2-rel-b-1 = {imports = [eu-west-1 r6a-large (ebs 80) (nodeRamPct 70) (group "preview2") node rel pre previewRelMig mithrilRelay (declMSigner "preview2-bp-b-1")];}; preview2-rel-c-1 = {imports = [us-east-2 r6a-large (ebs 80) (nodeRamPct 70) (group "preview2") node rel pre previewRelMig tcpTxOpt];}; @@ -817,24 +798,12 @@ in # mainnet1-rel-a-1 = {imports = [eu-central-1 m5a-2xlarge (ebs 300) (group "mainnet1") node nodeGhc963 (openFwTcp 3001) bp gcLogging];}; # mainnet1-rel-a-1 = {imports = [eu-central-1 m5a-2xlarge (ebs 300) (group "mainnet1") node nodeGhc963 (openFwTcp 3001)];}; # mainnet1-rel-a-1 = {imports = [eu-central-1 m5a-2xlarge (ebs 300) (group "mainnet1") node (openFwTcp 3001)];}; - mainnet1-rel-a-1 = { - imports = [ - eu-central-1 - r5-xlarge - (ebs 300) - (group "mainnet1") - node - bp - { - services.mithril-signer.enable = false; - } - ]; - }; + mainnet1-rel-a-1 = {imports = [eu-central-1 r5-xlarge (ebs 300) (group "mainnet1") node bp mithrilSignerDisable];}; # Also keep the lmdb and extra debug mainnet node in stopped state for now mainnet1-rel-a-2 = {imports = [eu-central-1 m5ad-large (ebs 300) (group "mainnet1") node lmdb ram8gib (openFwTcp 3001)];}; mainnet1-rel-a-3 = {imports = [eu-central-1 m5ad-large (ebs 300) (group "mainnet1") node lmdb ram8gib (openFwTcp 3001)];}; - mainnet1-rel-a-4 = {imports = [eu-central-1 r5-xlarge (ebs 300) (group "mainnet1") node (openFwTcp 3001)];}; + mainnet1-rel-a-4 = {imports = [eu-central-1 r5-xlarge (ebs 300) (group "mainnet1") node legacyT (openFwTcp 3001)];}; # --------------------------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------------------------- diff --git a/flake/opentofu/grafana/alerts/cardano-node-divergence.nix-import b/flake/opentofu/grafana/alerts/cardano-node-divergence.nix-import index b033bbc2..dbc54c53 100644 --- a/flake/opentofu/grafana/alerts/cardano-node-divergence.nix-import +++ b/flake/opentofu/grafana/alerts/cardano-node-divergence.nix-import @@ -4,26 +4,11 @@ let lagBlocks ? 6, lagSeconds ? 120, thresholdMinutes ? 5, - useLegacyTracing ? true, }: let - blockMetric = - if useLegacyTracing - then "cardano_node_metrics_blockNum_int" - # Some new tracing system metrics have converged with legacy, but changes - # are still in progress, so leave these separated for now in case this - # diverge again before being finalized. - else "cardano_node_metrics_blockNum_int"; - slotMetric = - if useLegacyTracing - then "cardano_node_metrics_slotNum_int" - # See comment above - else "cardano_node_metrics_slotNum_int"; + blockMetric = "cardano_node_metrics_blockNum_int"; + slotMetric = "cardano_node_metrics_slotNum_int"; in { - alert = "cardano_node_block_divergence_${env}${ - if useLegacyTracing - then "" - else "_new_tracing" - }"; + alert = "cardano_node_block_divergence_${env}"; expr = '' ( (abs(max(${blockMetric}{environment="${env}"}) - on() group_right() ${blockMetric}{environment="${env}"}) > bool ${toString lagBlocks}) @@ -44,17 +29,5 @@ in { (mkDivergenceAlert {env = "mainnet";}) (mkDivergenceAlert {env = "preprod";}) (mkDivergenceAlert {env = "preview";}) - (mkDivergenceAlert { - env = "mainnet"; - useLegacyTracing = false; - }) - (mkDivergenceAlert { - env = "preprod"; - useLegacyTracing = false; - }) - (mkDivergenceAlert { - env = "preview"; - useLegacyTracing = false; - }) ]; } diff --git a/flake/opentofu/grafana/alerts/cardano-node-forge.nix-import b/flake/opentofu/grafana/alerts/cardano-node-forge.nix-import index 9fbb10cf..18f88380 100644 --- a/flake/opentofu/grafana/alerts/cardano-node-forge.nix-import +++ b/flake/opentofu/grafana/alerts/cardano-node-forge.nix-import @@ -1,21 +1,10 @@ let mkKesAlert = { periodNotice, - useLegacyTracing ? true, }: let - kesPeriodsRemaining = - if useLegacyTracing - then "cardano_node_metrics_remainingKESPeriods_int" - # Some new tracing system metrics have converged with legacy, but changes - # are still in progress, so leave these separated for now in case this - # diverge again before being finalized. - else "cardano_node_metrics_remainingKESPeriods_int"; + kesPeriodsRemaining = "cardano_node_metrics_remainingKESPeriods_int"; in { - alert = "cardano_node_KES_expiration_metric_${toString periodNotice}period_notice${ - if useLegacyTracing - then "" - else "_new_tracing" - }"; + alert = "cardano_node_KES_expiration_metric_${toString periodNotice}period_notice"; expr = "${kesPeriodsRemaining} <= ${toString periodNotice}"; for = "5m"; labels.severity = "page"; @@ -66,7 +55,7 @@ in { } { alert = "cardano_node_forge_not_adopted_error_new_tracing"; - expr = "sum_over_time(abs((cardano_node_metrics_forgedSlotLast_int - cardano_node_metrics_forgedSlotLast_int == bool 0) - 1)[1h:]) > 0"; + expr = "increase(cardano_node_metrics_Forge_didnt_adopt_counter[1h]) > 0"; for = "1m"; labels.severity = "page"; annotations = { @@ -111,17 +100,5 @@ in { (mkKesAlert {periodNotice = 10;}) (mkKesAlert {periodNotice = 5;}) (mkKesAlert {periodNotice = 1;}) - (mkKesAlert { - periodNotice = 10; - useLegacyTracing = false; - }) - (mkKesAlert { - periodNotice = 5; - useLegacyTracing = false; - }) - (mkKesAlert { - periodNotice = 1; - useLegacyTracing = false; - }) ]; } diff --git a/flake/opentofu/grafana/alerts/cardano-node-network.nix-import b/flake/opentofu/grafana/alerts/cardano-node-network.nix-import index 60788b17..d47ef19e 100644 --- a/flake/opentofu/grafana/alerts/cardano-node-network.nix-import +++ b/flake/opentofu/grafana/alerts/cardano-node-network.nix-import @@ -21,21 +21,20 @@ in { for = "1m"; labels.severity = "page"; annotations = { - summary = "Blocks adoption delay have been above 4.5s for more than 5% of blocks."; - description = "Node average of blocks adoption delay have been above 4.5s for more than 5% of blocks for more than 6 hours."; + summary = "Blocks adoption delay has been above 4.5s for more than 5% of blocks."; + description = "Node average of blocks adoption delay has been above 4.5s for more than 5% of blocks for more than 6 hours."; + }; + } + { + alert = "blocks_adoption_delay_too_high_new_tracing"; + expr = "avg(quantile_over_time(0.95, cardano_node_metrics_blockfetchclient_blockdelay_real[6h])) >= 4.5"; + for = "1m"; + labels.severity = "page"; + annotations = { + summary = "Blocks adoption delay has been above 4.5s for more than 5% of blocks."; + description = "Node average of blocks adoption delay has been above 4.5s for more than 5% of blocks for more than 6 hours."; }; } - # TODO: This metric does not exist in the new tracing system yet - # { - # alert = "blocks_adoption_delay_too_high_new_tracing"; - # expr = "avg(quantile_over_time(0.95, cardano_node_metrics_blockadoption_forgeDelay_real[6h])) >= 4.5"; - # for = "1m"; - # labels.severity = "page"; - # annotations = { - # summary = "Blocks adoption delay have been above 4.5s for more than 5% of blocks."; - # description = "Node average of blocks adoption delay have been above 4.5s for more than 5% of blocks for more than 6 hours."; - # }; - # } # TODO: Static max block size until node publishes max block size metric { alert = "blocks_utilization_too_high"; @@ -48,17 +47,16 @@ in { }; } # TODO: Static max block size until node publishes max block size metric - # TODO: This metric does not exist in the new tracing system yet - # { - # alert = "blocks_utilization_too_high_new_tracing"; - # expr = "100 * avg(avg_over_time(Blockfetch_Client_Blocksize[6h]) / 90112) > ${highBlockUtilization}"; - # for = "5m"; - # labels.severity = "page"; - # annotations = { - # summary = "Block utilization is above ${highBlockUtilization}%."; - # description = "Block utilization has averaged above ${highBlockUtilization}% for more than 6h."; - # }; - # } + { + alert = "blocks_utilization_too_high_new_tracing"; + expr = "100 * avg(avg_over_time(cardano_node_metrics_blockfetchclient_blocksize_int[6h]) / 90112) > ${highBlockUtilization}"; + for = "5m"; + labels.severity = "page"; + annotations = { + summary = "Block utilization is above ${highBlockUtilization}%."; + description = "Block utilization has averaged above ${highBlockUtilization}% for more than 6h."; + }; + } { alert = "cardano_blockfetchclient_blockdelay_high"; expr = ''cardano_node_metrics_blockfetchclient_blockdelay_cdfFive < 0.90''; @@ -69,17 +67,16 @@ in { description = "{{$labels.instance}}: cardano-node has block fetch delays with less than 90% of blocks over a 6 hour window arriving in 5 seconds."; }; } - # TODO: This metric does not exist in the new tracing system yet - # { - # alert = "cardano_blockfetchclient_blockdelay_high_new_tracing"; - # expr = ''Blockfetch_Client_Blockdelay_cdfFive < 0.90''; - # for = "10m"; - # labels.severity = "page"; - # annotations = { - # summary = "{{$labels.instance}}: cardano-node has less than 90% of blocks arriving in 5 seconds."; - # description = "{{$labels.instance}}: cardano-node has block fetch delays with less than 90% of blocks over a 6 hour window arriving in 5 seconds."; - # }; - # } + { + alert = "cardano_blockfetchclient_blockdelay_high_new_tracing"; + expr = ''cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real < 0.90''; + for = "10m"; + labels.severity = "page"; + annotations = { + summary = "{{$labels.instance}}: cardano-node has less than 90% of blocks arriving in 5 seconds."; + description = "{{$labels.instance}}: cardano-node has block fetch delays with less than 90% of blocks over a 6 hour window arriving in 5 seconds."; + }; + } { alert = "cardano_blockfetchclient_blockdelay_critical"; expr = "cardano_node_metrics_blockfetchclient_blockdelay_cdfFive < 0.50"; @@ -90,17 +87,16 @@ in { description = "{{$labels.instance}}: cardano-node has block fetch delays with less than 50% of blocks over a 6 hour window arriving in 5 seconds. This is abnormal even for mempool trace enabled nodes."; }; } - # TODO: This metric does not exist in the new tracing system yet - # { - # alert = "cardano_blockfetchclient_blockdelay_critical_new_tracing"; - # expr = ''Blockfetch_Client_Blockdelay_cdfFive < 0.50''; - # for = "10m"; - # labels.severity = "page"; - # annotations = { - # summary = "{{$labels.instance}}: cardano-node has less than 50% of blocks arriving in 5 seconds."; - # description = "{{$labels.instance}}: cardano-node has block fetch delays with less than 50% of blocks over a 6 hour window arriving in 5 seconds. This is abnormal even for mempool trace enabled nodes."; - # }; - # } + { + alert = "cardano_blockfetchclient_blockdelay_critical_new_tracing"; + expr = ''cardano_node_metrics_blockfetchclient_blockdelay_cdfFive_real < 0.50''; + for = "10m"; + labels.severity = "page"; + annotations = { + summary = "{{$labels.instance}}: cardano-node has less than 50% of blocks arriving in 5 seconds."; + description = "{{$labels.instance}}: cardano-node has block fetch delays with less than 50% of blocks over a 6 hour window arriving in 5 seconds. This is abnormal even for mempool trace enabled nodes."; + }; + } { alert = "cardano_connection_count_high"; expr = ''cardano_node_metrics_connectionManager_incomingConns > 450''; @@ -111,5 +107,15 @@ in { description = "{{$labels.instance}}: cardano-node has an incoming connection count > 450 which is close to the hard limit of 512. Consider scaling or taking other action."; }; } + { + alert = "cardano_connection_count_high_new_tracing"; + expr = ''cardano_node_metrics_connectionManager_inboundConns_int > 450''; + for = "10m"; + labels.severity = "page"; + annotations = { + summary = "{{$labels.instance}}: cardano-node has a high number of connections."; + description = "{{$labels.instance}}: cardano-node has an incoming connection count > 450 which is close to the hard limit of 512. Consider scaling or taking other action."; + }; + } ]; } diff --git a/flake/opentofu/grafana/alerts/cardano-node-quality.nix-import b/flake/opentofu/grafana/alerts/cardano-node-quality.nix-import index 23506bf2..54265101 100644 --- a/flake/opentofu/grafana/alerts/cardano-node-quality.nix-import +++ b/flake/opentofu/grafana/alerts/cardano-node-quality.nix-import @@ -15,16 +15,6 @@ in { description = "Degraded Chain Density: more than 20% of nodes have low chain density (<${chainDensityLow}%) in environment {{$labels.environment}}."; }; } - { - alert = "chain_quality_degraded_new_tracing"; - expr = ''100 * quantile by(environment) (0.2, (cardano_node_metrics_density_real{environment!~"preview"} * 20)) < ${chainDensityLow}''; - for = "5m"; - labels.severity = "page"; - annotations = { - summary = "Degraded Chain Density: more than 20% of nodes have low chain density (<${chainDensityLow}%) in environment {{$labels.environment}}."; - description = "Degraded Chain Density: more than 20% of nodes have low chain density (<${chainDensityLow}%) in environment {{$labels.environment}}."; - }; - } { alert = "chain_quality_degraded_very_low"; expr = ''100 * quantile by(environment) (0.2, (cardano_node_metrics_density_real * 20)) < ${chainDensityVeryLow}''; @@ -35,15 +25,5 @@ in { description = "Degraded Chain Density: more than 20% of nodes have low chain density (<${chainDensityVeryLow}%) in environment {{$labels.environment}}."; }; } - { - alert = "chain_quality_degraded_very_low_new_tracing"; - expr = ''100 * quantile by(environment) (0.2, (cardano_node_metrics_density_real * 20)) < ${chainDensityVeryLow}''; - for = "5m"; - labels.severity = "page"; - annotations = { - summary = "Degraded Chain Density: more than 20% of nodes have low chain density (<${chainDensityVeryLow}%) in environment {{$labels.environment}}."; - description = "Degraded Chain Density: more than 20% of nodes have low chain density (<${chainDensityVeryLow}%) in environment {{$labels.environment}}."; - }; - } ]; } diff --git a/flake/opentofu/grafana/alerts/cardano-node.nix-import b/flake/opentofu/grafana/alerts/cardano-node.nix-import index dce00ee0..0e69eb8e 100644 --- a/flake/opentofu/grafana/alerts/cardano-node.nix-import +++ b/flake/opentofu/grafana/alerts/cardano-node.nix-import @@ -25,16 +25,6 @@ description = "{{$labels.instance}}: cardano-node blockheight unchanged for more than 10 minutes at a 2 minute rate resolution."; }; } - { - alert = "cardano_node_blockheight_unchanged_new_tracing"; - expr = "rate(cardano_node_metrics_blockNum_int[2m]) == 0"; - for = "10m"; - labels.severity = "page"; - annotations = { - summary = "{{$labels.instance}}: cardano-node blockheight unchanged for more than 10 minutes."; - description = "{{$labels.instance}}: cardano-node blockheight unchanged for more than 10 minutes at a 2 minute rate resolution."; - }; - } { alert = "cardano_node_mempool_high"; expr = "cardano_node_metrics_txsInMempool_int > 200"; @@ -45,16 +35,6 @@ description = "{{$labels.instance}}: cardano-node transactions in mempool have been greater than 200 for more than 10 minutes."; }; } - { - alert = "cardano_node_mempool_high_new_tracing"; - expr = "cardano_node_metrics_txsInMempool_int > 200"; - for = "10m"; - labels.severity = "page"; - annotations = { - summary = "{{$labels.instance}}: cardano-node transactions in mempool have been elevated for more than 10 minutes."; - description = "{{$labels.instance}}: cardano-node transactions in mempool have been greater than 200 for more than 10 minutes."; - }; - } # Note: This alert will automatically disable after 6 hours (360 minutes) even if there are still no expected metrics available. { alert = "cardano_node_metric_missing"; @@ -66,16 +46,5 @@ description = "{{$labels.instance}}: cardano-node blockheight metric has gone missing for more than 10 minutes at a 1 minute resolution."; }; } - # Note: This alert will automatically disable after 6 hours (360 minutes) even if there are still no expected metrics available. - { - alert = "cardano_node_metric_missing_new_tracing"; - expr = "(sum_over_time((cardano_node_metrics_blockNum_int != bool 0)[360m:1m]) < bool 350) > 0 unless cardano_node_metrics_blockNum_int"; - for = "1m"; - labels.severity = "page"; - annotations = { - summary = "{{$labels.instance}}: cardano-node blockheight metric has gone missing for more than 10 minutes."; - description = "{{$labels.instance}}: cardano-node blockheight metric has gone missing for more than 10 minutes at a 1 minute resolution."; - }; - } ]; } diff --git a/flake/opentofu/grafana/dashboards/cardano-node-new-tracing.json b/flake/opentofu/grafana/dashboards/cardano-node-new-tracing.json index 23bbc3c8..0fb40484 100644 --- a/flake/opentofu/grafana/dashboards/cardano-node-new-tracing.json +++ b/flake/opentofu/grafana/dashboards/cardano-node-new-tracing.json @@ -540,7 +540,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(cardano_node_metrics_nodeIsNotLeader_int{environment=\"$environment\"}[1h])/rate(cardano_node_metrics_slotNum_int{environment=\"$environment\"}[1h])) / 0.05", + "expr": "sum(rate(cardano_node_metrics_nodeIsLeader_int{environment=\"$environment\"}[1h])/rate(cardano_node_metrics_slotNum_int{environment=\"$environment\"}[1h])) / 0.05", "interval": "", "legendFormat": "Total led slots", "range": true, diff --git a/flake/opentofu/grafana/dashboards/cardano-node-p2p-new-tracing.json b/flake/opentofu/grafana/dashboards/cardano-node-p2p-new-tracing.json index 5e0bc524..b6363184 100644 --- a/flake/opentofu/grafana/dashboards/cardano-node-p2p-new-tracing.json +++ b/flake/opentofu/grafana/dashboards/cardano-node-p2p-new-tracing.json @@ -210,7 +210,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "cardano_node_metrics_peerSelection_hot_int{environment=\"$environment\"}", + "expr": "cardano_node_metrics_peerSelection_Hot_int{environment=\"$environment\"}", "interval": "", "legendFormat": "{{instance}}", "range": true, @@ -504,7 +504,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "cardano_node_metrics_peerSelection_warm_int{environment=\"$environment\"}", + "expr": "cardano_node_metrics_peerSelection_Warm_int{environment=\"$environment\"}", "interval": "", "legendFormat": "{{instance}}", "range": true, @@ -798,7 +798,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "cardano_node_metrics_peerSelection_cold_int{environment=\"$environment\"}", + "expr": "cardano_node_metrics_peerSelection_Cold_int{environment=\"$environment\"}", "interval": "", "legendFormat": "{{instance}}", "range": true, @@ -1192,7 +1192,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "avg(quantile_over_time(0.5,cardano_node_metrics_blockfetchclient_blockdelay_s{environment=\"$environment\"}[15m]))", + "expr": "avg(quantile_over_time(0.5,cardano_node_metrics_blockfetchclient_blockdelay_real{environment=\"$environment\"}[15m]))", "interval": "", "legendFormat": "P50", "range": true, @@ -1204,7 +1204,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "avg(quantile_over_time(0.95,cardano_node_metrics_blockfetchclient_blockdelay_s{environment=\"$environment\"}[15m]))", + "expr": "avg(quantile_over_time(0.95,cardano_node_metrics_blockfetchclient_blockdelay_real{environment=\"$environment\"}[15m]))", "hide": false, "interval": "", "legendFormat": "P95", @@ -1296,13 +1296,13 @@ "targets": [ { "exemplar": true, - "expr": "cardano_node_metrics_connectionManager_prunableConns{environment=\"$environment\"}", + "expr": "cardano_node_metrics_connectionManager_fullDuplexConns_int{environment=\"$environment\"}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], - "title": "Connections relevant for pruning", + "title": "Negotiated full duplex connections", "type": "timeseries" }, { @@ -1491,7 +1491,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "rate(cardano_node_metrics_blockfetchclient_lateblocks{environment=\"$environment\"}[1h])/rate(cardano_node_metrics_blockNum_int{environment=\"$environment\"}[1h])", + "expr": "rate(cardano_node_metrics_blockfetchclient_lateblocks_counter{environment=\"$environment\"}[1h])/rate(cardano_node_metrics_blockNum_int{environment=\"$environment\"}[1h])", "interval": "", "legendFormat": "{{instance}}", "range": true, diff --git a/flake/opentofu/grafana/dashboards/cardano-performance-new-tracing.json b/flake/opentofu/grafana/dashboards/cardano-performance-new-tracing.json index eafcbeba..83b081aa 100644 --- a/flake/opentofu/grafana/dashboards/cardano-performance-new-tracing.json +++ b/flake/opentofu/grafana/dashboards/cardano-performance-new-tracing.json @@ -603,7 +603,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "max(cardano_node_metrics_Mem_resident_intt{environment=\"$environment\",instance=~\".*-bp-.*\"})", + "expr": "max(cardano_node_metrics_Mem_resident_int{environment=\"$environment\",instance=~\".*-bp-.*\"})", "hide": false, "interval": "", "intervalFactor": 1,