From 456b0403ea018ce859bfa420dffa0a2f0c7a1268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Tue, 3 Dec 2024 14:24:17 +0100 Subject: [PATCH 1/8] cardano-node: forging_enabled metrics respects non producing node flag --- cardano-node/src/Cardano/Node/Run.hs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs index 3b88274a99e..9e183eb8090 100644 --- a/cardano-node/src/Cardano/Node/Run.hs +++ b/cardano-node/src/Cardano/Node/Run.hs @@ -232,11 +232,11 @@ handleNodeWithTracers cmdPc nc0 p networkMagic blockType runP = do startupInfo <- getStartupInfo nc p fp mapM_ (traceWith $ startupTracer tracers) startupInfo traceNodeStartupInfo (nodeStartupInfoTracer tracers) startupInfo - -- sends initial BlockForgingUpdate blockForging <- snd (Api.protocolInfo runP) + let isNonProducing = ncStartAsNonProducingNode nc traceWith (startupTracer tracers) - (BlockForgingUpdate (if null blockForging + (BlockForgingUpdate (if isNonProducing || null blockForging then DisabledBlockForging else EnabledBlockForging)) @@ -278,10 +278,10 @@ handleNodeWithTracers cmdPc nc0 p networkMagic blockType runP = do >>= mapM_ (traceWith $ startupTracer tracers) traceWith (nodeVersionTracer tracers) getNodeVersion - + let isNonProducing = ncStartAsNonProducingNode nc blockForging <- snd (Api.protocolInfo runP) traceWith (startupTracer tracers) - (BlockForgingUpdate (if null blockForging + (BlockForgingUpdate (if isNonProducing || null blockForging then DisabledBlockForging else EnabledBlockForging)) @@ -724,12 +724,13 @@ updateBlockForging startupTracer blockType nodeKernel nc = do Just Refl -> do -- TODO: check if runP' has changed blockForging <- snd (Api.protocolInfo runP') + let isNonProducing = ncStartAsNonProducingNode nc traceWith startupTracer - (BlockForgingUpdate (bool EnabledBlockForging - DisabledBlockForging - (null blockForging))) - - setBlockForging nodeKernel blockForging + (BlockForgingUpdate (if isNonProducing || null blockForging + then DisabledBlockForging + else EnabledBlockForging)) + unless (ncStartAsNonProducingNode nc) $ + setBlockForging nodeKernel blockForging Nothing -> traceWith startupTracer $ BlockForgingBlockTypeMismatch From 495634a893879bb81c7b375a62393c2e7e335207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Wed, 4 Dec 2024 17:08:30 +0100 Subject: [PATCH 2/8] cardano-node: Add mainnet config test case --- cardano-node/src/Cardano/Node/Run.hs | 2 +- .../Cardano/Tracing/NewTracing/Consistency.hs | 29 +++++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs index 9e183eb8090..4aa90a221c7 100644 --- a/cardano-node/src/Cardano/Node/Run.hs +++ b/cardano-node/src/Cardano/Node/Run.hs @@ -729,7 +729,7 @@ updateBlockForging startupTracer blockType nodeKernel nc = do (BlockForgingUpdate (if isNonProducing || null blockForging then DisabledBlockForging else EnabledBlockForging)) - unless (ncStartAsNonProducingNode nc) $ + unless isNonProducing $ setBlockForging nodeKernel blockForging Nothing -> traceWith startupTracer diff --git a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs index 728aea333dc..95b3a2075d3 100644 --- a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs +++ b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs @@ -16,27 +16,32 @@ tests = H.checkSequential $ H.Group "Configuration Consistency tests" $ test <$> [ ( [] - , "goodConfig.yaml") + , "mainnet-config-new-tracing.json" + , configPrefix) + , ( [] + , "goodConfig.yaml" + , testPrefix) , ( [ "Config namespace error: Illegal namespace ChainDB.CopyToImmutableDBEvent2.CopiedBlockToImmutableDB" , "Config namespace error: Illegal namespace SubscriptionDNS" ] - , "badConfig.yaml") - -- TODO: add mainnet config as good + , "badConfig.yaml" + , testPrefix) ] where - test (actualValue, goldenBaseName) = - (PropertyName goldenBaseName, goldenTestJSON actualValue goldenBaseName) + test (actualValue, goldenBaseName, prefix) = + (PropertyName goldenBaseName, goldenTestJSON actualValue goldenBaseName prefix) -goldenTestJSON :: [Text] -> FilePath -> Property -goldenTestJSON expectedOutcome goldenFileBaseName = +goldenTestJSON :: [Text] -> FilePath -> FilePath -> Property +goldenTestJSON expectedOutcome goldenFileBaseName prefix = H.withTests 1 $ H.withShrinks 0 $ H.property $ do - goldenFp <- H.Base.note $ addPrefix goldenFileBaseName + goldenFp <- H.Base.note $ prefix <> goldenFileBaseName actualValue <- liftIO $ checkNodeTraceConfiguration goldenFp actualValue H.=== expectedOutcome --- | NB: this function is only used in 'goldenTestJSON' but it is defined at the --- top level so that we can refer to it in the documentation of this module. -addPrefix :: FilePath -> FilePath -addPrefix fname = "test/Test/Cardano/Tracing/NewTracing/data/" <> fname +configPrefix :: FilePath +configPrefix = "../configuration/cardano/" + +testPrefix :: FilePath +testPrefix = "test/Test/Cardano/Tracing/NewTracing/data/" From a2d1784e3abeab3a282134aeb4b6af322feab25d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Fri, 6 Dec 2024 13:57:12 +0100 Subject: [PATCH 3/8] cardano-node: Fixes --- .../src/Cardano/Node/TraceConstraints.hs | 1 + .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 62 ++++++++++++++++--- .../Cardano/Tracing/NewTracing/Consistency.hs | 1 + 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/cardano-node/src/Cardano/Node/TraceConstraints.hs b/cardano-node/src/Cardano/Node/TraceConstraints.hs index 1cdea0b56f2..8f29ac2030d 100644 --- a/cardano-node/src/Cardano/Node/TraceConstraints.hs +++ b/cardano-node/src/Cardano/Node/TraceConstraints.hs @@ -44,6 +44,7 @@ type TraceConstraints blk = , HasKESInfo blk , GetKESInfo blk , RunNode blk + , HasIssuer blk , ToObject (ApplyTxErr blk) , ToObject (GenTx blk) diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index adabc99b0d6..17356f9b9de 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -19,6 +19,7 @@ import Cardano.Node.Tracing.Era.Shelley () import Cardano.Node.Tracing.Formatting () import Cardano.Node.Tracing.Render import Cardano.Prelude (maximumDef) +import Cardano.Tracing.HasIssuer import Ouroboros.Consensus.Block import Ouroboros.Consensus.HeaderValidation (HeaderEnvelopeError (..), HeaderError (..), OtherHeaderEnvelopeError) @@ -41,6 +42,7 @@ import Ouroboros.Consensus.Util.Enclose import qualified Ouroboros.Network.AnchoredFragment as AF import Data.Aeson (Value (String), object, toJSON, (.=)) +import qualified Data.ByteString.Base16 as B16 import Data.Int (Int64) import Data.Text (Text) import qualified Data.Text as Text @@ -50,7 +52,7 @@ import Numeric (showFFloat) -- {-# ANN module ("HLint: ignore Redundant bracket" :: Text) #-} --- TODO implement differently so that it uses configuration +-- A limiter that is not coming from configuration, because it carries a special filter withAddedToCurrentChainEmptyLimited :: Trace IO (ChainDB.TraceEvent blk) -> IO (Trace IO (ChainDB.TraceEvent blk)) @@ -79,6 +81,7 @@ instance ( LogFormatting (Header blk) , ConvertRawHash (Header blk) , LedgerSupportsProtocol blk , InspectLedger blk + , HasIssuer blk ) => LogFormatting (ChainDB.TraceEvent blk) where forHuman ChainDB.TraceLastShutdownUnclean = "ChainDB is not clean. Validating all immutable chunks" @@ -394,6 +397,7 @@ instance ( LogFormatting (Header blk) , ConvertRawHash (Header blk) , LedgerSupportsProtocol blk , InspectLedger blk + , HasIssuer blk ) => LogFormatting (ChainDB.TraceAddBlockEvent blk) where forHuman (ChainDB.IgnoreBlockOlderThanK pt) = "Ignoring block older than K: " <> renderRealPointAsPhrase pt @@ -481,7 +485,14 @@ instance ( LogFormatting (Header blk) mconcat [ "kind" .= String "TraceAddBlockEvent.ChangingSelection" , "block" .= forMachine dtal pt ] forMachine dtal (ChainDB.AddedToCurrentChain events selChangedInfo base extended) = - mconcat $ + let ChainInformation { .. } = chainInformation selChangedInfo base extended 0 + tipBlockIssuerVkHashText :: Text + tipBlockIssuerVkHashText = + case tipBlockIssuerVerificationKeyHash of + NoBlockIssuer -> "NoBlockIssuer" + BlockIssuerVerificationKeyHash bs -> + Text.decodeLatin1 (B16.encode bs) + in mconcat $ [ "kind" .= String "AddedToCurrentChain" , "newtip" .= renderPointForDetails dtal (AF.headPoint extended) , "newTipSelectView" .= forMachine dtal (ChainDB.newTipSelectView selChangedInfo) @@ -493,8 +504,18 @@ instance ( LogFormatting (Header blk) | dtal == DDetailed ] ++ [ "events" .= toJSON (map (forMachine dtal) events) | not (null events) ] + ++ [ "tipBlockHash" .= tipBlockHash] + ++ [ "tipBlockParentHash" .= tipBlockParentHash] + ++ [ "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] forMachine dtal (ChainDB.SwitchedToAFork events selChangedInfo old new) = - mconcat $ + let ChainInformation { .. } = chainInformation selChangedInfo old new 0 + tipBlockIssuerVkHashText :: Text + tipBlockIssuerVkHashText = + case tipBlockIssuerVerificationKeyHash of + NoBlockIssuer -> "NoBlockIssuer" + BlockIssuerVerificationKeyHash bs -> + Text.decodeLatin1 (B16.encode bs) + in mconcat $ [ "kind" .= String "TraceAddBlockEvent.SwitchedToAFork" , "newtip" .= renderPointForDetails dtal (AF.headPoint new) , "newTipSelectView" .= forMachine dtal (ChainDB.newTipSelectView selChangedInfo) @@ -506,6 +527,10 @@ instance ( LogFormatting (Header blk) | dtal == DDetailed ] ++ [ "events" .= toJSON (map (forMachine dtal) events) | not (null events) ] + ++ [ "tipBlockHash" .= tipBlockHash] + ++ [ "tipBlockParentHash" .= tipBlockParentHash] + ++ [ "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + forMachine dtal (ChainDB.AddBlockValidation ev') = forMachine dtal ev' forMachine dtal (ChainDB.AddedBlockToVolatileDB pt (BlockNo bn) _ enclosing) = @@ -544,7 +569,7 @@ instance ( LogFormatting (Header blk) asMetrics (ChainDB.SwitchedToAFork _warnings selChangedInfo oldChain newChain) = let forkIt = not $ AF.withinFragmentBounds (AF.headPoint oldChain) newChain - ChainInformation { .. } = chainInformation selChangedInfo newChain 0 + ChainInformation { .. } = chainInformation selChangedInfo oldChain newChain 0 in [ DoubleM "density" (fromRational density) , IntM "slotNum" (fromIntegral slots) , IntM "blockNum" (fromIntegral blocks) @@ -552,9 +577,9 @@ instance ( LogFormatting (Header blk) , IntM "epoch" (fromIntegral (unEpochNo epoch)) , CounterM "forks" (Just (if forkIt then 1 else 0)) ] - asMetrics (ChainDB.AddedToCurrentChain _warnings selChangedInfo _oldChain newChain) = + asMetrics (ChainDB.AddedToCurrentChain _warnings selChangedInfo oldChain newChain) = let ChainInformation { .. } = - chainInformation selChangedInfo newChain 0 + chainInformation selChangedInfo oldChain newChain 0 in [ DoubleM "density" (fromRational density) , IntM "slotNum" (fromIntegral slots) , IntM "blockNum" (fromIntegral blocks) @@ -1488,7 +1513,6 @@ instance MetaTrace (ChainDB.UnknownRange blk) where namespaceFor ChainDB.MissingBlock {} = Namespace [] ["MissingBlock"] namespaceFor ChainDB.ForkTooOld {} = Namespace [] ["ForkTooOld"] - -- TODO Tracers Is this really as intended? severityFor _ _ = Just Debug documentFor (Namespace _ ["MissingBlock"]) = Just @@ -2097,22 +2121,44 @@ data ChainInformation = ChainInformation -- ^ Relative slot number of the tip of the current chain within the -- epoch. , blocksUncoupledDelta :: Int64 + , tipBlockHash :: Text + -- ^ Hash of the last adopted block. + , tipBlockParentHash :: Text + -- ^ Hash of the parent block of the last adopted block. + , tipBlockIssuerVerificationKeyHash :: BlockIssuerVerificationKeyHash + -- ^ Hash of the last adopted block issuer's verification key. } + chainInformation :: forall blk. HasHeader (Header blk) + => HasIssuer blk + => ConvertRawHash blk => ChainDB.SelectionChangedInfo blk -> AF.AnchoredFragment (Header blk) + -> AF.AnchoredFragment (Header blk) -- ^ New fragment. -> Int64 -> ChainInformation -chainInformation selChangedInfo frag blocksUncoupledDelta = ChainInformation +chainInformation selChangedInfo oldFrag frag blocksUncoupledDelta = ChainInformation { slots = unSlotNo $ fromWithOrigin 0 (AF.headSlot frag) , blocks = unBlockNo $ fromWithOrigin (BlockNo 1) (AF.headBlockNo frag) , density = fragmentChainDensity frag , epoch = ChainDB.newTipEpoch selChangedInfo , slotInEpoch = ChainDB.newTipSlotInEpoch selChangedInfo , blocksUncoupledDelta = blocksUncoupledDelta + , tipBlockHash = renderHeaderHash (Proxy @blk) $ realPointHash (ChainDB.newTipPoint selChangedInfo) + , tipBlockParentHash = renderChainHash (Text.decodeLatin1 . B16.encode . toRawHash (Proxy @blk)) $ AF.headHash oldFrag + , tipBlockIssuerVerificationKeyHash = tipIssuerVkHash } + where + tipIssuerVkHash :: BlockIssuerVerificationKeyHash + tipIssuerVkHash = + case AF.head frag of + Left AF.AnchorGenesis -> + NoBlockIssuer + Left (AF.Anchor _s _h _b) -> + NoBlockIssuer + Right blk -> getIssuerVerificationKeyHash blk fragmentChainDensity :: HasHeader (Header blk) diff --git a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs index 95b3a2075d3..34c0ae7d460 100644 --- a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs +++ b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs @@ -16,6 +16,7 @@ tests = H.checkSequential $ H.Group "Configuration Consistency tests" $ test <$> [ ( [] + -- This file name shoud reference the current standard config with new tracing , "mainnet-config-new-tracing.json" , configPrefix) , ( [] From 4a2ff748af81761433b5bee2a2f26c8ab44c7ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Mon, 9 Dec 2024 19:04:58 +0100 Subject: [PATCH 4/8] Review changes --- cardano-node/src/Cardano/Node/Run.hs | 3 +- .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 20 +++--- .../Cardano/Tracing/NewTracing/Consistency.hs | 61 +++++++++++-------- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Run.hs b/cardano-node/src/Cardano/Node/Run.hs index 4aa90a221c7..c383befb163 100644 --- a/cardano-node/src/Cardano/Node/Run.hs +++ b/cardano-node/src/Cardano/Node/Run.hs @@ -729,8 +729,7 @@ updateBlockForging startupTracer blockType nodeKernel nc = do (BlockForgingUpdate (if isNonProducing || null blockForging then DisabledBlockForging else EnabledBlockForging)) - unless isNonProducing $ - setBlockForging nodeKernel blockForging + setBlockForging nodeKernel blockForging Nothing -> traceWith startupTracer $ BlockForgingBlockTypeMismatch diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 17356f9b9de..27f3f666137 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -504,9 +504,9 @@ instance ( LogFormatting (Header blk) | dtal == DDetailed ] ++ [ "events" .= toJSON (map (forMachine dtal) events) | not (null events) ] - ++ [ "tipBlockHash" .= tipBlockHash] - ++ [ "tipBlockParentHash" .= tipBlockParentHash] - ++ [ "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + ++ [ "tipBlockHash" .= tipBlockHash + , "tipBlockParentHash" .= tipBlockParentHash + , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] forMachine dtal (ChainDB.SwitchedToAFork events selChangedInfo old new) = let ChainInformation { .. } = chainInformation selChangedInfo old new 0 tipBlockIssuerVkHashText :: Text @@ -527,9 +527,9 @@ instance ( LogFormatting (Header blk) | dtal == DDetailed ] ++ [ "events" .= toJSON (map (forMachine dtal) events) | not (null events) ] - ++ [ "tipBlockHash" .= tipBlockHash] - ++ [ "tipBlockParentHash" .= tipBlockParentHash] - ++ [ "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + ++ [ "tipBlockHash" .= tipBlockHash + , "tipBlockParentHash" .= tipBlockParentHash + , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] forMachine dtal (ChainDB.AddBlockValidation ev') = forMachine dtal ev' @@ -2152,13 +2152,7 @@ chainInformation selChangedInfo oldFrag frag blocksUncoupledDelta = ChainInforma } where tipIssuerVkHash :: BlockIssuerVerificationKeyHash - tipIssuerVkHash = - case AF.head frag of - Left AF.AnchorGenesis -> - NoBlockIssuer - Left (AF.Anchor _s _h _b) -> - NoBlockIssuer - Right blk -> getIssuerVerificationKeyHash blk + tipIssuerVkHash = either (const NoBlockIssuer) getIssuerVerificationKeyHash (AF.head frag) fragmentChainDensity :: HasHeader (Header blk) diff --git a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs index 34c0ae7d460..94d94e419be 100644 --- a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs +++ b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs @@ -1,48 +1,59 @@ +{-# LANGUAGE FlexibleContexts #-} + + -- | Check namespace consistencies agains configurations module Test.Cardano.Tracing.NewTracing.Consistency (tests) where import Cardano.Node.Tracing.Consistency (checkNodeTraceConfiguration) -import Control.Monad.IO.Class (liftIO) +import Control.Monad.IO.Class (MonadIO, liftIO) import Data.Text +import System.Directory (canonicalizePath) +import System.FilePath (()) import Hedgehog (Property) import qualified Hedgehog as H -import qualified Hedgehog.Extras.Test.Base as H.Base +import qualified Hedgehog.Extras.Test.Base as H +import qualified Hedgehog.Extras.Test.Process as H import Hedgehog.Internal.Property (PropertyName (PropertyName)) -tests :: IO Bool -tests = H.checkSequential +tests :: MonadIO m => m Bool +tests = do + H.checkSequential $ H.Group "Configuration Consistency tests" - $ test - <$> [ ( [] - -- This file name shoud reference the current standard config with new tracing - , "mainnet-config-new-tracing.json" - , configPrefix) - , ( [] - , "goodConfig.yaml" - , testPrefix) - , ( [ "Config namespace error: Illegal namespace ChainDB.CopyToImmutableDBEvent2.CopiedBlockToImmutableDB" - , "Config namespace error: Illegal namespace SubscriptionDNS" - ] - , "badConfig.yaml" - , testPrefix) - ] + $ Prelude.map test + [ ( [] + -- This file name shoud reference the current standard config with new tracing + , "mainnet-config-new-tracing.json" + , configPrefix) + , ( [] + , "goodConfig.yaml" + , testPrefix) + , ( [ "Config namespace error: Illegal namespace ChainDB.CopyToImmutableDBEvent2.CopiedBlockToImmutableDB" + , "Config namespace error: Illegal namespace SubscriptionDNS" + ] + , "badConfig.yaml" + , testPrefix) + ] where test (actualValue, goldenBaseName, prefix) = (PropertyName goldenBaseName, goldenTestJSON actualValue goldenBaseName prefix) -goldenTestJSON :: [Text] -> FilePath -> FilePath -> Property -goldenTestJSON expectedOutcome goldenFileBaseName prefix = +goldenTestJSON :: [Text] -> FilePath -> (FilePath -> IO FilePath) -> Property +goldenTestJSON expectedOutcome goldenFileBaseName prefixFunc = H.withTests 1 $ H.withShrinks 0 $ H.property $ do - goldenFp <- H.Base.note $ prefix <> goldenFileBaseName + basePath <- H.getProjectBase + prefixPath <- liftIO $ prefixFunc basePath + goldenFp <- H.note $ prefixPath goldenFileBaseName actualValue <- liftIO $ checkNodeTraceConfiguration goldenFp actualValue H.=== expectedOutcome -configPrefix :: FilePath -configPrefix = "../configuration/cardano/" +configPrefix :: FilePath -> IO FilePath +configPrefix projectBase = do + base <- canonicalizePath projectBase + return $ base "configuration/cardano" -testPrefix :: FilePath -testPrefix = "test/Test/Cardano/Tracing/NewTracing/data/" +testPrefix :: FilePath -> IO FilePath +testPrefix _ = pure "test/Test/Cardano/Tracing/NewTracing/data/" From 437f2db2fbfe0161012b501059dbb5dbfa315d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Thu, 12 Dec 2024 12:48:44 +0100 Subject: [PATCH 5/8] cardano-node: tipBlockHash metrics and tracing properties --- .../src/Cardano/Node/TraceConstraints.hs | 1 - .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 64 +++++++++++++++---- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/cardano-node/src/Cardano/Node/TraceConstraints.hs b/cardano-node/src/Cardano/Node/TraceConstraints.hs index 8f29ac2030d..1cdea0b56f2 100644 --- a/cardano-node/src/Cardano/Node/TraceConstraints.hs +++ b/cardano-node/src/Cardano/Node/TraceConstraints.hs @@ -44,7 +44,6 @@ type TraceConstraints blk = , HasKESInfo blk , GetKESInfo blk , RunNode blk - , HasIssuer blk , ToObject (ApplyTxErr blk) , ToObject (GenTx blk) diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 27f3f666137..5e2a8bd9d4b 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -484,7 +484,8 @@ instance ( LogFormatting (Header blk) forMachine dtal (ChainDB.ChangingSelection pt) = mconcat [ "kind" .= String "TraceAddBlockEvent.ChangingSelection" , "block" .= forMachine dtal pt ] - forMachine dtal (ChainDB.AddedToCurrentChain events selChangedInfo base extended) = + + forMachine DDetailed (ChainDB.AddedToCurrentChain events selChangedInfo base extended) = let ChainInformation { .. } = chainInformation selChangedInfo base extended 0 tipBlockIssuerVkHashText :: Text tipBlockIssuerVkHashText = @@ -493,6 +494,22 @@ instance ( LogFormatting (Header blk) BlockIssuerVerificationKeyHash bs -> Text.decodeLatin1 (B16.encode bs) in mconcat $ + [ "kind" .= String "AddedToCurrentChain" + , "newtip" .= renderPointForDetails DDetailed (AF.headPoint extended) + , "newTipSelectView" .= forMachine DDetailed (ChainDB.newTipSelectView selChangedInfo) + ] + ++ [ "oldTipSelectView" .= forMachine DDetailed oldTipSelectView + | Just oldTipSelectView <- [ChainDB.oldTipSelectView selChangedInfo] + ] + ++ [ "headers" .= toJSON (forMachine DDetailed `map` addedHdrsNewChain base extended) + ] + ++ [ "events" .= toJSON (map (forMachine DDetailed) events) + | not (null events) ] + ++ [ "tipBlockHash" .= tipBlockHash + , "tipBlockParentHash" .= tipBlockParentHash + , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + forMachine dtal (ChainDB.AddedToCurrentChain events selChangedInfo _base extended) = + mconcat $ [ "kind" .= String "AddedToCurrentChain" , "newtip" .= renderPointForDetails dtal (AF.headPoint extended) , "newTipSelectView" .= forMachine dtal (ChainDB.newTipSelectView selChangedInfo) @@ -500,14 +517,10 @@ instance ( LogFormatting (Header blk) ++ [ "oldTipSelectView" .= forMachine dtal oldTipSelectView | Just oldTipSelectView <- [ChainDB.oldTipSelectView selChangedInfo] ] - ++ [ "headers" .= toJSON (forMachine dtal `map` addedHdrsNewChain base extended) - | dtal == DDetailed ] ++ [ "events" .= toJSON (map (forMachine dtal) events) | not (null events) ] - ++ [ "tipBlockHash" .= tipBlockHash - , "tipBlockParentHash" .= tipBlockParentHash - , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] - forMachine dtal (ChainDB.SwitchedToAFork events selChangedInfo old new) = + + forMachine DDetailed (ChainDB.SwitchedToAFork events selChangedInfo old new) = let ChainInformation { .. } = chainInformation selChangedInfo old new 0 tipBlockIssuerVkHashText :: Text tipBlockIssuerVkHashText = @@ -516,6 +529,22 @@ instance ( LogFormatting (Header blk) BlockIssuerVerificationKeyHash bs -> Text.decodeLatin1 (B16.encode bs) in mconcat $ + [ "kind" .= String "TraceAddBlockEvent.SwitchedToAFork" + , "newtip" .= renderPointForDetails DDetailed (AF.headPoint new) + , "newTipSelectView" .= forMachine DDetailed (ChainDB.newTipSelectView selChangedInfo) + ] + ++ [ "oldTipSelectView" .= forMachine DDetailed oldTipSelectView + | Just oldTipSelectView <- [ChainDB.oldTipSelectView selChangedInfo] + ] + ++ [ "headers" .= toJSON (forMachine DDetailed `map` addedHdrsNewChain old new) + ] + ++ [ "events" .= toJSON (map (forMachine DDetailed) events) + | not (null events) ] + ++ [ "tipBlockHash" .= tipBlockHash + , "tipBlockParentHash" .= tipBlockParentHash + , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + forMachine dtal (ChainDB.SwitchedToAFork events selChangedInfo _old new) = + mconcat $ [ "kind" .= String "TraceAddBlockEvent.SwitchedToAFork" , "newtip" .= renderPointForDetails dtal (AF.headPoint new) , "newTipSelectView" .= forMachine dtal (ChainDB.newTipSelectView selChangedInfo) @@ -523,13 +552,8 @@ instance ( LogFormatting (Header blk) ++ [ "oldTipSelectView" .= forMachine dtal oldTipSelectView | Just oldTipSelectView <- [ChainDB.oldTipSelectView selChangedInfo] ] - ++ [ "headers" .= toJSON (forMachine dtal `map` addedHdrsNewChain old new) - | dtal == DDetailed ] ++ [ "events" .= toJSON (map (forMachine dtal) events) | not (null events) ] - ++ [ "tipBlockHash" .= tipBlockHash - , "tipBlockParentHash" .= tipBlockParentHash - , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] forMachine dtal (ChainDB.AddBlockValidation ev') = forMachine dtal ev' @@ -570,21 +594,37 @@ instance ( LogFormatting (Header blk) let forkIt = not $ AF.withinFragmentBounds (AF.headPoint oldChain) newChain ChainInformation { .. } = chainInformation selChangedInfo oldChain newChain 0 + tipBlockIssuerVkHashText = + case tipBlockIssuerVerificationKeyHash of + NoBlockIssuer -> "NoBlockIssuer" + BlockIssuerVerificationKeyHash bs -> + Text.decodeLatin1 (B16.encode bs) in [ DoubleM "density" (fromRational density) , IntM "slotNum" (fromIntegral slots) , IntM "blockNum" (fromIntegral blocks) , IntM "slotInEpoch" (fromIntegral slotInEpoch) , IntM "epoch" (fromIntegral (unEpochNo epoch)) , CounterM "forks" (Just (if forkIt then 1 else 0)) + , PrometheusM "tipBlock" [("hash",tipBlockHash) + ,("parent hash",tipBlockParentHash) + ,("issuer verification key hash", tipBlockIssuerVkHashText)] ] asMetrics (ChainDB.AddedToCurrentChain _warnings selChangedInfo oldChain newChain) = let ChainInformation { .. } = chainInformation selChangedInfo oldChain newChain 0 + tipBlockIssuerVkHashText = + case tipBlockIssuerVerificationKeyHash of + NoBlockIssuer -> "NoBlockIssuer" + BlockIssuerVerificationKeyHash bs -> + Text.decodeLatin1 (B16.encode bs) in [ DoubleM "density" (fromRational density) , IntM "slotNum" (fromIntegral slots) , IntM "blockNum" (fromIntegral blocks) , IntM "slotInEpoch" (fromIntegral slotInEpoch) , IntM "epoch" (fromIntegral (unEpochNo epoch)) + , PrometheusM "tipBlock" [("hash",tipBlockHash) + ,("parent hash",tipBlockParentHash) + ,("issuer verification key hash", tipBlockIssuerVkHashText)] ] asMetrics _ = [] From 2146f13aad63f1b20b6f60795787c604814b42a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Tue, 10 Dec 2024 18:16:18 +0100 Subject: [PATCH 6/8] cardano-node | tests: internal/external subdir resolution for new tracing --- .../Cardano/Tracing/NewTracing/Consistency.hs | 60 +++++++++++-------- configuration/cardano/update-config-files.sh | 1 + nix/haskell.nix | 1 + 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs index 94d94e419be..7ff1d998edd 100644 --- a/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs +++ b/cardano-node/test/Test/Cardano/Tracing/NewTracing/Consistency.hs @@ -6,9 +6,9 @@ module Test.Cardano.Tracing.NewTracing.Consistency (tests) where import Cardano.Node.Tracing.Consistency (checkNodeTraceConfiguration) -import Control.Monad.IO.Class (MonadIO, liftIO) +import Control.Monad.IO.Class (MonadIO) import Data.Text -import System.Directory (canonicalizePath) +import qualified System.Directory as IO import System.FilePath (()) import Hedgehog (Property) @@ -22,38 +22,46 @@ tests = do H.checkSequential $ H.Group "Configuration Consistency tests" $ Prelude.map test - [ ( [] - -- This file name shoud reference the current standard config with new tracing + [ ( [] + -- This file name should reference the current standard config with new tracing + , configSubdir , "mainnet-config-new-tracing.json" - , configPrefix) - , ( [] + ) + , + ( [] + , testSubdir , "goodConfig.yaml" - , testPrefix) + ) , ( [ "Config namespace error: Illegal namespace ChainDB.CopyToImmutableDBEvent2.CopiedBlockToImmutableDB" , "Config namespace error: Illegal namespace SubscriptionDNS" ] + , testSubdir , "badConfig.yaml" - , testPrefix) + ) ] where - test (actualValue, goldenBaseName, prefix) = - (PropertyName goldenBaseName, goldenTestJSON actualValue goldenBaseName prefix) - - -goldenTestJSON :: [Text] -> FilePath -> (FilePath -> IO FilePath) -> Property -goldenTestJSON expectedOutcome goldenFileBaseName prefixFunc = - H.withTests 1 $ H.withShrinks 0 $ H.property $ do - basePath <- H.getProjectBase - prefixPath <- liftIO $ prefixFunc basePath - goldenFp <- H.note $ prefixPath goldenFileBaseName - actualValue <- liftIO $ checkNodeTraceConfiguration goldenFp - actualValue H.=== expectedOutcome + test (actualValue, subDir, goldenBaseName) = + (PropertyName goldenBaseName, goldenTestJSON subDir actualValue goldenBaseName) +goldenTestJSON :: SubdirSelection -> [Text] -> FilePath -> Property +goldenTestJSON subDir expectedOutcome goldenFileBaseName = + H.withTests 1 $ H.withShrinks 0 $ H.property $ do + base <- resolveDir + goldenFp <- H.note $ base goldenFileBaseName + actualValue <- H.evalIO $ checkNodeTraceConfiguration goldenFp + actualValue H.=== expectedOutcome + where + resolveDir = case subDir of + ExternalSubdir d -> do + base <- H.evalIO . IO.canonicalizePath =<< H.getProjectBase + pure $ base d + InternalSubdir d -> + pure d -configPrefix :: FilePath -> IO FilePath -configPrefix projectBase = do - base <- canonicalizePath projectBase - return $ base "configuration/cardano" +data SubdirSelection = + InternalSubdir FilePath + | ExternalSubdir FilePath -testPrefix :: FilePath -> IO FilePath -testPrefix _ = pure "test/Test/Cardano/Tracing/NewTracing/data/" +testSubdir, configSubdir :: SubdirSelection +testSubdir = InternalSubdir "test/Test/Cardano/Tracing/NewTracing/data" +configSubdir = ExternalSubdir $ "configuration" "cardano" diff --git a/configuration/cardano/update-config-files.sh b/configuration/cardano/update-config-files.sh index dcb306f3790..a702143ed26 100755 --- a/configuration/cardano/update-config-files.sh +++ b/configuration/cardano/update-config-files.sh @@ -20,6 +20,7 @@ copyFile "mainnet-alonzo-genesis.json" copyFile "mainnet-byron-genesis.json" copyFile "mainnet-conway-genesis.json" copyFile "mainnet-config.json" +copyFile "mainnet-config-new-tracing.json" copyFile "mainnet-shelley-genesis.json" copyFile "mainnet-topology.json" diff --git a/nix/haskell.nix b/nix/haskell.nix index 3a0975211f8..71ee8f37b40 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -194,6 +194,7 @@ let mainnetConfigFiles = [ "configuration/cardano/mainnet-config.yaml" "configuration/cardano/mainnet-config.json" + "configuration/cardano/mainnet-config-new-tracing.json" "configuration/cardano/mainnet-byron-genesis.json" "configuration/cardano/mainnet-shelley-genesis.json" "configuration/cardano/mainnet-alonzo-genesis.json" From afaf32e1be0890cc353666ebac0b43cbc6a00a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Fri, 13 Dec 2024 15:49:38 +0100 Subject: [PATCH 7/8] cardano-node: metrics comments --- .../Cardano/Node/Tracing/Tracers/ChainDB.hs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 5e2a8bd9d4b..2c655353df2 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -507,7 +507,7 @@ instance ( LogFormatting (Header blk) | not (null events) ] ++ [ "tipBlockHash" .= tipBlockHash , "tipBlockParentHash" .= tipBlockParentHash - , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + , "tipBlockIssuerVKeyHash" .= tipBlockIssuerVkHashText] forMachine dtal (ChainDB.AddedToCurrentChain events selChangedInfo _base extended) = mconcat $ [ "kind" .= String "AddedToCurrentChain" @@ -542,7 +542,7 @@ instance ( LogFormatting (Header blk) | not (null events) ] ++ [ "tipBlockHash" .= tipBlockHash , "tipBlockParentHash" .= tipBlockParentHash - , "tipBlockIssuerVerificationKeyHash" .= tipBlockIssuerVkHashText] + , "tipBlockIssuerVKeyHash" .= tipBlockIssuerVkHashText] forMachine dtal (ChainDB.SwitchedToAFork events selChangedInfo _old new) = mconcat $ [ "kind" .= String "TraceAddBlockEvent.SwitchedToAFork" @@ -606,8 +606,8 @@ instance ( LogFormatting (Header blk) , IntM "epoch" (fromIntegral (unEpochNo epoch)) , CounterM "forks" (Just (if forkIt then 1 else 0)) , PrometheusM "tipBlock" [("hash",tipBlockHash) - ,("parent hash",tipBlockParentHash) - ,("issuer verification key hash", tipBlockIssuerVkHashText)] + ,("parent_hash",tipBlockParentHash) + ,("issuer_VKey_hash", tipBlockIssuerVkHashText)] ] asMetrics (ChainDB.AddedToCurrentChain _warnings selChangedInfo oldChain newChain) = let ChainInformation { .. } = @@ -745,7 +745,14 @@ instance MetaTrace (ChainDB.TraceAddBlockEvent blk) where , ( "epoch" , "In which epoch is the tip of the current chain." ) + , ( "forks" + , "counter for forks" + ) + , ( "tipBlock" + , "Hash vallues for tipBlockHash, tipBlockParentHash and tipBlockIssuerVkHashText" + ) ] + metricsDocFor (Namespace _ ["AddedToCurrentChain"]) = [ ( "density" , mconcat @@ -768,6 +775,9 @@ instance MetaTrace (ChainDB.TraceAddBlockEvent blk) where , ( "epoch" , "In which epoch is the tip of the current chain." ) + , ( "tipBlock" + , "Hash vallues for tipBlockHash, tipBlockParentHash and tipBlockIssuerVkHashText" + ) ] metricsDocFor _ = [] From 5c4f271211512bd3e7b31bf83e5c9bd326c5a784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Nicklisch-Franken?= Date: Fri, 13 Dec 2024 22:33:22 +0100 Subject: [PATCH 8/8] cardano-node: fix typos --- cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs index 2c655353df2..696c710fdcc 100644 --- a/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs +++ b/cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs @@ -749,7 +749,7 @@ instance MetaTrace (ChainDB.TraceAddBlockEvent blk) where , "counter for forks" ) , ( "tipBlock" - , "Hash vallues for tipBlockHash, tipBlockParentHash and tipBlockIssuerVkHashText" + , "Values for hash, parent hash and issuer verification key hash" ) ] @@ -776,7 +776,7 @@ instance MetaTrace (ChainDB.TraceAddBlockEvent blk) where , "In which epoch is the tip of the current chain." ) , ( "tipBlock" - , "Hash vallues for tipBlockHash, tipBlockParentHash and tipBlockIssuerVkHashText" + , "Values for hash, parent hash and issuer verification key hash" ) ] metricsDocFor _ = []