Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cardano-testnet: fix that genesis files generated by create-testnet-data were ignored #5732

Merged
merged 7 commits into from
Apr 3, 2024
Merged

cardano-testnet: fix that genesis files generated by create-testnet-data were ignored #5732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
873f9d9
cardano-testnet: fix that genesis files generated by create-testnet-d…
smelc Mar 28, 2024
544120c
cardano-testnet: simplify tests using DReps, thanks to newer create-t…
smelc Mar 28, 2024
cd3c037
cardano-testnet: Share definition of path to DRep keys
smelc Mar 28, 2024
bf451fe
cardano-testnet: rename a function
smelc Mar 28, 2024
1561146
cardano-testnet: remove rewrite that became obsolete
smelc Mar 29, 2024
38c754b
cardano-testnet: Use encodePretty when writing some files to disk, fo…
smelc Apr 2, 2024
45d1b8c
cardano-testnet: Update golden files
smelc Mar 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cardano-testnet/cardano-testnet.cabal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ library
import: project-config

build-depends: aeson
, aeson-pretty
, ansi-terminal
, bytestring
, cardano-api ^>= 8.42.0.0
Expand Down
74 changes: 35 additions & 39 deletions cardano-testnet/src/Testnet/Components/Configuration.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

module Testnet.Components.Configuration
( anyEraToString
, createConfigYaml
, createConfigJson
, createSPOGenesisAndFiles
, eraToString
, mkTopologyConfig
Expand All @@ -32,6 +32,7 @@ import Control.Monad
import Control.Monad.Catch (MonadCatch)
import Data.Aeson
import qualified Data.Aeson as Aeson
import qualified Data.Aeson.Encode.Pretty as Aeson
import qualified Data.Aeson.KeyMap as Aeson
import qualified Data.Aeson.Lens as L
import qualified Data.ByteString.Lazy as LBS
Expand All @@ -41,6 +42,7 @@ import qualified Data.Text as Text
import GHC.Stack (HasCallStack)
import qualified GHC.Stack as GHC
import Lens.Micro
import qualified System.Directory as System
import System.FilePath.Posix (takeDirectory, (</>))

import Testnet.Defaults
Expand All @@ -56,18 +58,18 @@ import qualified Hedgehog.Extras.Test.Base as H
import qualified Hedgehog.Extras.Test.File as H

-- | Returns JSON encoded hashes of the era, as well as the hard fork configuration toggle.
createConfigYaml :: ()
createConfigJson :: ()
=> (MonadTest m, MonadIO m, HasCallStack)
=> TmpAbsolutePath
-> AnyCardanoEra -- ^ The era used for generating the hard fork configuration toggle
-> m LBS.ByteString
createConfigYaml (TmpAbsolutePath tempAbsPath) era = GHC.withFrozenCallStack $ do
createConfigJson (TmpAbsolutePath tempAbsPath) era = GHC.withFrozenCallStack $ do
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's use pretty JSON here. Makes debugging a little easier.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, I'm myself always reformatting those JSON files before inspecting them 😆

byronGenesisHash <- getByronGenesisHash $ tempAbsPath </> "byron/genesis.json"
shelleyGenesisHash <- getHash ShelleyEra "ShelleyGenesisHash"
alonzoGenesisHash <- getHash AlonzoEra "AlonzoGenesisHash"
conwayGenesisHash <- getHash ConwayEra "ConwayGenesisHash"

return . Aeson.encode . Aeson.Object
return . Aeson.encodePretty . Aeson.Object
$ mconcat [ byronGenesisHash
, shelleyGenesisHash
, alonzoGenesisHash
Expand Down Expand Up @@ -95,34 +97,29 @@ createSPOGenesisAndFiles
-> ConwayGenesis StandardCrypto -- ^ The conway genesis to use, for example 'Defaults.defaultConwayGenesis'.
-> TmpAbsolutePath
-> m FilePath -- ^ Shelley genesis directory
createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesis conwayGenesis (TmpAbsolutePath tempAbsPath) = do
let genesisShelleyFpAbs = tempAbsPath </> defaultGenesisFilepath ShelleyEra
genesisAlonzoFpAbs = tempAbsPath </> defaultGenesisFilepath AlonzoEra
genesisConwayFpAbs = tempAbsPath </> defaultGenesisFilepath ConwayEra
genesisShelleyDirAbs = takeDirectory genesisShelleyFpAbs
createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesis conwayGenesis (TmpAbsolutePath tempAbsPath) = GHC.withFrozenCallStack $ do
let inputGenesisShelleyFp = tempAbsPath </> genesisInputFilepath ShelleyEra
inputGenesisAlonzoFp = tempAbsPath </> genesisInputFilepath AlonzoEra
inputGenesisConwayFp = tempAbsPath </> genesisInputFilepath ConwayEra

-- We write the genesis files to disk, to pass them to create-testnet-data.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These files shouldn't be modified after they are created. create-testnet-data should create the final genesis files we want to use and then we create a configuration file based on those genesis files.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep agreed, we should remove the last rewrites that remain later on 👍

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you update the comment so it's obvious this isn't the behaviour we want?

-- Then, create-testnet-data will output (possibly augmented/modified) versions
-- and we remove those input files (see below), to avoid confusion.
H.evalIO $ do
LBS.writeFile inputGenesisShelleyFp $ Aeson.encodePretty shelleyGenesis
LBS.writeFile inputGenesisAlonzoFp $ Aeson.encodePretty alonzoGenesis
LBS.writeFile inputGenesisConwayFp $ Aeson.encodePretty conwayGenesis

let genesisShelleyDirAbs = takeDirectory inputGenesisShelleyFp
genesisShelleyDir <- H.createDirectoryIfMissing genesisShelleyDirAbs
let testnetMagic = sgNetworkMagic shelleyGenesis
numStakeDelegators = 3 :: Int
startTime = sgSystemStart shelleyGenesis

-- We create the initial genesis file to avoid having to re-write the genesis file later
-- with the parameters we want. The user must provide genesis files or we will use a default.
-- We should *never* be modifying the genesis file after @cardanoTestnet@ is run because this
-- is sure to be a source of confusion if users provide genesis files and we are mutating them
-- without their knowledge.
H.evalIO $ do
LBS.writeFile genesisShelleyFpAbs $ encode shelleyGenesis
LBS.writeFile genesisAlonzoFpAbs $ encode alonzoGenesis
LBS.writeFile genesisConwayFpAbs $ encode conwayGenesis

-- TODO: Remove this rewrite.
-- TODO: Remove this rewrite.
-- 50 second epochs
-- Epoch length should be "10 * k / f" where "k = securityParam, f = activeSlotsCoeff"
H.rewriteJsonFile @Value genesisShelleyFpAbs $ \o -> o
-- TODO: remove rho and tau adjustment after https://github.com/IntersectMBO/cardano-api/pull/425 gets
-- integrated with newer cardano-api into node
& L.key "protocolParams" . L.key "rho" . L._Number .~ 0.1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💪

& L.key "protocolParams" . L.key "tau" . L._Number .~ 0.1
H.rewriteJsonFile @Value inputGenesisShelleyFp $ \o -> o
& L.key "securityParam" . L._Integer .~ 5
& L.key "updateQuorum" . L._Integer .~ 2

Expand All @@ -135,9 +132,9 @@ createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesi

execCli_
[ anyEraToString era, "genesis", "create-testnet-data"
, "--spec-shelley", genesisShelleyFpAbs
, "--spec-alonzo", genesisAlonzoFpAbs
, "--spec-conway", genesisConwayFpAbs
, "--spec-shelley", inputGenesisShelleyFp
, "--spec-alonzo", inputGenesisAlonzoFp
, "--spec-conway", inputGenesisConwayFp
, "--testnet-magic", show testnetMagic
, "--pools", show numPoolNodes
, "--total-supply", show @Int 2_000_000_000_000
Expand All @@ -149,35 +146,34 @@ createSPOGenesisAndFiles (NumPools numPoolNodes) era shelleyGenesis alonzoGenesi
, "--out-dir", tempAbsPath
]

-- Here we move all of the keys etc generated by create-testnet-data
-- for the nodes to use
-- Remove the input files. We don't need them anymore, since create-testnet-data wrote new versions.
forM_ [inputGenesisShelleyFp, inputGenesisAlonzoFp, inputGenesisConwayFp] (liftIO . System.removeFile)

-- Move all genesis related files

genesisByronDir <- H.createDirectoryIfMissing $ tempAbsPath </> "byron"

files <- H.listDirectory tempAbsPath
forM_ files $ \file -> do
H.note file
forM_ files H.note

H.renameFile (tempAbsPath </> "byron-gen-command" </> "genesis.json") (genesisByronDir </> "genesis.json")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to make the byron genesis handled like the others (i.e. to end up at byron-genesis.json), but it makes the node refuse to start, because of a hash mismatch. I will solve this later on, because I didn't want to block this PR.

H.renameFile (tempAbsPath </> "shelley-genesis.json") (genesisShelleyDir </> "genesis.shelley.json")

-- For some reason when setting "--total-supply 10E16" in create-testnet-data, we're getting negative
-- treasury. TODO: This should be fixed by https://github.com/IntersectMBO/cardano-cli/pull/644
-- So this can be removed when cardano-cli is upgraded above 8.20.3.0.
H.rewriteJsonFile @Value (genesisShelleyDir </> "genesis.shelley.json") $ \o -> o
-- treasury. TODO: This was supposed to be fixed by https://github.com/IntersectMBO/cardano-cli/pull/644,
-- but no, it's still there.
H.rewriteJsonFile @Value (tempAbsPath </> defaultGenesisFilepath ShelleyEra) $ \o -> o
& L.key "maxLovelaceSupply" . L._Integer .~ 10_000_000_000_000_000
Comment on lines +163 to 164
Copy link
Contributor Author

@smelc smelc Mar 29, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to get rid of this rewrite, but it makes the Treasury Growth fail, so something is still wrong somewhere. I will look into it in another PR. I don't want to block this one.


return genesisShelleyDir
where
genesisInputFilepath e = "genesis-input." <> anyEraToString (AnyCardanoEra e) <> ".json"

ifaceAddress :: String
ifaceAddress = "127.0.0.1"

-- TODO: Reconcile all other mkTopologyConfig functions. NB: We only intend
-- to support current era on mainnet and the upcoming era.
mkTopologyConfig :: Int -> [Int] -> Int -> Bool -> LBS.ByteString
mkTopologyConfig numNodes allPorts port False = Aeson.encode topologyNonP2P
mkTopologyConfig numNodes allPorts port False = Aeson.encodePretty topologyNonP2P
where
topologyNonP2P :: NonP2P.NetworkTopology
topologyNonP2P =
Expand All @@ -187,7 +183,7 @@ mkTopologyConfig numNodes allPorts port False = Aeson.encode topologyNonP2P
(numNodes - 1)
| peerPort <- allPorts List.\\ [port]
]
mkTopologyConfig numNodes allPorts port True = Aeson.encode topologyP2P
mkTopologyConfig numNodes allPorts port True = Aeson.encodePretty topologyP2P
where
rootConfig :: P2P.RootConfig
rootConfig =
Expand Down
20 changes: 18 additions & 2 deletions cardano-testnet/src/Testnet/Defaults.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ module Testnet.Defaults
, defaultByronProtocolParamsJsonValue
, defaultYamlConfig
, defaultConwayGenesis
, defaultDRepVkeyFp
, defaultDRepSkeyFp
, defaultShelleyGenesis
, defaultGenesisFilepath
, defaultYamlHardforkViaConfig
Expand Down Expand Up @@ -64,7 +66,7 @@ import Data.Word
import GHC.Stack
import Lens.Micro
import Numeric.Natural
import System.FilePath.Posix ((</>))
import System.FilePath ((</>))

import Test.Cardano.Ledger.Core.Rational
import Testnet.Start.Types
Expand Down Expand Up @@ -552,7 +554,21 @@ defaultMainnetTopology =
in RealNodeTopology [single]

defaultGenesisFilepath :: CardanoEra a -> FilePath
defaultGenesisFilepath era = "shelley" </> ("genesis." <> eraToString era <> ".json")
defaultGenesisFilepath era =
-- This path is actually generated by create-testnet-data. Don't change it.
eraToString era <> "-genesis.json"
Copy link
Contributor

@Jimbo4350 Jimbo4350 Apr 2, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍


-- | The relative path to DRep keys in directories created by cardano-testnet
defaultDRepVkeyFp
:: Int -- ^ The DRep's index (starts at 1)
-> FilePath
defaultDRepVkeyFp n = "drep-keys" </> ("drep" <> show n) </> "drep.vkey"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍


-- | The relative path to DRep secret keys in directories created by cardano-testnet
defaultDRepSkeyFp
:: Int -- ^ The DRep's index (starts at 1)
-> FilePath
defaultDRepSkeyFp n = "drep-keys" </> ("drep" <> show n) </> "drep.skey"

-- TODO: We should not hardcode a script like this. We need to move
-- plutus-example from plutus apps to cardano-node-testnet. This will
Expand Down
4 changes: 2 additions & 2 deletions cardano-testnet/src/Testnet/Start/Cardano.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,9 @@ cardanoTestnet
mkNodeName i = "pool" <> show i

-- Add Byron, Shelley and Alonzo genesis hashes to node configuration
finalYamlConfig <- createConfigYaml (TmpAbsolutePath tmpAbsPath) era
config <- createConfigJson (TmpAbsolutePath tmpAbsPath) era

H.evalIO $ LBS.writeFile configurationFile finalYamlConfig
H.evalIO $ LBS.writeFile configurationFile config

-- Byron related
forM_ (zip [1..] portNumbers) $ \(i, portNumber) -> do
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/allegra_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -15,7 +15,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TestAllegraHardForkAtEpoch": 0,
"TestShelleyHardForkAtEpoch": 0,
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/alonzo_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -15,7 +15,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TestAllegraHardForkAtEpoch": 0,
"TestAlonzoHardForkAtEpoch": 0,
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/babbage_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -15,7 +15,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TestAllegraHardForkAtEpoch": 0,
"TestAlonzoHardForkAtEpoch": 0,
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/byron_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -13,7 +13,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TraceBlockFetchClient": false,
"TraceBlockFetchDecisions": false,
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/conway_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -15,7 +15,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TestAllegraHardForkAtEpoch": 0,
"TestAlonzoHardForkAtEpoch": 0,
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/mary_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -15,7 +15,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TestAllegraHardForkAtEpoch": 0,
"TestMaryHardForkAtEpoch": 0,
Expand Down
6 changes: 3 additions & 3 deletions cardano-testnet/test/cardano-testnet-golden/files/golden/shelley_node_default_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"AlonzoGenesisFile": "shelley/genesis.alonzo.json",
"AlonzoGenesisFile": "alonzo-genesis.json",
"ByronGenesisFile": "byron/genesis.json",
"ConwayGenesisFile": "shelley/genesis.conway.json",
"ConwayGenesisFile": "conway-genesis.json",
"EnableLogMetrics": false,
"EnableLogging": true,
"EnableP2P": false,
Expand All @@ -15,7 +15,7 @@
"PBftSignatureThreshold": 0.6,
"Protocol": "Cardano",
"RequiresNetworkMagic": "RequiresMagic",
"ShelleyGenesisFile": "shelley/genesis.shelley.json",
"ShelleyGenesisFile": "shelley-genesis.json",
"SocketPath": "db/node.socket",
"TestShelleyHardForkAtEpoch": 0,
"TraceBlockFetchClient": false,
Expand Down
4 changes: 2 additions & 2 deletions ...-testnet/test/cardano-testnet-test/Cardano/Testnet/Test/Cli/Babbage/LeadershipSchedule.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,8 @@ hprop_leadershipSchedule = H.integrationRetryWorkspace 2 "babbage-leadership-sch
, "--out-file", testSpoOperationalCertFp
]

yamlBs <- createConfigYaml tempAbsPath (cardanoNodeEra cTestnetOptions)
H.lbsWriteFile configurationFile yamlBs
jsonBS <- createConfigJson tempAbsPath (cardanoNodeEra cTestnetOptions)
H.lbsWriteFile configurationFile jsonBS
[newNodePort] <- requestAvailablePortNumbers 1
eRuntime <- lift . lift . runExceptT $ startNode (TmpAbsolutePath work) "test-spo" "127.0.0.1" newNodePort testnetMagic
[ "run"
Expand Down
Loading
Loading