Leios: Accept a JSON array of BLS key envelopes in --shelley-bls-key

cabal.project

@@ -108,12 +108,19 @@ source-repository-package
   subdir:
     typed-protocols
 
--- Points to ouroboros-ledger/leios-prototype
+-- Points to ouroboros-consensus/leios-prototype
+source-repository-package
+  type: git
+  location: https://github.com/IntersectMBO/ouroboros-consensus
+  tag: e5b82026e4f3395fbb94f0af62f711655d96fec7
+  --sha256: sha256-WfinPoIUwdq3fm5aeDpyJCgXHjuIMtqrq57QZMq5E7Y=
+
+-- Points to cardano-ledger/leios-prototype
 source-repository-package
   type: git
   location: https://github.com/IntersectMBO/cardano-ledger
-  tag: 1587f21a7d1306dc590c2749a5c66232ef66aad0
-  --sha256: sha256-zGKFgOU+p0uMTNbIm0Em6GMYMUVZMJsJtynzXG46E58=
+  tag: daa38fec95ddba68ad4e780b933e7e094d68241d
+  --sha256: sha256-UtLWITAOilFFXXRwCyAoG+6NHd/34wIYG296z3+L7FA=
   subdir:
     libs/cardano-data
     libs/cardano-ledger-api
@@ -150,8 +157,8 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/IntersectMBO/cardano-api
-  tag: e32d8c07035283233fbdca4beeafbd78abc66512
-  --sha256: sha256-yYKuOUoIOvJy6872Sj3CruRT+UXfoQsqqaHXcYu8N1A=
+  tag: 8749531d1b09f8ab5c50c3ac7b41cb4bb8ed9445
+  --sha256: sha256-9892q0kBsCy5CLrYPjxHv5C7GOK/41XZZ5Z3JBZpB9E=
   subdir:
     cardano-api
     cardano-api-gen
@@ -161,8 +168,8 @@ source-repository-package
 source-repository-package
   type: git
   location: https://github.com/IntersectMBO/cardano-cli
-  tag: 517ffe9ed02bba2b9ed644454a74c280ad62c2fa
-  --sha256: sha256-vNw0fqUF/dboMVVOgUs1eKGEdT36xN5lJEbJ2tRV1EI=
+  tag: cca726ea4026f6e0a40d38d549ff7ea5d458fc12
+  --sha256: sha256-Oz9WGXDF7sqRLlqDXBJjtQr4NxxcprT+lCZAaDZRDPk=
   subdir:
     cardano-cli
 

cardano-node/ChangeLog.md

@@ -4,9 +4,7 @@
 
 * Added a `--shelley-bls-key FILEPATH` option to `cardano-node run` for supplying a block producer's BLS (Leios) signing key alongside the existing VRF/KES/operational-certificate keys. The key is threaded into the consensus block-producer credentials and used as the Leios voting key. It is optional: producers that do not supply it no longer vote (previously a placeholder key was derived from cold-key material). Generate one with `cardano-cli dijkstra node key-gen-BLS`.
 
-* `LeiosDbConfig` with `Backend: SQLite` now takes two paths, `VolatileFilepath` and `ImmutableFilepath`, one per LeiosDb partition, replacing the single `Filepath` key. The defaults are `leios.db.vol` and `leios.db.imm`.
-
-* Resolve relative `LeiosDbConfig` SQLite paths against `--database-path`, so the default `leios.db.vol` and `leios.db.imm` are placed alongside `immutable/` and `volatile/`.
+* `LeiosDbConfig` with `Backend: SQLite` no longer takes any paths. Its two partitions follow the node's own `DatabasePath` the way the VolatileDB and the ImmutableDB do: `leios.vol.db` next to `volatile/`, `leios.imm.db` next to `immutable/`, which are separate volumes when `--immutable-database-path` and `--volatile-database-path` are given. Any `Filepath`, `VolatileFilepath` or `ImmutableFilepath` key is ignored, and an existing LeiosDb under the old `leios.db.vol`/`leios.db.imm` names is not picked up: the node starts a fresh one and re-fetches the endorser-block closures.
 
 
 * Added txsSyncDurationTotal counter for tracking the total time spent syncing the mempool.

cardano-node/src/Cardano/Node/Configuration/Leios.hs

@@ -8,29 +8,29 @@ module Cardano.Node.Configuration.Leios(
 import           Data.Aeson (FromJSON (parseJSON), ToJSON (toJSON), Value (String), object,
                    withObject, (.:), (.=))
 
+-- | Which LeiosDB backend to run. Where its files live is not configurable
+-- here: the SQLite backend has a volatile and an immutable partition, and
+-- each follows the node's own 'DatabasePath' the way the VolatileDB and the
+-- ImmutableDB do. Naming them separately could only put them somewhere that
+-- contradicts that.
 data LeiosDbConfig = LeiosDbInMemory
-   | LeiosDbSQLite !FilePath !FilePath
+   | LeiosDbSQLite
    deriving (Eq, Show)
 
 instance FromJSON LeiosDbConfig where
   parseJSON = withObject "LeiosDbConfig" $ \o -> do
     backend :: String <- o .: "Backend"
     case backend of
       "InMemory" -> return LeiosDbInMemory
-      "SQLite" -> do
-        volPath <- o .: "VolatileFilepath"
-        immPath <- o .: "ImmutableFilepath"
-        return $ LeiosDbSQLite volPath immPath
+      "SQLite" -> return LeiosDbSQLite
       _ -> fail $ "Invalid LeiosDb backend " <> backend <> ", did you mean InMemory or SQLite?"
 
 instance ToJSON LeiosDbConfig where
   toJSON LeiosDbInMemory =
     object
       [ "Backend" .=  String "InMemory"
       ]
-  toJSON (LeiosDbSQLite volPath immPath) =
+  toJSON LeiosDbSQLite =
     object
-      [ "Backend" .= String "SQLite",
-        "VolatileFilepath" .= volPath,
-        "ImmutableFilepath" .= immPath
+      [ "Backend" .= String "SQLite"
       ]

cardano-node/src/Cardano/Node/Configuration/POM.hs

@@ -755,7 +755,7 @@
     , pncTxSubmissionLogicVersion = Last $ Just TxSubmissionLogicV1
     , pncTxSubmissionInitDelay = Last $ Just defaultTxSubmissionInitDelay
 
-    , pncLeiosDbConfig = Last (Just (LeiosDbSQLite "leios.db.vol" "leios.db.imm"))
+    , pncLeiosDbConfig = Last (Just (LeiosDbSQLite))
     }
 
 lastOption :: Parser a -> Parser (Last a)

cardano-node/src/Cardano/Node/Parsers.hs

@@ -419,7 +419,11 @@ parseBlsKeyFilePath =
   strOption
     ( long "shelley-bls-key"
         <> metavar "FILEPATH"
-        <> help "Path to the BLS (Leios) signing key."
+        <> help
+             ( "Path to the BLS (Leios) signing key: a single text envelope, "
+            <> "or a JSON array of them to vote with every key that holds a "
+            <> "committee seat (e.g. a rotation pair, or a bundle of pool keys)."
+             )
         <> completer (bashCompleter "file")
     )
 

cardano-node/src/Cardano/Node/Protocol/Shelley.hs

@@ -53,10 +53,10 @@ import qualified Data.Aeson as Aeson
 import qualified Data.ByteString as BS
 import qualified Data.Text as T
 import           System.Directory (getFileSize)
+import           System.FilePath (takeDirectory)
 import           System.FS.API (SomeHasFS (..))
 import           System.FS.API.Types (MountPoint (MountPoint))
 import           System.FS.IO (ioHasFS)
-import           System.FilePath (takeDirectory)
 import qualified System.IO.MMap as MMap
 
 
@@ -190,11 +190,9 @@ readLeaderCredentialsSingleton
     vrfSKey <-
       firstExceptT FileError (newExceptT $ readFileTextEnvelope (File vrfFile))
 
-    -- The BLS (Leios) key is optional: only block producers participating in
-    -- Leios supply one alongside their VRF/KES/opcert credentials.
-    blsSKey <-
-      firstExceptT FileError $
-        traverse (\blsFile -> newExceptT $ readFileTextEnvelope (File blsFile)) mBlsFile
+    -- The BLS (Leios) keys are optional: only block producers participating in
+    -- Leios supply any alongside their VRF/KES/opcert credentials.
+    blsSKeys <- maybe (pure []) readBlsSigningKeys mBlsFile
 
     (credentialsSource, vkey) <- case kesSource of
       KESKeyFilePath kesFile -> do
@@ -209,7 +207,7 @@ readLeaderCredentialsSingleton
         OperationalCertificate _ vkey <- firstExceptT FileError $ newExceptT $ readFileTextEnvelope $ File opCertFile
         pure (PraosCredentialsAgent socketFile, vkey)
 
-    return [mkPraosLeaderCredentials credentialsSource vkey vrfSKey blsSKey]
+    return [mkPraosLeaderCredentials credentialsSource vkey vrfSKey blsSKeys]
 
 -- But not OK to supply some of the files without the others.
 readLeaderCredentialsSingleton ProtocolFilepaths {shelleyCertFile = Nothing} =
@@ -259,8 +257,8 @@ readLeaderCredentialsBulk ProtocolFilepaths { shelleyBulkCredsFile = mfp } =
      KesSigningKey kesKey <- parseEnvelope scKes
      let credentialsSource = PraosCredentialsUnsound opCert kesKey
      vrfSKey <- parseEnvelope scVrf
-     -- Bulk credentials files do not carry a BLS (Leios) key.
-     pure $ mkPraosLeaderCredentials credentialsSource vkey vrfSKey Nothing
+     -- Bulk credentials files do not carry BLS (Leios) keys.
+     pure $ mkPraosLeaderCredentials credentialsSource vkey vrfSKey []
 
    readBulkFile
      :: Maybe FilePath
@@ -285,26 +283,42 @@ mkPraosLeaderCredentials ::
      PraosCredentialsSource StandardCrypto
   -> VerificationKey StakePoolKey
   -> SigningKey VrfKey
-  -> Maybe (SigningKey BlsKey)
+  -> [SigningKey BlsKey]
   -> ShelleyLeaderCredentials StandardCrypto
 mkPraosLeaderCredentials
     credentialsSource
     (StakePoolVerificationKey vkey)
     (VrfSigningKey vrfKey)
-    mBlsKey =
+    blsKeys =
     ShelleyLeaderCredentials
     { shelleyLeaderCredentialsCanBeLeader =
         PraosCanBeLeader {
           praosCanBeLeaderCredentialsSource = credentialsSource,
           praosCanBeLeaderColdVerKey = coerceKeyRole vkey,
           praosCanBeLeaderSignKeyVRF = vrfKey,
-          praosCanBeLeaderSignKeyBLS = unBlsSigningKey <$> mBlsKey
+          praosCanBeLeaderSignKeyBLS = unBlsSigningKey <$> blsKeys
         },
       shelleyLeaderCredentialsLabel = "Shelley"
     }
   where
     unBlsSigningKey (BlsSigningKey k) = k
 
+-- | Read the BLS (Leios) signing keys from a file holding either a single text
+-- envelope or a JSON array of them. A pair of keys keeps a rotated key voting
+-- across the epoch boundary; a larger bundle casts one vote per committee seat
+-- held.
+readBlsSigningKeys ::
+     FilePath
+  -> ExceptT PraosLeaderCredentialsError IO [SigningKey BlsKey]
+readBlsSigningKeys fp = do
+  content <- handleIOExceptT (CredentialsReadError fp) $ BS.readFile fp
+  envelopes <-
+    firstExceptT (EnvelopeParseError fp) . hoistEither $
+      case Aeson.eitherDecodeStrict' content of
+        Right tes -> Right tes
+        Left _ -> (:[]) <$> Aeson.eitherDecodeStrict' content
+  traverse (\te -> parseEnvelope (te, fp)) envelopes
+
 parseEnvelope ::
      HasTextEnvelope a
   => (TextEnvelope, String)

cardano-node/src/Cardano/Node/Run.hs

@@ -67,7 +67,11 @@ import           Cardano.Logging.Utils (showT)
 import qualified Ouroboros.Consensus.Config as Consensus
 import           Ouroboros.Consensus.Config.SupportsNode (ConfigSupportsNode (..))
 import           Ouroboros.Consensus.Node (SnapshotPolicyArgs (..),
-                   NodeDatabasePaths (..), nonImmutableDbPath, RunNodeArgs (..), StdRunNodeArgs (..))
+                   NodeDatabasePaths (..), immutableDbPath, nonImmutableDbPath,
+                   RunNodeArgs (..), StdRunNodeArgs (..))
+import           Ouroboros.Consensus.Node.DbMarker (checkDbMarker)
+import           System.FS.API.Types (MountPoint (..))
+import           System.FS.IO (ioHasFS)
 import           Ouroboros.Consensus.Protocol.Praos.AgentClient (KESAgentClientTrace)
 import           Ouroboros.Consensus.Ledger.SupportsMempool (GenTxId)
 import           Ouroboros.Consensus.Node (RunNodeArgs (..),
@@ -156,7 +160,7 @@ import           Data.Time.Clock (getCurrentTime)
 import           Network.DNS (Resolver)
 import           Network.Socket (Socket)
 import           System.Directory (canonicalizePath, createDirectoryIfMissing, makeAbsolute)
-import           System.FilePath (isAbsolute, takeDirectory, (</>))
+import           System.FilePath (isAbsolute, (</>))
 import           System.IO (hPutStrLn)
 #ifdef UNIX
 import           GHC.Weak (deRefWeak)
@@ -169,7 +173,7 @@ import           System.Win32.File
 import           Ouroboros.Consensus.Mempool (MempoolTimeoutConfig(..))
 import           GHC.Stack
 
-import           LeiosDemoDb (newLeiosDBInMemory, newLeiosDBSQLite)
+import           LeiosDemoDb (LeiosDbHandle (close), newLeiosDBInMemory, newLeiosDBSQLite)
 import           LeiosDemoTypes (TraceLeiosKernel (TraceLeiosDb))
 
 {- HLINT ignore "Fuse concatMap/map" -}
@@ -365,23 +369,33 @@ handleSimpleNode blockType shelleyGenesisHash runP tracers nc networkMagic onKer
                          $ Proxy @blk
                          ))
 
+  -- Establish the ChainDB's marker before anything else writes into its
+  -- directory. The check refuses a directory that holds files but no marker
+  -- of its own, and the Leios DB's files land in that same directory
+  -- whenever the node runs on a single database path. 'Node.run' makes this
+  -- same check later; it is idempotent, so doing it here only moves it
+  -- earlier.
+  let dbMarkerMountPoint = MountPoint (immutableDbPath dbPath)
+  either Exception.throwIO pure
+    =<< checkDbMarker (ioHasFS dbMarkerMountPoint) dbMarkerMountPoint networkMagic
+
   leiosDB <- case ncLeiosDbConfig nc of
     LeiosDbInMemory -> newLeiosDBInMemory
-    LeiosDbSQLite leiosVolDbPath leiosImmDbPath -> do
-      let resolvedVolPath
-            | isAbsolute leiosVolDbPath = leiosVolDbPath
-            | otherwise = nonImmutableDbPath dbPath </> leiosVolDbPath
-          resolvedImmPath
-            | isAbsolute leiosImmDbPath = leiosImmDbPath
-            | otherwise = nonImmutableDbPath dbPath </> leiosImmDbPath
-      createDirectoryIfMissing True (takeDirectory resolvedVolPath)
-      createDirectoryIfMissing True (takeDirectory resolvedImmPath)
+    LeiosDbSQLite -> do
+      -- Each partition follows the node's own split: the volatile one
+      -- churns and is swept, so it belongs on the performant volume with
+      -- the VolatileDB; the immutable one only grows, so it belongs with
+      -- the ImmutableDB. Identical under 'OnePathForAllDbs'.
+      let resolvedVolPath = nonImmutableDbPath dbPath </> "leios.vol.db"
+          resolvedImmPath = immutableDbPath dbPath </> "leios.imm.db"
       newLeiosDBSQLite
         (contramap TraceLeiosDb (Consensus.leiosKernelTracer (consensusTracers tracers)))
         resolvedVolPath
         resolvedImmPath
 
-  withShutdownHandling (ncShutdownConfig nc) (shutdownTracer tracers) $ do
+  -- Orderly shutdown of the LeiosDB: everything below has stopped by then, so
+  -- this flushes the pending writes and closes the connections.
+  (`Exception.finally` close leiosDB) $ withShutdownHandling (ncShutdownConfig nc) (shutdownTracer tracers) $ do
     traceWith (startupTracer tracers)
               (StartupP2PInfo (ncDiffusionMode nc))
     nt@NetworkTopology

cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs

@@ -50,7 +50,7 @@ import           Ouroboros.Consensus.Genesis.Governor (DensityBounds (..), GDDDe
 import           Ouroboros.Consensus.Ledger.Extended (ExtValidationError)
 import           Ouroboros.Consensus.Ledger.Inspect (LedgerEvent (..), LedgerUpdate, LedgerWarning)
 import           Ouroboros.Consensus.Ledger.SupportsMempool (ApplyTxErr, ByteSize32 (..), GenTxId,
-                   HasTxId, LedgerSupportsMempool, txForgetValidated, txId)
+                   HasTxId, LedgerSupportsMempool, txForgetValidated, txId, txMeasureByteSize)
 import           Ouroboros.Consensus.Ledger.SupportsProtocol
 import           Ouroboros.Consensus.Mempool (MempoolRejectionDetails (..), MempoolSize (..),
                    TraceEventMempool (..), jsonMempoolRejectionDetails)
@@ -1142,6 +1142,12 @@ instance
     mconcat
       [ "kind" .= String "TraceMempoolTipMovedBetweenSTMBlocks"
       ]
+  forMachine _dtal (TraceMempoolCapacityChanged capBefore capAfter) =
+    mconcat
+      [ "kind" .= String "TraceMempoolCapacityChanged"
+      , "capacityBytesBefore" .= unByteSize32 (txMeasureByteSize capBefore)
+      , "capacityBytesAfter" .= unByteSize32 (txMeasureByteSize capAfter)
+      ]
 
   asMetrics (TraceMempoolAddedTx _tx _mpSzBefore mpSz) =
     [ IntM "txsInMempool" (fromIntegral $ msNumTxs mpSz)
@@ -1176,6 +1182,10 @@ instance
 
   asMetrics TraceMempoolTipMovedBetweenSTMBlocks {} = []
 
+  asMetrics (TraceMempoolCapacityChanged _capBefore capAfter) =
+    [ IntM "mempoolCapacityBytes" (fromIntegral . unByteSize32 . txMeasureByteSize $ capAfter)
+    ]
+
 instance LogFormatting MempoolSize where
   forMachine _dtal MempoolSize{msNumTxs, msNumBytes} =
     mconcat
@@ -1193,6 +1203,7 @@ instance MetaTrace (TraceEventMempool blk) where
     namespaceFor TraceMempoolSyncNotNeeded {} = Namespace [] ["SyncNotNeeded"]
     namespaceFor TraceMempoolAttemptingAdd {} = Namespace [] ["AttemptAdd"]
     namespaceFor TraceMempoolTipMovedBetweenSTMBlocks {} = Namespace [] ["TipMovedBetweenSTMBlocks"]
+    namespaceFor TraceMempoolCapacityChanged {} = Namespace [] ["CapacityChanged"]
 
 
     severityFor (Namespace _ ["AddedTx"]) _ = Just Info
@@ -1203,6 +1214,7 @@ instance MetaTrace (TraceEventMempool blk) where
     severityFor (Namespace _ ["SyncNotNeeded"]) _ = Just Debug
     severityFor (Namespace _ ["AttemptAdd"]) _ = Just Debug
     severityFor (Namespace [] ["TipMovedBetweenSTMBlocks"]) _ = Just Debug
+    severityFor (Namespace _ ["CapacityChanged"]) _ = Just Info
     severityFor _ _ = Nothing
 
     metricsDocFor (Namespace _ ["AddedTx"]) =
@@ -1227,6 +1239,9 @@ instance MetaTrace (TraceEventMempool blk) where
       [ ("txsSyncDuration", "Latest time to sync the mempool in ms after block adoption")
       , (txsSyncDurationTotalCounterName, "Cumulative time spent syncing the mempool in ms after block adoption")
       ]
+    metricsDocFor (Namespace _ ["CapacityChanged"]) =
+      [ ("mempoolCapacityBytes", "Byte capacity of the mempool")
+      ]
     metricsDocFor _ = []
 
     documentFor (Namespace _ ["AddedTx"]) = Just
@@ -1250,6 +1265,9 @@ instance MetaTrace (TraceEventMempool blk) where
       "Mempool is about to try to validate and add a transaction."
     documentFor (Namespace _ ["TipMovedBetweenSTMBlocks"]) = Just
       "LedgerDB moved to an alternative fork between two reads during re-sync."
+    documentFor (Namespace _ ["CapacityChanged"]) = Just
+      "The mempool capacity changed while syncing with the ledger, e.g. because\
+      \ a protocol parameter update was adopted."
     documentFor _ = Nothing
 
     allNamespaces =
@@ -1261,6 +1279,7 @@ instance MetaTrace (TraceEventMempool blk) where
       , Namespace [] ["SyncNotNeeded"]
       , Namespace [] ["AttemptAdd"]
       , Namespace [] ["TipMovedBetweenSTMBlocks"]
+      , Namespace [] ["CapacityChanged"]
       ]
 
 --------------------------------------------------------------------------------
@@ -2472,8 +2491,6 @@ instance LogFormatting TraceLeiosKernel where
       [ CounterM "leiosDbEvictedEbs" (Just evictedEbs) ]
     TraceLeiosDb TraceLeiosDbGCError{} ->
       [ CounterM "leiosDbSweepErrors" (Just 1) ]
-    TraceLeiosDb TraceLeiosDbCopyQueueFull{} ->
-      [ CounterM "leiosDbCopyQueueFull" (Just 1) ]
     TraceLeiosDb TraceLeiosDbCopyError{} ->
       [ CounterM "leiosDbCopyErrors" (Just 1) ]
     _ -> []

cardano-node/test/Test/Cardano/Node/POM.hs

@@ -294,7 +294,7 @@ eExpectedConfig = do
     , ncRpcConfig
     , ncTxSubmissionLogicVersion = TxSubmissionLogicV1
     , ncTxSubmissionInitDelay = defaultTxSubmissionInitDelay
-    , ncLeiosDbConfig = LeiosDbSQLite "leios.db.vol" "leios.db.imm"
+    , ncLeiosDbConfig = LeiosDbSQLite
     }
 
 -- | Test that the legacy flat LedgerDB snapshot config format (options directly