Merge pull request #852 from gofractally/native-dbs

Cleanup to native databases

libraries/psibase/common/include/psibase/db.hpp

@@ -50,14 +50,7 @@ namespace psibase
       /// then that node will reject the write. If the producers
       /// accepted the write into a block, then the node will stop
       /// following the chain until it's upgraded to a newer version.
-      nativeConstrained,
-
-      /// Tables used by native code
-      ///
-      /// This database doesn't enforce constraints during write.
-      /// Only writable by privileged services, but readable by all
-      /// services.
-      nativeUnconstrained,
+      native,
 
       /// Block log
       ///

libraries/psibase/common/include/psibase/nativeTables.hpp

@@ -29,7 +29,7 @@ namespace psibase
       std::optional<std::tuple<Consensus, BlockNum>> nextConsensus;
       std::vector<BlockHeaderAuthAccount>            authServices;
 
-      static constexpr auto db = psibase::DbId::nativeUnconstrained;
+      static constexpr auto db = psibase::DbId::native;
       static auto           key() -> KeyPrefixType;
       PSIO_REFLECT(StatusRow, chainId, current, head, consensus, nextConsensus, authServices)
    };
@@ -39,7 +39,7 @@ namespace psibase
       uint32_t maxKeySize   = 128;
       uint32_t maxValueSize = 8 << 20;
 
-      static constexpr auto db = psibase::DbId::nativeConstrained;
+      static constexpr auto db = psibase::DbId::native;
 
       static auto key() -> KeyPrefixType;
       PSIO_REFLECT(ConfigRow, maxKeySize, maxValueSize)
@@ -79,7 +79,7 @@ namespace psibase
       uint32_t  numExecutionMemories = 32;
       VMOptions vmOptions;
 
-      static constexpr auto db = psibase::DbId::nativeConstrained;
+      static constexpr auto db = psibase::DbId::native;
 
       static auto key(NativeTableNum TableNum) -> KeyPrefixType;
       PSIO_REFLECT(WasmConfigRow, numExecutionMemories, vmOptions)
@@ -108,15 +108,16 @@ namespace psibase
       uint8_t     vmType    = 0;
       uint8_t     vmVersion = 0;
 
-      static constexpr auto db = psibase::DbId::nativeConstrained;
+      static constexpr auto db = psibase::DbId::native;
       auto                  key() const -> CodeKeyType;
       PSIO_REFLECT(CodeRow, codeNum, flags, codeHash, vmType, vmVersion)
    };
 
    using CodeByHashKeyType =
        std::tuple<std::uint16_t, std::uint8_t, Checksum256, std::uint8_t, std::uint8_t>;
-   auto codeByHashKey(const Checksum256& codeHash, uint8_t vmType, uint8_t vmVersion)
-       -> CodeByHashKeyType;
+   auto codeByHashKey(const Checksum256& codeHash,
+                      uint8_t            vmType,
+                      uint8_t            vmVersion) -> CodeByHashKeyType;
 
    /// where code is actually stored, duplicate services are reused
    struct CodeByHashRow
@@ -128,11 +129,11 @@ namespace psibase
       uint32_t             numRefs = 0;   // number accounts that ref this
       std::vector<uint8_t> code    = {};  // actual code, TODO: compressed
 
-      // The code table is in nativeConstrained. The native code
+      // The code table is in native. The native code
       // verifies codeHash and the key. This prevents a poison block
       // that could happen if the key->code map doesn't match the
       // key->(jitted code) map or the key->(optimized code) map.
-      static constexpr auto db = psibase::DbId::nativeConstrained;
+      static constexpr auto db = psibase::DbId::native;
       auto                  key() const -> CodeByHashKeyType;
       PSIO_REFLECT(CodeByHashRow, codeHash, vmType, vmVersion, numRefs, code)
    };
@@ -149,14 +150,15 @@ namespace psibase
 
       uint64_t blockMerkleEventNumber = 1;
 
-      // This table is in nativeConstrained. The native code blocks services
+      // This table is in native. The native code blocks services
       // from writing to this since it could break backing stores.
-      static constexpr auto db = psibase::DbId::nativeConstrained;
+      static constexpr auto db = psibase::DbId::native;
       static auto           key() -> KeyPrefixType;
       PSIO_REFLECT(DatabaseStatusRow,
                    nextHistoryEventNumber,
                    nextUIEventNumber,
-                   nextMerkleEventNumber)
+                   nextMerkleEventNumber,
+                   blockMerkleEventNumber)
    };
 
    // Notifications are sent by native code
@@ -183,7 +185,7 @@ namespace psibase
       std::vector<Action> actions;
 
       // TODO: we need a native subjective table
-      static constexpr auto db = psibase::DbId::nativeConstrained;
+      static constexpr auto db = psibase::DbId::native;
       auto                  key() const -> NotifyKeyType;
       PSIO_REFLECT(NotifyRow, type, actions)
    };

libraries/psibase/native/src/NativeFunctions.cpp

@@ -34,9 +34,7 @@ namespace psibase
                "database access disabled during proof verification or first auth");
          if (db == uint32_t(DbId::service))
             return (DbId)db;
-         if (db == uint32_t(DbId::nativeConstrained))
-            return (DbId)db;
-         if (db == uint32_t(DbId::nativeUnconstrained))
+         if (db == uint32_t(DbId::native))
             return (DbId)db;
          if (db == uint32_t(DbId::subjective))
          {
@@ -125,11 +123,7 @@ namespace psibase
 
          if (db == uint32_t(DbId::service))
             return {(DbId)db, true, true};
-         if (db == uint32_t(DbId::nativeConstrained) &&
-             (self.code.flags & CodeRow::allowWriteNative))
-            return {(DbId)db, true, true};
-         if (db == uint32_t(DbId::nativeUnconstrained) &&
-             (self.code.flags & CodeRow::allowWriteNative))
+         if (db == uint32_t(DbId::native) && (self.code.flags & CodeRow::allowWriteNative))
             return {(DbId)db, true, true};
          throw std::runtime_error("service may not write this db (" + std::to_string(db) +
                                   "), or must use another intrinsic");
@@ -158,6 +152,19 @@ namespace psibase
                                   "), or must use another intrinsic");
       }
 
+      void verifyStatusRow(psio::input_stream                key,
+                           psio::input_stream                value,
+                           std::optional<psio::input_stream> oldValue)
+      {
+         check(psio::fracpack_validate_strict<StatusRow>({value.pos, value.end}),
+               "StatusRow has invalid format");
+         // TODO: Verify that only nextConsensus has changed
+         auto expectedKey = psio::convert_to_key(statusKey());
+         check(key.remaining() == expectedKey.size() &&
+                   !std::memcmp(key.pos, expectedKey.data(), key.remaining()),
+               "StatusRow has incorrect key");
+      }
+
       void verifyCodeRow(TransactionContext&               ctx,
                          psio::input_stream                key,
                          psio::input_stream                value,
@@ -258,7 +265,9 @@ namespace psibase
          check(key.remaining() >= sizeof(table), "Unrecognized key in nativeConstrained");
          memcpy(&table, key.pos, sizeof(table));
          std::reverse((char*)&table, (char*)(&table + 1));
-         if (table == codeTable)
+         if (table == statusTable)
+            verifyStatusRow(key, value, existing);
+         else if (table == codeTable)
             verifyCodeRow(context, key, value, existing);
          else if (table == codeByHashTable)
             verifyCodeByHashRow(key, value);
@@ -483,7 +492,7 @@ namespace psibase
                       delta.valueBytes -= existing->remaining();
                    }
                    // nativeConstrained is both refundable and chargeable
-                   if (db == uint32_t(DbId::nativeConstrained))
+                   if (db == uint32_t(DbId::native))
                    {
                       verifyWriteConstrained(transactionContext, {key.data(), key.size()},
                                              {value.data(), value.size()}, existing);
@@ -545,7 +554,7 @@ namespace psibase
                           delta.records -= 1;
                           delta.keyBytes -= key.size();
                           delta.valueBytes -= existing->remaining();
-                          if (db == uint32_t(DbId::nativeConstrained))
+                          if (db == uint32_t(DbId::native))
                           {
                              verifyRemoveConstrained(transactionContext, {key.data(), key.size()},
                                                      *existing);

rust/psibase/src/boot.rs

@@ -2,7 +2,7 @@ use crate::services::{accounts, auth_delegate, producers, transact};
 use crate::{
     method_raw, new_account_action, set_auth_service_action, validate_dependencies, AccountNumber,
     Action, AnyPublicKey, Claim, ExactAccountNumber, GenesisActionData, MethodNumber,
-    PackagedService, ProducerConfigRow, SignedTransaction, Tapos, TimePointSec, Transaction,
+    PackagedService, Producer, SignedTransaction, Tapos, TimePointSec, Transaction,
 };
 use fracpack::Pack;
 use serde_bytes::ByteBuf;
@@ -19,9 +19,9 @@ macro_rules! method {
 }
 
 fn set_producers_action(name: AccountNumber, key: Claim) -> Action {
-    producers::Wrapper::pack().setProducers(vec![ProducerConfigRow {
-        producerName: name,
-        producerAuth: key,
+    producers::Wrapper::pack().setProducers(vec![Producer {
+        name: name,
+        auth: key,
     }])
 }
 

rust/psibase/src/db.rs

@@ -47,14 +47,7 @@ pub enum DbId {
     /// then that node will reject the write. If the producers
     /// accepted the write into a block, then the node will stop
     /// following the chain until it's upgraded to a newer version.
-    NativeConstrained,
-
-    /// Tables used by native code
-    ///
-    /// This database doesn't enforce constraints during write.
-    /// Only writable by privileged services, but readable by all
-    /// services.
-    NativeUnconstrained,
+    Native,
 
     /// Block log
     ///

rust/psibase/src/native_tables.rs

@@ -1,8 +1,8 @@
 #![allow(non_snake_case)]
 
 use crate::{
-    AccountNumber, BlockHeader, BlockHeaderAuthAccount, BlockInfo, BlockNum, Checksum256, Claim,
-    Consensus, DbId, Pack, ToSchema, Unpack,
+    AccountNumber, Action, BlockHeader, BlockHeaderAuthAccount, BlockInfo, BlockNum, Checksum256,
+    Consensus, DbId, Hex, Pack, ToSchema, Unpack,
 };
 use serde::{Deserialize, Serialize};
 
@@ -16,7 +16,7 @@ pub const DATABASE_STATUS_TABLE: NativeTable = 4;
 pub const TRANSACTION_WASM_CONFIG_TABLE: NativeTable = 5;
 pub const PROOF_WASM_CONFIG_TABLE: NativeTable = 6; // Also for first auth
 pub const CONFIG_TABLE: NativeTable = 7;
-pub const PRODUCER_CONFIG_TABLE: NativeTable = 8;
+pub const NOTIFY_TABLE: NativeTable = 8;
 
 pub const NATIVE_TABLE_PRIMARY_INDEX: NativeIndex = 0;
 
@@ -36,7 +36,7 @@ pub struct StatusRow {
 }
 
 impl StatusRow {
-    pub const DB: DbId = DbId::NativeUnconstrained;
+    pub const DB: DbId = DbId::Native;
 
     pub fn key(&self) -> (NativeTable, NativeIndex) {
         status_key()
@@ -51,28 +51,115 @@ pub struct ConfigRow {
 }
 
 impl ConfigRow {
-    pub const DB: DbId = DbId::NativeConstrained;
+    pub const DB: DbId = DbId::Native;
 
     pub fn key(&self) -> (NativeTable, NativeIndex) {
         (CONFIG_TABLE, NATIVE_TABLE_PRIMARY_INDEX)
     }
 }
 
-pub fn producer_config_key(producer: AccountNumber) -> (NativeTable, AccountNumber) {
-    (PRODUCER_CONFIG_TABLE, producer)
+#[derive(Debug, Clone, Pack, Unpack, ToSchema, Serialize, Deserialize)]
+#[fracpack(fracpack_mod = "fracpack")]
+pub struct VMOptions {
+    max_mutable_global_bytes: u32,
+    max_pages: u32,
+    max_table_elements: u32,
+    max_stack_bytes: u32,
+}
+
+#[derive(Debug, Clone, Pack, Unpack, ToSchema, Serialize, Deserialize)]
+#[fracpack(fracpack_mod = "fracpack")]
+pub struct WasmConfigRow {
+    numExecutionMemories: u32,
+    vmOptions: VMOptions,
+}
+
+impl WasmConfigRow {
+    pub const DB: DbId = DbId::Native;
+    pub fn key(table: NativeTable) -> (NativeTable, NativeIndex) {
+        (table, NATIVE_TABLE_PRIMARY_INDEX)
+    }
+}
+
+#[derive(Debug, Clone, Pack, Unpack, ToSchema, Serialize, Deserialize)]
+#[fracpack(fracpack_mod = "fracpack")]
+pub struct CodeRow {
+    codeNum: AccountNumber,
+    flags: u64,
+
+    codeHash: Checksum256,
+    vmType: u8,
+    vmVersion: u8,
+}
+
+impl CodeRow {
+    pub const DB: DbId = DbId::Native;
+    pub const ALLOW_SUDO: u64 = 1u64 << 0;
+    pub const ALLOW_WRITE_NATIVE: u64 = 1u64 << 1;
+    pub const IS_SUBJECTIVE: u64 = 1u64 << 2;
+    pub const ALLOW_WRITE_SUBJECTIVE: u64 = 1u64 << 3;
+    pub const CANNOT_TIME_OUT: u64 = 1u64 << 4;
+    pub const CAN_SET_TIME_LIMIT: u64 = 1u64 << 5;
+    pub const IS_AUTH_SERVICE: u64 = 1u64 << 6;
+    pub const FORCE_REPLAY: u64 = 1u64 << 7;
+    pub const ALLOW_SOCKET: u64 = 1u64 << 8;
+    pub fn key(&self) -> (NativeTable, NativeIndex, AccountNumber) {
+        (CODE_TABLE, NATIVE_TABLE_PRIMARY_INDEX, self.codeNum)
+    }
 }
 
+/// where code is actually stored, duplicate services are reused
 #[derive(Debug, Clone, Pack, Unpack, ToSchema, Serialize, Deserialize)]
 #[fracpack(fracpack_mod = "fracpack")]
-pub struct ProducerConfigRow {
-    pub producerName: AccountNumber,
-    pub producerAuth: Claim,
+pub struct CodeByHashRow {
+    codeHash: Checksum256,
+    vmType: u8,
+    vmVersion: u8,
+
+    numRefs: u32,
+    code: Hex<Vec<u8>>,
 }
 
-impl ProducerConfigRow {
-    pub const DB: DbId = DbId::NativeConstrained;
+impl CodeByHashRow {
+    pub const DB: DbId = DbId::Native;
+    pub fn key(&self) -> (NativeTable, NativeIndex, Checksum256, u8, u8) {
+        (
+            CODE_BY_HASH_TABLE,
+            NATIVE_TABLE_PRIMARY_INDEX,
+            self.codeHash.clone(),
+            self.vmType,
+            self.vmVersion,
+        )
+    }
+}
 
-    pub fn key(&self) -> (NativeTable, AccountNumber) {
-        producer_config_key(self.producerName)
+#[derive(Debug, Clone, Pack, Unpack, ToSchema, Serialize, Deserialize)]
+#[fracpack(fracpack_mod = "fracpack")]
+pub struct DatabaseStatusRow {
+    nextHistoryEventNumber: u64,
+    nextUIEventNumber: u64,
+    nextMerkleEventNumber: u64,
+
+    blockMerkleEventNumber: u64,
+}
+
+impl DatabaseStatusRow {
+    pub const DB: DbId = DbId::Native;
+    pub fn key(&self) -> (NativeTable, NativeIndex) {
+        (DATABASE_STATUS_TABLE, NATIVE_TABLE_PRIMARY_INDEX)
+    }
+}
+
+#[derive(Debug, Clone, Pack, Unpack, ToSchema, Serialize, Deserialize)]
+#[fracpack(fracpack_mod = "fracpack")]
+pub struct NotifyRow {
+    type_: u32,
+    actions: Vec<Action>,
+}
+
+impl NotifyRow {
+    pub const DB: DbId = DbId::Native;
+    pub fn key(&self) -> (NativeTable, NativeIndex) {
+        (NOTIFY_TABLE, NATIVE_TABLE_PRIMARY_INDEX)
     }
 }

rust/psibase/src/services/producers.rs

@@ -9,9 +9,13 @@ pub const PRODUCER_ACCOUNT_STRONG: AccountNumber = account!("prods-strong");
 #[crate::service(name = "producers", dispatch = false, psibase_mod = "crate")]
 #[allow(non_snake_case, unused_variables)]
 mod service {
+    #[action]
+    fn setConsensus(prods: crate::Consensus) {
+        unimplemented!();
+    }
 
     #[action]
-    fn setProducers(prods: Vec<crate::ProducerConfigRow>) {
+    fn setProducers(prods: Vec<crate::Producer>) {
         unimplemented!();
     }
 }

services/psibase_tests/event-service.cpp

@@ -9,4 +9,9 @@ psibase::EventNumber EventService::foo(std::string s, int i)
    return emit().history().e(s, i);
 }
 
+psibase::EventNumber EventService::emitMerkle(std::string s)
+{
+   return emit().merkle().m(s);
+}
+
 PSIBASE_DISPATCH(EventService)