diff --git a/Gruntfile.js b/Gruntfile.js index a0a614b104..92cfa7771e 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -389,7 +389,7 @@ module.exports = (grunt) => { // The `--require` flag ensures custom Babel support in our test files. test: { cmd: 'node node_modules/mocha/bin/mocha --require ./scripts/mocha-helper.js --exit -R spec --bail "./{test/,!(node_modules|ignored|dist|historical|test)/**/}*.test.js"', - options: { env: { ...process.env, ENABLE_UNSAFE_NULL_CRYPTO: 'true' } } + options: { env: process.env } }, chelDeployAll: 'find contracts -iname "*.manifest.json" | xargs -r ./node_modules/.bin/chel deploy ./data' } diff --git a/backend/auth.js b/backend/auth.js index dfd58f22c5..e2bf0b263e 100644 --- a/backend/auth.js +++ b/backend/auth.js @@ -13,7 +13,7 @@ exports.plugin = { return { authenticate: function (request, h) { const { authorization } = request.headers - if (!authorization) h.unauthenticated(Boom.unauthorized('Missing authorization')) + if (!authorization) return h.unauthenticated(Boom.unauthorized('Missing authorization')) let [scheme, json] = authorization.split(/\s+/) // NOTE: if you want to add any signature verification, do it here diff --git a/backend/routes.js b/backend/routes.js index ad973008e6..a4dd29225a 100644 --- a/backend/routes.js +++ b/backend/routes.js @@ -41,23 +41,27 @@ route.POST('/event', { }, async function (request, h) { try { console.debug('/event handler') - const entry = GIMessage.deserialize(request.payload) + const deserializedHEAD = GIMessage.deserializeHEAD(request.payload) try { - await sbp('backend/server/handleEntry', entry) + await sbp('backend/server/handleEntry', deserializedHEAD, request.payload) } catch (err) { - if (err.name === 'ChelErrorDBBadPreviousHEAD') { - console.error(err, chalk.bold.yellow('ChelErrorDBBadPreviousHEAD')) - const HEADinfo = await sbp('chelonia/db/latestHEADinfo', entry.contractID()) ?? { HEAD: null, height: 0 } + console.error(err, chalk.bold.yellow(err.name)) + if (err.name === 'ChelErrorDBBadPreviousHEAD' || err.name === 'ChelErrorAlreadyProcessed') { + const HEADinfo = await sbp('chelonia/db/latestHEADinfo', deserializedHEAD.contractID) ?? { HEAD: null, height: 0 } const r = Boom.conflict(err.message, { HEADinfo }) Object.assign(r.output.headers, { 'shelter-headinfo-head': HEADinfo.HEAD, 'shelter-headinfo-height': HEADinfo.height }) return r + } else if (err.name === 'ChelErrorSignatureError') { + return Boom.badData('Invalid signature') + } else if (err.name === 'ChelErrorSignatureKeyUnauthorized') { + return Boom.forbidden('Unauthorized signing key') } throw err // rethrow error } - return entry.hash() + return deserializedHEAD.hash } catch (err) { logger.error(err, 'POST /event', err.message) return err diff --git a/backend/server.js b/backend/server.js index eb32378b14..70e0c9e72d 100644 --- a/backend/server.js +++ b/backend/server.js @@ -3,7 +3,6 @@ import sbp from '@sbp/sbp' import Hapi from '@hapi/hapi' import initDB from './database.js' -import { GIMessage } from '~/shared/domains/chelonia/GIMessage.js' import { SERVER_RUNNING } from './events.js' import { SERVER_INSTANCE, PUBSUB_INSTANCE } from './instance-keys.js' import { @@ -16,6 +15,8 @@ import { } from './pubsub.js' import { pushServerActionhandlers } from './push.js' import chalk from 'chalk' +import '~/shared/domains/chelonia/chelonia.js' +import { SERVER } from '~/shared/domains/chelonia/presets.js' const { CONTRACTS_VERSION, GI_VERSION } = process.env @@ -61,17 +62,75 @@ hapi.ext({ sbp('okTurtles.data/set', SERVER_INSTANCE, hapi) sbp('sbp/selectors/register', { - 'backend/server/broadcastEntry': async function (entry: GIMessage) { + 'backend/server/persistState': async function (deserializedHEAD: Object, entry: string) { + const contractID = deserializedHEAD.contractID + const cheloniaState = sbp('chelonia/private/state') + // If the contract has been removed or the height hasn't been updated, + // there's nothing to persist + if (!cheloniaState.contracts[contractID] || cheloniaState.contracts[contractID].height < deserializedHEAD.head.height) { + return + } + // If the current HEAD is not what we expect, don't save (the state could + // have been updated by a later message). This ensures that we save the + // latest state and also reduces the number of write operations + if (cheloniaState.contracts[contractID].HEAD === deserializedHEAD.hash) { + // Extract the parts of the state relevant to this contract + const state = { + contractState: cheloniaState[contractID], + cheloniaContractInfo: cheloniaState.contracts[contractID] + } + // Save the state under a 'contract partition' key, so that updating a + // contract doesn't require saving the entire state. + // Although it's not important for the server right now, this will fail to + // persist changes to the state for other contracts. + // For example, when watching foreign keys, this happens: whenever a + // foreign key for contract A is added to contract B, the private state + // for both contract A and B is updated (when both contracts are being + // monitored by Chelonia). However, here in this case, the updated state + // for contract A will not be saved immediately here, and it will only be + // saved if some other event happens later on contract A. + // TODO: If, in the future, processing a message affects other contracts + // in a way that is meaningful to the server, there'll need to be a way + // to detect these changes as well. One example could be, expanding on the + // previous scenario, if we decide that the server should enforce key + // rotations, so that updating a foreign key 'locks' that contract until + // the foreign key is rotated or deleted. For this to work reliably, we'd + // need to ensure that the state for both contract B and contract A are + // saved when the foreign key gets added to contract B. + await sbp('chelonia/db/set', '_private_cheloniaState_' + contractID, JSON.stringify(state)) + } + // If this is a new contract, we also need to add it to the index, which + // is used when starting up the server to know which keys to fetch. + // In the future, consider having a multi-level index, since the index can + // get pretty large. + if (contractID === deserializedHEAD.hash) { + // We want to ensure that the index is updated atomically (i.e., if there + // are multiple new contracts, all of them should be added), so a queue + // is needed for the load & store operation. + await sbp('okTurtles.eventQueue/queueEvent', 'update-contract-indices', async () => { + const currentIndex = await sbp('chelonia/db/get', '_private_cheloniaState_index') + // Add the current contract ID to the contract index. Entries in the + // index are separated by \x00 (NUL). The index itself is used to know + // which entries to load. + const updatedIndex = `${currentIndex ? `${currentIndex}\x00` : ''}${contractID}` + await sbp('chelonia/db/set', '_private_cheloniaState_index', updatedIndex) + }) + } + }, + 'backend/server/broadcastEntry': async function (deserializedHEAD: Object, entry: string) { const pubsub = sbp('okTurtles.data/get', PUBSUB_INSTANCE) - const pubsubMessage = createMessage(NOTIFICATION_TYPE.ENTRY, entry.serialize()) - const subscribers = pubsub.enumerateSubscribers(entry.contractID()) - console.debug(chalk.blue.bold(`[pubsub] Broadcasting ${entry.description()}`)) + const pubsubMessage = createMessage(NOTIFICATION_TYPE.ENTRY, entry) + const subscribers = pubsub.enumerateSubscribers(deserializedHEAD.contractID) + console.debug(chalk.blue.bold(`[pubsub] Broadcasting ${deserializedHEAD.description()}`)) await pubsub.broadcast(pubsubMessage, { to: subscribers }) }, - 'backend/server/handleEntry': async function (entry: GIMessage) { - sbp('okTurtles.data/get', PUBSUB_INSTANCE).channels.add(entry.contractID()) - await sbp('chelonia/db/addEntry', entry) - await sbp('backend/server/broadcastEntry', entry) + 'backend/server/handleEntry': async function (deserializedHEAD: Object, entry: string) { + const contractID = deserializedHEAD.contractID + sbp('okTurtles.data/get', PUBSUB_INSTANCE).channels.add(contractID) + await sbp('chelonia/private/in/enqueueHandleEvent', contractID, entry) + // Persist the Chelonia state after processing a message + await sbp('backend/server/persistState', deserializedHEAD, entry) + await sbp('backend/server/broadcastEntry', deserializedHEAD, entry) }, 'backend/server/stop': function () { return hapi.stop() @@ -119,6 +178,25 @@ sbp('okTurtles.data/set', PUBSUB_INSTANCE, createServer(hapi.listener, { })) ;(async function () { + await initDB() + await sbp('chelonia/configure', SERVER) + // Load the saved Chelonia state + // First, get the contract index + const savedStateIndex = await sbp('chelonia/db/get', '_private_cheloniaState_index') + if (savedStateIndex) { + // Now, we contract the contract state by reading each contract state + // partition + const recoveredState = Object.create(null) + recoveredState.contracts = Object.create(null) + await Promise.all(savedStateIndex.split('\x00').map(async (contractID) => { + const cpSerialized = await sbp('chelonia/db/get', `_private_cheloniaState_${contractID}`) + if (!cpSerialized) return + const cp = JSON.parse(cpSerialized) + recoveredState[contractID] = cp.contractState + recoveredState.contracts[contractID] = cp.cheloniaContractInfo + })) + Object.assign(sbp('chelonia/private/state'), recoveredState) + } // https://hapi.dev/tutorials/plugins await hapi.register([ { plugin: require('./auth.js') }, @@ -130,7 +208,6 @@ sbp('okTurtles.data/set', PUBSUB_INSTANCE, createServer(hapi.listener, { // } // } ]) - await initDB() require('./routes.js') await hapi.start() console.info('Backend server running at:', hapi.info.uri) diff --git a/frontend/model/captureLogs.js b/frontend/model/captureLogs.js index c62aa208ff..64f634dab2 100644 --- a/frontend/model/captureLogs.js +++ b/frontend/model/captureLogs.js @@ -61,7 +61,16 @@ function captureLogEntry (type, ...args) { msg: args.map((arg) => { try { return JSON.parse( - JSON.stringify(arg instanceof Error ? (arg.stack ?? arg.message) : arg) + JSON.stringify(arg, (_, v) => { + if (v instanceof Error) { + return { + name: v.name, + message: v.message, + stack: v.stack + } + } + return v + }) ) } catch (e) { return `[captureLogs failed to stringify argument of type '${typeof arg}'. Err: ${e.message}]` diff --git a/frontend/model/contracts/chatroom.js b/frontend/model/contracts/chatroom.js index 79bb3dc883..d938fdf495 100644 --- a/frontend/model/contracts/chatroom.js +++ b/frontend/model/contracts/chatroom.js @@ -172,7 +172,11 @@ sbp('chelonia/defineContract', { throw new Error('The new member must be given either explicitly or implcitly with an inner signature') } if (!state.onlyRenderMessage) { - if (state.members[memberID]) { + // For private chatrooms, group members can see the '/join' actions + // but nothing else. Because of this, `state.members` may be missing + if (!state.members) { + Vue.set(state, 'members', {}) + } else if (state.members[memberID]) { throw new Error(`Can not join the chatroom which ${memberID} is already part of`) } @@ -300,6 +304,7 @@ sbp('chelonia/defineContract', { if (!state.onlyRenderMessage) { if (!state.members) { console.error('Missing state.members: ' + JSON.stringify(state)) + throw new Error('Missing members state') } if (!state.members[memberID]) { throw new Error(`Can not leave the chatroom ${contractID} which ${memberID} is not part of`) diff --git a/frontend/views/components/Avatar.vue b/frontend/views/components/Avatar.vue index 206132e28d..a423581280 100644 --- a/frontend/views/components/Avatar.vue +++ b/frontend/views/components/Avatar.vue @@ -31,7 +31,8 @@ export default ({ }, mounted () { console.log(`Avatar under ${this.$parent.$vnode.tag} blobURL:`, this.blobURL, 'src:', this.src) - if (typeof this.src === 'object') { + // typeof null === 'object', so both checks are needed + if (this.src && typeof this.src === 'object') { this.downloadFile(this.src).catch((e) => { console.error('[Avatar.vue] Error in downloadFile', e) }) @@ -81,7 +82,7 @@ export default ({ }, watch: { src (to) { - if (typeof to === 'object') { + if (to && typeof to === 'object') { this.downloadFile(to).catch((e) => { console.error('[Avatar.vue] Error in downloadFile', e) }) diff --git a/shared/domains/chelonia/GIMessage.js b/shared/domains/chelonia/GIMessage.js index 5b98708fde..8dcf69bab2 100644 --- a/shared/domains/chelonia/GIMessage.js +++ b/shared/domains/chelonia/GIMessage.js @@ -324,6 +324,10 @@ export class GIMessage { }, get contractID () { return result.head?.contractID ?? result.hash + }, + description (): string { + const type = this.head.op + return `` } } return result diff --git a/shared/domains/chelonia/chelonia.js b/shared/domains/chelonia/chelonia.js index e7054d3d04..3e0c78ca53 100644 --- a/shared/domains/chelonia/chelonia.js +++ b/shared/domains/chelonia/chelonia.js @@ -205,8 +205,19 @@ export default (sbp('sbp/selectors/register', { whitelisted: (action: string): boolean => !!this.whitelistedActions[action], reactiveSet: (obj, key, value) => { obj[key] = value; return value }, // example: set to Vue.set reactiveDel: (obj, key) => { delete obj[key] }, + // acceptAllMessages disables checking whether we are expecting a message + // or not for processing + acceptAllMessages: false, skipActionProcessing: false, skipSideEffects: false, + // Strict processing will treat all processing errors as unrecoverable + // This is useful, e.g., in the server, to prevent invalid messages from + // being added to the database + strictProcessing: false, + // Strict ordering will throw on past events with ChelErrorAlreadyProcessed + // Similarly, future events will not be reingested and will throw + // with ChelErrorDBBadPreviousHEAD + strictOrdering: false, connectionOptions: { maxRetries: Infinity, // See https://github.com/okTurtles/group-income/issues/1183 reconnectOnTimeout: true, // can be enabled since we are not doing auth via web sockets diff --git a/shared/domains/chelonia/encryptedData.js b/shared/domains/chelonia/encryptedData.js index d30e83ba89..cdd27cf6a5 100644 --- a/shared/domains/chelonia/encryptedData.js +++ b/shared/domains/chelonia/encryptedData.js @@ -295,6 +295,10 @@ export const isRawEncryptedData = (data: any): boolean => { export const unwrapMaybeEncryptedData = (data: any): { encryptionKeyId: string | null, data: any } | void => { if (isEncryptedData(data)) { + // If not running on a browser, we don't decrypt data to avoid filling the + // logs with unable to decrypt messages. + // This variable is set in Gruntfile.js for web builds + if (process.env.BUILD !== 'web') return try { return { encryptionKeyId: data.encryptionKeyId, diff --git a/shared/domains/chelonia/errors.js b/shared/domains/chelonia/errors.js index c859dc8eb6..be9c008634 100644 --- a/shared/domains/chelonia/errors.js +++ b/shared/domains/chelonia/errors.js @@ -17,6 +17,7 @@ export const ChelErrorGenerator = ( }: any): typeof Error) export const ChelErrorWarning: typeof Error = ChelErrorGenerator('ChelErrorWarning') +export const ChelErrorAlreadyProcessed: typeof Error = ChelErrorGenerator('ChelErrorAlreadyProcessed') export const ChelErrorDBBadPreviousHEAD: typeof Error = ChelErrorGenerator('ChelErrorDBBadPreviousHEAD') export const ChelErrorDBConnection: typeof Error = ChelErrorGenerator('ChelErrorDBConnection') export const ChelErrorUnexpected: typeof Error = ChelErrorGenerator('ChelErrorUnexpected') @@ -24,4 +25,5 @@ export const ChelErrorUnrecoverable: typeof Error = ChelErrorGenerator('ChelErro export const ChelErrorDecryptionError: typeof Error = ChelErrorGenerator('ChelErrorDecryptionError') export const ChelErrorDecryptionKeyNotFound: typeof Error = ChelErrorGenerator('ChelErrorDecryptionKeyNotFound', ChelErrorDecryptionError) export const ChelErrorSignatureError: typeof Error = ChelErrorGenerator('ChelErrorSignatureError') +export const ChelErrorSignatureKeyUnauthorized: typeof Error = ChelErrorGenerator('ChelErrorSignatureKeyUnauthorized', ChelErrorSignatureError) export const ChelErrorSignatureKeyNotFound: typeof Error = ChelErrorGenerator('ChelErrorSignatureKeyNotFound', ChelErrorSignatureError) diff --git a/shared/domains/chelonia/internals.js b/shared/domains/chelonia/internals.js index 48fa6953bc..b0d7457117 100644 --- a/shared/domains/chelonia/internals.js +++ b/shared/domains/chelonia/internals.js @@ -11,7 +11,7 @@ import { deserializeKey, keyId, verifySignature } from './crypto.js' import './db.js' import { encryptedIncomingData, encryptedOutgoingData, unwrapMaybeEncryptedData } from './encryptedData.js' import type { EncryptedData } from './encryptedData.js' -import { ChelErrorUnrecoverable, ChelErrorWarning } from './errors.js' +import { ChelErrorUnrecoverable, ChelErrorWarning, ChelErrorDBBadPreviousHEAD, ChelErrorAlreadyProcessed } from './errors.js' import { CONTRACTS_MODIFIED, CONTRACT_HAS_RECEIVED_KEYS, CONTRACT_IS_SYNCING, EVENT_HANDLED, EVENT_PUBLISHED, EVENT_PUBLISHING_ERROR } from './events.js' import { findKeyIdByName, findSuitablePublicKeyIds, findSuitableSecretKeyId, getContractIDfromKeyId, keyAdditionProcessor, recreateEvent, validateKeyPermissions, validateKeyAddPermissions, validateKeyDelPermissions, validateKeyUpdatePermissions } from './utils.js' import { isSignedData, signedIncomingData } from './signedData.js' @@ -601,7 +601,7 @@ export default (sbp('sbp/selectors/register', { [GIMessage.OP_ATOMIC] (v: GIOpAtomic) { v.forEach((u) => { if (u[0] === GIMessage.OP_ATOMIC) throw new Error('Cannot nest OP_ATOMIC') - if (!validateKeyPermissions(state, signingKeyId, u[0], u[1], direction)) { + if (!validateKeyPermissions(config, state, signingKeyId, u[0], u[1], direction)) { throw new Error('Inside OP_ATOMIC: no matching signing key was defined') } opFns[u[0]](u[1]) @@ -623,7 +623,6 @@ export default (sbp('sbp/selectors/register', { if (!config.skipActionProcessing) { opFns[GIMessage.OP_ACTION_UNENCRYPTED](v.valueOf()) } - console.log('OP_ACTION_ENCRYPTED: skipped action processing') }, [GIMessage.OP_ACTION_UNENCRYPTED] (v: GIOpActionUnencrypted) { if (!config.skipActionProcessing) { @@ -1022,7 +1021,7 @@ export default (sbp('sbp/selectors/register', { }, [GIMessage.OP_PROTOCOL_UPGRADE]: notImplemented } - if (!this.manifestToContract[manifestHash]) { + if (!this.config.skipActionProcessing && !this.manifestToContract[manifestHash]) { const rootState = sbp(this.config.stateSelector) const contractName = has(rootState.contracts, contractID) ? rootState.contracts[contractID].type @@ -1059,7 +1058,7 @@ export default (sbp('sbp/selectors/register', { // Verify that the signing key is found, has the correct purpose and is // allowed to sign this particular operation - if (!validateKeyPermissions(stateForValidation, signingKeyId, opT, opV, direction)) { + if (!validateKeyPermissions(config, stateForValidation, signingKeyId, opT, opV, direction)) { throw new Error('No matching signing key was defined') } @@ -1273,6 +1272,7 @@ export default (sbp('sbp/selectors/register', { const contractState = rootState[contractID] const keysToDelete = [] + const keysToUpdate = [] pendingWatch.forEach(([keyName, externalId]) => { // Does the key exist? If not, it has probably been removed and instead @@ -1281,6 +1281,10 @@ export default (sbp('sbp/selectors/register', { if (!keyId) { keysToDelete.push(externalId) return + } else if (keyId !== externalId) { + // Or, the key has been updated and we need to update it in the external + // contract as well + keysToUpdate.push(externalId) } // Add keys to watchlist as another contract is waiting on these @@ -1295,7 +1299,7 @@ export default (sbp('sbp/selectors/register', { // If there are keys that need to be revoked, queue an event to handle the // deletion - if (keysToDelete.length) { + if (keysToDelete.length || keysToUpdate.length) { if (!externalContractState._volatile) { this.config.reactiveSet(externalContractState, '_volatile', Object.create(null)) } @@ -1303,23 +1307,91 @@ export default (sbp('sbp/selectors/register', { this.config.reactiveSet(externalContractState._volatile, 'pendingKeyRevocations', Object.create(null)) } keysToDelete.forEach((id) => this.config.reactiveSet(externalContractState._volatile.pendingKeyRevocations, id, 'del')) + keysToUpdate.forEach((id) => this.config.reactiveSet(externalContractState._volatile.pendingKeyRevocations, id, true)) - sbp('chelonia/private/queueEvent', externalContractID, ['chelonia/private/deleteRevokedKeys', externalContractID]).catch((e) => { - console.error(`Error at deleteRevokedKeys for contractID ${contractID} and externalContractID ${externalContractID}`, e) + sbp('chelonia/private/queueEvent', externalContractID, ['chelonia/private/deleteOrRotateRevokedKeys', externalContractID]).catch((e) => { + console.error(`Error at deleteOrRotateRevokedKeys for contractID ${contractID} and externalContractID ${externalContractID}`, e) }) } }, - 'chelonia/private/deleteRevokedKeys': function (contractID: string) { + // The following function gets called when we start watching a contract for + // foreign keys for the first time, and it ensures that, at the point the + // watching starts, keys are in sync between the two contracts (later on, + // this will be handled automatically for incoming OP_KEY_DEL and + // OP_KEY_UPDATE). + // For any given foreign key, there are three possible states: + // 1. The key is in sync with the foreign contract. In this case, there's + // nothing left to do. + // 2. The key has been rotated in the foreign contract (replaced by another + // key of the same name). We need to mirror this operation manually + // since watching only affects new messages we receive. + // 3. The key has been removed in the foreign contract. We also need to + // mirror the operation. + 'chelonia/private/deleteOrRotateRevokedKeys': function (contractID: string) { const rootState = sbp(this.config.stateSelector) const contractState = rootState[contractID] const pendingKeyRevocations = contractState?._volatile.pendingKeyRevocations if (!pendingKeyRevocations || Object.keys(pendingKeyRevocations).length === 0) return + // First, we handle keys that have been rotated + const keysToUpdate: string[] = Object.entries(pendingKeyRevocations).filter(([, v]) => v === true).map(([id]) => id) + + // Aggregate the keys that we can update to send them in a single operation + const [, keyUpdateSigningKeyId, keyUpdateArgs] = keysToUpdate.reduce((acc, keyId) => { + const key = contractState._vm?.authorizedKeys?.[keyId] + if (!key || !key.foreignKey) return acc + const foreignKey = String(key.foreignKey) + const fkUrl = new URL(foreignKey) + const foreignContractID = fkUrl.pathname + const foreignKeyName = fkUrl.searchParams.get('keyName') + if (!foreignKeyName) throw new Error('Missing foreign key name') + const foreignState = rootState[foreignContractID] + if (!foreignState) return acc + const fKeyId = findKeyIdByName(foreignState, foreignKeyName) + if (!fKeyId) { + // Key was deleted + pendingKeyRevocations.find(([id]) => id === keyId)[1] = 'del' + return acc + } + + const [currentRingLevel, currentSigningKeyId, currentKeyArgs] = acc + const ringLevel = Math.min(currentRingLevel, key.ringLevel ?? Number.POSITIVE_INFINITY) + if (ringLevel >= currentRingLevel) { + (currentKeyArgs: any).push({ + name: key.name, + oldKeyId: keyId, + id: fKeyId, + data: foreignState._vm.authorizedKeys[fKeyId].data + }) + return [currentRingLevel, currentSigningKeyId, currentKeyArgs] + } else if (Number.isFinite(ringLevel)) { + const signingKeyId = findSuitableSecretKeyId(contractState, [GIMessage.OP_KEY_DEL], ['sig'], ringLevel) + if (signingKeyId) { + (currentKeyArgs: any).push({ + keyId + }) + return [ringLevel, signingKeyId, currentKeyArgs] + } + } + return acc + }, [Number.POSITIVE_INFINITY, '', []]) + + if (keyUpdateArgs.length !== 0) { + const contractName = contractState._vm.type + + // This is safe to do without await because it's sending an operation + // Using await could deadlock when retrying to send the message + sbp('chelonia/out/keyUpdate', { contractID, contractName, data: keyUpdateArgs, signingKeyId: keyUpdateSigningKeyId }).catch(e => { + console.error(`[chelonia/private/deleteOrRotateRevokedKeys] Error sending OP_KEY_UPDATE for ${contractID}`, e.message) + }) + } + + // And then, we handle keys that have been deleted const keysToDelete = Object.entries(pendingKeyRevocations).filter(([, v]) => v === 'del').map(([id]) => id) // Aggregate the keys that we can delete to send them in a single operation - const [, signingKeyId, keyIds] = keysToDelete.reduce((acc, keyId) => { + const [, keyDelSigningKeyId, keyIdsToDelete] = keysToDelete.reduce((acc, keyId) => { const [currentRingLevel, currentSigningKeyId, currentKeyIds] = acc const ringLevel = Math.min(currentRingLevel, contractState._vm?.authorizedKeys?.[keyId]?.ringLevel ?? Number.POSITIVE_INFINITY) if (ringLevel >= currentRingLevel) { @@ -1335,15 +1407,15 @@ export default (sbp('sbp/selectors/register', { return acc }, [Number.POSITIVE_INFINITY, '', []]) - if (keyIds.length === 0) return - - const contractName = contractState._vm.type + if (keyIdsToDelete.length !== 0) { + const contractName = contractState._vm.type - // This is safe to do without await because it's sending an operation - // Using await could deadlock when retrying to send the message - sbp('chelonia/out/keyDel', { contractID, contractName, data: keyIds, signingKeyId }).catch(e => { - console.error(`[chelonia/private/deleteRevokedKeys] Error sending OP_KEY_DEL for ${contractID}`) - }) + // This is safe to do without await because it's sending an operation + // Using await could deadlock when retrying to send the message + sbp('chelonia/out/keyDel', { contractID, contractName, data: keyIdsToDelete, signingKeyId: keyDelSigningKeyId }).catch(e => { + console.error(`[chelonia/private/deleteRevokedKeys] Error sending OP_KEY_DEL for ${contractID}`, e.message) + }) + } }, 'chelonia/private/respondToAllKeyRequests': function (contractID: string) { const state = sbp(this.config.stateSelector) @@ -1554,7 +1626,7 @@ export default (sbp('sbp/selectors/register', { // Errors in side effects result in dropped messages to be reprocessed try { // verify we're expecting to hear from this contract - if (!this.pending.some((entry) => entry?.contractID === contractID) && !this.subscriptionSet.has(contractID)) { + if (!this.config.acceptAllMessages && !this.pending.some((entry) => entry?.contractID === contractID) && !this.subscriptionSet.has(contractID)) { console.warn(`[chelonia] WARN: ignoring unexpected event for ${contractID}:`, rawMessage) return } @@ -1596,10 +1668,10 @@ export default (sbp('sbp/selectors/register', { } preHandleEvent?.(message) // the order the following actions are done is critically important! - // first we make sure we save this message to the db + // first we make sure we can save this message to the db // if an exception is thrown here we do not need to revert the state // because nothing has been processed yet - const proceed = await handleEvent.addMessageToDB(message) + const proceed = handleEvent.checkMessageOrdering.call(this, message) if (proceed === false) return // If the contract was marked as dirty, we stop processing @@ -1630,6 +1702,9 @@ export default (sbp('sbp/selectors/register', { } // we revert any changes to the contract state that occurred, ignoring this mutation console.warn(`[chelonia] Error processing ${message.description()}: ${message.serialize()}. Any side effects will be skipped!`) + if (this.config.strictProcessing) { + throw e + } processingErrored = e?.name !== 'ChelErrorWarning' this.config.hooks.processError?.(e, message, getMsgMeta(message, contractID, state)) // special error that prevents the head from being updated, effectively killing the contract @@ -1659,9 +1734,14 @@ export default (sbp('sbp/selectors/register', { // possible in the code to reduce the chances of still ending up with // an inconsistent state if a sudden failure happens while this code // is executing. In particular, everything in between should be synchronous. + // This block will apply all the changes related to modifying the state + // after an event has been processed: + // 1. Adding the messge to the DB + // 2. Applying changes to the contract state + // 3. Applying changes to rootState.contracts try { const state = sbp(this.config.stateSelector) - handleEvent.applyProcessResult.call(this, { message, state, contractState: contractStateCopy, processingErrored, postHandleEvent }) + await handleEvent.applyProcessResult.call(this, { message, state, contractState: contractStateCopy, processingErrored, postHandleEvent }) } catch (e) { console.error(`[chelonia] ERROR '${e.name}' for ${message.description()} marking the event as processed: ${e.message}`, e, { message: message.serialize() }) } @@ -1677,39 +1757,67 @@ export default (sbp('sbp/selectors/register', { } }): string[]) -const eventsToReinjest = [] +const eventsToReingest = [] const reprocessDebounced = debounce((contractID) => sbp('chelonia/contract/sync', contractID, { force: true }).catch((e) => { console.error(`[chelonia] Error at reprocessDebounced for ${contractID}`) }), 1000) const handleEvent = { - async addMessageToDB (message: GIMessage) { + checkMessageOrdering (message: GIMessage) { const contractID = message.contractID() const hash = message.hash() - try { - await sbp('chelonia/db/addEntry', message) - const reprocessIdx = eventsToReinjest.indexOf(hash) - if (reprocessIdx !== -1) { - console.warn(`[chelonia] WARN: successfully reinjested ${message.description()}`) - eventsToReinjest.splice(reprocessIdx, 1) + const height = message.height() + const state = sbp(this.config.stateSelector) + // The latest height we want to use is the one from `state.contracts` and + // not the one from the DB. The height in the state reflects the latest + // message that's been processed, which is desired here. On the other hand, + // the DB function includes the latest known message for that contract, + // which can be ahead of the latest message processed. + const latestProcessedHeight = state.contracts[contractID]?.height + if (!Number.isSafeInteger(height)) { + throw new ChelErrorDBBadPreviousHEAD(`Message ${hash} in contract ${contractID} has an invalid height.`) + } + // Avoid re-processing already processed messages + if ( + message.isFirstMessage() + // If this is the first message, the height is is expected not to exist + ? latestProcessedHeight != null + // If this isn't the first message, the height must not be lower than the + // current's message height. The check is negated to handle NaN values + : !(latestProcessedHeight < height) + ) { + // The web client may sometimes get repeated messages. If strict ordering + // isn't enabled, instead of throwing we return false. + // On the other hand, the server must enforce strict ordering. + if (!this.config.strictOrdering) { + return false } - } catch (e) { - if (e.name === 'ChelErrorDBBadPreviousHEAD') { - // sometimes we simply miss messages, it's not clear why, but it happens - // in rare cases. So we attempt to re-sync this contract once - if (eventsToReinjest.length > 100) { - throw new ChelErrorUnrecoverable('more than 100 different bad previousHEAD errors') - } - if (!eventsToReinjest.includes(hash)) { - console.warn(`[chelonia] WARN bad previousHEAD for ${message.description()}, will attempt to re-sync contract to reinjest message`) - eventsToReinjest.push(hash) - reprocessDebounced(contractID) - return false // ignore the error for now - } else { - console.error(`[chelonia] ERROR already attempted to reinjest ${message.description()}, will not attempt again!`) - } + throw new ChelErrorAlreadyProcessed(`Message ${hash} with height ${height} in contract ${contractID} has already been processed. Current height: ${latestProcessedHeight}.`) + } + // If the message is from the future, add it to eventsToReingest + if ((latestProcessedHeight + 1) < height) { + if (this.config.strictOrdering) { + throw new ChelErrorDBBadPreviousHEAD(`Unexpected message ${hash} with height ${height} in contract ${contractID}: height is too high. Current height: ${latestProcessedHeight}.`) } - throw e + // sometimes we simply miss messages, it's not clear why, but it happens + // in rare cases. So we attempt to re-sync this contract once + if (eventsToReingest.length > 100) { + throw new ChelErrorUnrecoverable('more than 100 different bad previousHEAD errors') + } + if (!eventsToReingest.includes(hash)) { + console.warn(`[chelonia] WARN bad previousHEAD for ${message.description()}, will attempt to re-sync contract to reingest message`) + eventsToReingest.push(hash) + reprocessDebounced(contractID) + return false // ignore the error for now + } else { + console.error(`[chelonia] ERROR already attempted to reingest ${message.description()}, will not attempt again!`) + throw new ChelErrorDBBadPreviousHEAD(`Already attempted to reingest ${hash}`) + } + } + const reprocessIdx = eventsToReingest.indexOf(hash) + if (reprocessIdx !== -1) { + console.warn(`[chelonia] WARN: successfully reingested ${message.description()}`) + eventsToReingest.splice(reprocessIdx, 1) } }, async processMutation (message: GIMessage, state: Object, internalSideEffectStack?: any[]) { @@ -1786,11 +1894,12 @@ const handleEvent = { } }) }, - applyProcessResult ({ message, state, contractState, processingErrored, postHandleEvent }: { message: GIMessage, state: Object, contractState: Object, processingErrored: boolean, postHandleEvent: ?Function }) { + async applyProcessResult ({ message, state, contractState, processingErrored, postHandleEvent }: { message: GIMessage, state: Object, contractState: Object, processingErrored: boolean, postHandleEvent: ?Function }) { const contractID = message.contractID() const hash = message.hash() const height = message.height() + await sbp('chelonia/db/addEntry', message) if (!processingErrored) { // Once side-effects are called, we apply changes to the state. // This means, as mentioned above, that retrieving the contract state diff --git a/shared/domains/chelonia/presets.js b/shared/domains/chelonia/presets.js new file mode 100644 index 0000000000..a182f0e92e --- /dev/null +++ b/shared/domains/chelonia/presets.js @@ -0,0 +1,19 @@ +// Right now, we only have a single preset, for the server. If this remains the +// case and only the server is special regarding configuration, consider +// introducing a `server: true` key to `chelonia/confgure` instead. + +export const SERVER = { + // We don't check the subscriptionSet in the server because we accpt new + // contract registrations, and are also not subcribed to contracts the same + // way clients are + acceptAllMessages: true, + // The server also doesn't process actions + skipActionProcessing: true, + // The previous setting implies this one, which we set to be on the safe side + skipSideEffects: true, + // If an error occurs during processing, the message is rejected rather than + // ignored + strictProcessing: true, + // The server expects events to be received in order (no past or future events) + strictOrdering: true +} diff --git a/shared/domains/chelonia/signedData.js b/shared/domains/chelonia/signedData.js index 8894907011..6cfdac6ff0 100644 --- a/shared/domains/chelonia/signedData.js +++ b/shared/domains/chelonia/signedData.js @@ -3,7 +3,7 @@ import { has } from '~/frontend/model/contracts/shared/giLodash.js' import { blake32Hash } from '~/shared/functions.js' import type { Key } from './crypto.js' import { deserializeKey, keyId, serializeKey, sign, verifySignature } from './crypto.js' -import { ChelErrorSignatureError, ChelErrorSignatureKeyNotFound, ChelErrorUnexpected } from './errors.js' +import { ChelErrorSignatureError, ChelErrorSignatureKeyNotFound, ChelErrorSignatureKeyUnauthorized } from './errors.js' const rootStateFn = () => sbp('chelonia/rootState') @@ -115,7 +115,7 @@ const verifySignatureData = function (height: number, data: any, additionalData: if (!designatedKey || (height > designatedKey._notAfterHeight) || (height < designatedKey._notBeforeHeight) || !designatedKey.purpose.includes( 'sig' )) { - throw new ChelErrorUnexpected( + throw new ChelErrorSignatureKeyUnauthorized( `Key ${sKeyId} is unauthorized or expired for the current contract` ) } @@ -226,20 +226,14 @@ export const signedOutgoingDataWithRawKey = (key: Key, data: T, height?: numb export const signedIncomingData = (contractID: string, state: ?Object, data: any, height: number, additionalData: string, mapperFn?: Function): SignedData => { const stringValueFn = () => data let verifySignedValue - // TODO: Temporary until the server can validate signatures - const verifySignedValueFn = process.env.BUILD === 'web' - ? () => { - if (verifySignedValue) { - return verifySignedValue[1] - } - verifySignedValue = verifySignatureData.call(state || rootStateFn()[contractID], height, data, additionalData) - if (mapperFn) verifySignedValue[1] = mapperFn(verifySignedValue[1]) - return verifySignedValue[1] - } - : () => { - const signedValue = JSON.parse(data._signedData[0]) - return mapperFn ? mapperFn(signedValue) : signedValue - } + const verifySignedValueFn = () => { + if (verifySignedValue) { + return verifySignedValue[1] + } + verifySignedValue = verifySignatureData.call(state || rootStateFn()[contractID], height, data, additionalData) + if (mapperFn) verifySignedValue[1] = mapperFn(verifySignedValue[1]) + return verifySignedValue[1] + } return wrapper({ get signingKeyId () { diff --git a/shared/domains/chelonia/utils.js b/shared/domains/chelonia/utils.js index b65cfc95ef..5a52ce883e 100644 --- a/shared/domains/chelonia/utils.js +++ b/shared/domains/chelonia/utils.js @@ -110,7 +110,7 @@ const validateActionPermissions = (signingKey: GIKey, state: Object, opT: string return true } -export const validateKeyPermissions = (state: Object, signingKeyId: string, opT: string, opV: GIOpValue, direction?: string): boolean => { +export const validateKeyPermissions = (config: Object, state: Object, signingKeyId: string, opT: string, opV: GIOpValue, direction?: string): boolean => { const signingKey = state._vm?.authorizedKeys?.[signingKeyId] if ( !signingKey || @@ -135,6 +135,7 @@ export const validateKeyPermissions = (state: Object, signingKeyId: string, opT: } if ( + !config.skipActionProcessing && opT === GIMessage.OP_ACTION_ENCRYPTED && !validateActionPermissions(signingKey, state, opT, (opV: any).valueOf(), direction) ) { @@ -278,18 +279,16 @@ export const keyAdditionProcessor = function (hash: string, keys: (GIKey | Encry key.meta.private.content && !sbp('chelonia/haveSecretKey', key.id, !key.meta.private.transient) ) { - try { - decryptedKey = key.meta.private.content.valueOf() - decryptedKeys.push([key.id, decryptedKey]) - storeSecretKey(key, decryptedKey) - } catch (e) { - console.warn(`Secret key decryption error '${e.message || e}':`, e) - // Ricardo feels this is an ambiguous situation, however if we rethrow it will - // render the contract unusable because it will undo all our changes to the state, - // and it's possible that an error here shouldn't necessarily break the entire - // contract. For example, in some situations we might read a contract as - // read-only and not have the key to write to it. + const decryptedKeyResult = unwrapMaybeEncryptedData(key.meta.private.content) + // Unable to decrypt + if (!decryptedKeyResult) continue + // Data aren't encrypted + if (decryptedKeyResult.encryptionKeyId == null) { + throw new Error('Expected encrypted data but got unencrypted data for key with ID: ' + key.id) } + decryptedKey = decryptedKeyResult.data + decryptedKeys.push([key.id, decryptedKey]) + storeSecretKey(key, decryptedKey) } } diff --git a/test/backend.test.js b/test/backend.test.js index 66d1d6057b..3136d530fc 100644 --- a/test/backend.test.js +++ b/test/backend.test.js @@ -15,7 +15,7 @@ import chalk from 'chalk' import { THEME_LIGHT } from '~/frontend/model/settings/themes.js' import manifests from '~/frontend/model/contracts/manifests.json' // Using relative path to crypto.js instead of ~-path to workaround some esbuild bug -import { SNULL, keyId, keygen, serializeKey } from '../shared/domains/chelonia/crypto.js' +import { EDWARDS25519SHA512BATCH, keyId, keygen, serializeKey } from '../shared/domains/chelonia/crypto.js' // Necessary since we are going to use a WebSocket pubsub client in the backend. global.WebSocket = require('ws') @@ -121,7 +121,7 @@ describe('Full walkthrough', function () { } async function createIdentity (username, email, testFn) { - const CSK = keygen(SNULL) + const CSK = keygen(EDWARDS25519SHA512BATCH) const CSKid = keyId(CSK) const CSKp = serializeKey(CSK, false) @@ -159,7 +159,7 @@ describe('Full walkthrough', function () { return msg } function createGroup (name: string, creator: any, hooks: Object = {}): Promise { - const CSK = keygen(SNULL) + const CSK = keygen(EDWARDS25519SHA512BATCH) const CSKid = keyId(CSK) const CSKp = serializeKey(CSK, false)