Skip to content

Commit

Permalink
all bulkInsert functions and insert functions patched to avoid future…
Browse files Browse the repository at this point in the history 
… sql injections in archive-server and ajv validations has been added for originalTxData and archiverReceipt
  • Loading branch information
@devendra-shardeum
devendra-shardeum committed Dec 8, 2024
1 parent ead4455 commit fb7aac5
Show file tree
Hide file tree
Showing 15 changed files with 730 additions and 327 deletions.
199 changes: 22 additions & 177 deletions src/Data/Collector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import { verifyAppReceiptData } from '../shardeum/verifyAppReceiptData'
import { Cycle as DbCycle } from '../dbstore/types'
import { Utils as StringUtils } from '@shardus/types'
import { offloadReceipt } from '../primary-process'
import { verifyPayload } from '../types/ajv/Helpers'
import { AJVSchemaEnum } from '../types/enum/AJVSchemaEnum'

export let storingAccountData = false
const processedReceiptsMap: Map<string, number> = new Map()
Expand Down Expand Up @@ -276,167 +278,17 @@ const isReceiptRobust = async (
* @returns boolean
*/
export const validateArchiverReceipt = (receipt: Receipt.ArchiverReceipt): boolean => {
// Add type and field existence check
let err = Utils.validateTypes(receipt, {
tx: 'o',
cycle: 'n',
afterStates: 'a',
beforeStates: 'a',
signedReceipt: 'o',
appReceiptData: 'o?',
executionShardKey: 's',
globalModification: 'b',
})
if (err) {
Logger.mainLogger.error('Invalid receipt data', err)
return false
}
err = Utils.validateTypes(receipt.tx, {
txId: 's',
timestamp: 'n',
originalTxData: 'o',
})
if (err) {
Logger.mainLogger.error('Invalid receipt tx data', err)
return false
}
for (const account of receipt.beforeStates) {
err = Utils.validateTypes(account, {
hash: 's',
data: 'o',
isGlobal: 'b',
accountId: 's',
timestamp: 'n',
// cycleNumber: 'n', it is not present in the beforeStateAccounts data
})
if (err) {
Logger.mainLogger.error('Invalid receipt beforeStateAccounts data', err)
return false
}
}
for (const account of receipt.afterStates) {
err = Utils.validateTypes(account, {
hash: 's',
data: 'o',
isGlobal: 'b',
accountId: 's',
timestamp: 'n',
// cycleNumber: 'n', it is not present in the beforeStateAccounts data
})
if (err) {
Logger.mainLogger.error('Invalid receipt accounts data', err)
return false
}
}
if (receipt.globalModification) {
const signedReceipt = receipt.signedReceipt as P2PTypes.GlobalAccountsTypes.GlobalTxReceipt
err = Utils.validateTypes(signedReceipt, {
tx: 'o',
signs: 'a',
})
if (err) {
Logger.mainLogger.error('Invalid receipt globalModification data', err)
return false
}
err = Utils.validateTypes(signedReceipt.tx, {
address: 's',
addressHash: 's',
value: 'o',
when: 'n',
source: 's',
})
if (err) {
Logger.mainLogger.error('Invalid receipt globalModification tx data', err)
return false
}
for (const sign of signedReceipt.signs) {
err = Utils.validateTypes(sign, {
owner: 's',
sig: 's',
})
if (err) {
Logger.mainLogger.error('Invalid receipt globalModification signs data', err)
return false
}
}
return true
}
// Global Modification Tx does not have appliedReceipt
const signedReceipt = receipt.signedReceipt as Receipt.SignedReceipt
const signedReceiptToValidate = {
proposal: 'o',
proposalHash: 's',
signaturePack: 'a',
voteOffsets: 'a',
}
// if (config.newPOQReceipt === false) delete appliedReceiptToValidate.confirmOrChallenge
err = Utils.validateTypes(signedReceipt, signedReceiptToValidate)
if (err) {
Logger.mainLogger.error('Invalid receipt signedReceipt data', err)
return false
}
const proposalToValidate = {
txid: 's',
applied: 'b',
accountIDs: 'a',
cant_preApply: 'b',
afterStateHashes: 'a',
beforeStateHashes: 'a',
appReceiptDataHash: 's',
}
// if (config.newPOQReceipt === false) {
// delete appliedVoteToValidate.node_id
// delete appliedVoteToValidate.sign
// }
err = Utils.validateTypes(signedReceipt.proposal, proposalToValidate)
if (err) {
Logger.mainLogger.error('Invalid receipt signedReceipt appliedVote data', err)

let errors = verifyPayload(AJVSchemaEnum.ArchiverReceipt, receipt)

if (errors) {
Logger.mainLogger.error(
'Invalid Archiver Receipt',
errors,
'where receipt was', StringUtils.safeStringify(receipt)
);
return false
}
for (const signature of signedReceipt.signaturePack) {
err = Utils.validateTypes(signature, {
owner: 's',
sig: 's',
})
if (err) {
Logger.mainLogger.error('Invalid receipt signedReceipt signatures data', err)
return false
}
}
for (const voteOffset of signedReceipt.voteOffsets) {
const isValid = typeof voteOffset === 'number' || !isNaN(voteOffset)
if (!isValid) {
Logger.mainLogger.error('Invalid receipt signedReceipt voteOffsets data', voteOffset)
return false
}
}
// if (config.newPOQReceipt === false) return true
// err = Utils.validateTypes(receipt.appliedReceipt.appliedVote.sign, {
// owner: 's',
// sig: 's',
// })
// if (err) {
// Logger.mainLogger.error('Invalid receipt appliedReceipt appliedVote signature data', err)
// return false
// }
// err = Utils.validateTypes(receipt.appliedReceipt.confirmOrChallenge, {
// message: 's',
// nodeId: 's',
// appliedVote: 'o',
// sign: 'o',
// })
// if (err) {
// Logger.mainLogger.error('Invalid receipt appliedReceipt confirmOrChallenge data', err)
// return false
// }
// err = Utils.validateTypes(receipt.appliedReceipt.confirmOrChallenge.sign, {
// owner: 's',
// sig: 's',
// })
// if (err) {
// Logger.mainLogger.error('Invalid receipt appliedReceipt confirmOrChallenge signature data', err)
// return false
// }
return true
}

Expand Down Expand Up @@ -1423,25 +1275,18 @@ interface validateResponse {
}

export const validateOriginalTxData = (originalTxData: OriginalTxsData.OriginalTxData): boolean => {
const err = Utils.validateTypes(originalTxData, {
txId: 's',
timestamp: 'n',
cycle: 'n',
// sign: 'o',
originalTxData: 'o',
})
if (err) {
Logger.mainLogger.error('Invalid originalTxsData', err)
return false
}
// err = Utils.validateTypes(originalTxData.sign, {
// owner: 's',
// sig: 's',
// })
if (err) {
Logger.mainLogger.error('Invalid originalTxsData signature', err)
return false

const errors = verifyPayload(AJVSchemaEnum.OriginalTxData, originalTxData)

if (errors) {
Logger.mainLogger.error(
'Invalid originalTxsData',
errors,
'where originalTxData was: ', StringUtils.safeStringify(originalTxData)
);
return false;
}

return true
}

Expand Down
72 changes: 51 additions & 21 deletions src/dbstore/accounts.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import * as Logger from '../Logger'
import { config } from '../Config'
import { DeSerializeFromJsonString, SerializeToJsonString } from '../utils/serialization'


/** Same as type AccountsCopy in the shardus core */
export type AccountsCopy = {
accountId: string
Expand All @@ -19,38 +20,67 @@ type DbAccountCopy = AccountsCopy & {
}

export async function insertAccount(account: AccountsCopy): Promise<void> {

try {
const fields = Object.keys(account).join(', ')
const placeholders = Object.keys(account).fill('?').join(', ')
const values = db.extractValues(account)
const sql = 'INSERT OR REPLACE INTO accounts (' + fields + ') VALUES (' + placeholders + ')'
await db.run(accountDatabase, sql, values)

// Define the table columns based on schema
const columns = ['accountId', 'data', 'timestamp', 'hash', 'cycleNumber', 'isGlobal'];

// Construct the SQL query with placeholders
const placeholders = `(${columns.map(() => '?').join(', ')})`;
const sql = `INSERT OR REPLACE INTO accounts (${columns.join(', ')}) VALUES ${placeholders}`;

// Map the `account` object to match the columns
const values = columns.map((column) =>
typeof account[column] === 'object'
? SerializeToJsonString(account[column]) // Serialize objects to JSON
: account[column]
);

// Execute the query directly (single-row insert)
await db.run(accountDatabase, sql, values);

if (config.VERBOSE) {
Logger.mainLogger.debug('Successfully inserted Account', account.accountId)
Logger.mainLogger.debug('Successfully inserted Account', account.accountId);
}
} catch (e) {
Logger.mainLogger.error(e)
} catch (err) {
Logger.mainLogger.error(err);
Logger.mainLogger.error(
'Unable to insert Account or it is already stored in to database',
'Unable to insert Account or it is already stored in the database',
account.accountId
)
);
}
}

export async function bulkInsertAccounts(accounts: AccountsCopy[]): Promise<void> {

try {
const fields = Object.keys(accounts[0]).join(', ')
const placeholders = Object.keys(accounts[0]).fill('?').join(', ')
const values = db.extractValuesFromArray(accounts)
let sql = 'INSERT OR REPLACE INTO accounts (' + fields + ') VALUES (' + placeholders + ')'
for (let i = 1; i < accounts.length; i++) {
sql = sql + ', (' + placeholders + ')'

// Define the table columns based on schema
const columns = ['accountId', 'data', 'timestamp', 'hash', 'cycleNumber', 'isGlobal'];

// Construct the SQL query for bulk insertion with all placeholders
const placeholders = accounts.map(() => `(${columns.map(() => '?').join(', ')})`).join(', ');
const sql = `INSERT OR REPLACE INTO accounts (${columns.join(', ')}) VALUES ${placeholders}`;

// Flatten the `accounts` array into a single list of values
const values = accounts.flatMap((account) =>
columns.map((column) =>
typeof account[column] === 'object'
? SerializeToJsonString(account[column]) // Serialize objects to JSON
: account[column]
)
);

// Execute the single query for all accounts
await db.run(accountDatabase, sql, values);

if (config.VERBOSE) {
Logger.mainLogger.debug('Successfully inserted Accounts', accounts.length);
}
await db.run(accountDatabase, sql, values)
if (config.VERBOSE) Logger.mainLogger.debug('Successfully inserted Accounts', accounts.length)
} catch (e) {
Logger.mainLogger.error(e)
Logger.mainLogger.error('Unable to bulk insert Accounts', accounts.length)
} catch (err) {
Logger.mainLogger.error(err);
Logger.mainLogger.error('Unable to bulk insert Accounts', accounts.length);
}
}

Expand Down
Loading

0 comments on commit fb7aac5

Please sign in to comment.