From cdaa236ff531d07daa9ecfeec514a9bd12eafffa Mon Sep 17 00:00:00 2001 From: brianleroux Date: Fri, 26 Jan 2024 12:05:40 -0800 Subject: [PATCH 01/10] first pass --- package.json | 10 ++--- src/count.js | 6 ++- src/destroy/batch.js | 6 ++- src/destroy/one.js | 6 ++- src/get/batch.js | 6 ++- src/get/one.js | 6 ++- src/get/page.js | 6 ++- src/helpers/_atomic-counters.js | 8 ++-- src/helpers/_dynamo.js | 74 ++++++++++++--------------------- src/helpers/_get-ports.js | 43 +++++++++++-------- src/helpers/_get-table-name.js | 43 ++++++++++++------- src/helpers/_is-node-18.js | 1 - src/set/batch.js | 6 ++- src/set/one.js | 6 ++- 14 files changed, 123 insertions(+), 104 deletions(-) delete mode 100644 src/helpers/_is-node-18.js diff --git a/package.json b/package.json index f1fb52d..a46cd49 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "src/index.js", "scripts": { "lint": "eslint --fix .", + "t": "tape test/one.js | tap-spec", "test": "npm run lint && tape test/*-test.js | tap-spec", "rc": "npm version prerelease --preid RC" }, @@ -30,11 +31,10 @@ }, "devDependencies": { "@architect/eslint-config": "^2.0.1", - "@architect/sandbox": "^5.1.0-RC.0", - "@aws-sdk/client-dynamodb": "^3.216.0", - "@aws-sdk/client-ssm": "^3.216.0", - "@aws-sdk/lib-dynamodb": "^3.216.0", - "aws-sdk": "^2.1261.0", + "@architect/sandbox": "^6.0.0-RC.1", + "@aws-lite/client": "^0.14.2", + "@aws-lite/dynamodb": "^0.3.1", + "@aws-lite/ssm": "^0.2.2", "eslint": "^8.10.0", "tap-spec": "^5.0.0", "tape": "^5.5.2", diff --git a/src/count.js b/src/count.js index ca8a3ed..1e7c520 100644 --- a/src/count.js +++ b/src/count.js @@ -2,9 +2,10 @@ * @module count */ let waterfall = require('run-waterfall') +let util = require('util') let getTableName = require('./helpers/_get-table-name') let getKey = require('./helpers/_get-key') -let dynamo = require('./helpers/_dynamo').doc +let dynamo = require('./helpers/_dynamo') /** * Get document count for given table @@ -36,7 +37,8 @@ module.exports = function count ({ table }, callback) { }) }, function counts (TableName, doc, callback) { - doc.query({ + let query = util.callbackify(doc.Query) + query({ TableName, Select: 'COUNT', KeyConditionExpression: '#scopeID = :scopeID and begins_with(#dataID, :dataID)', diff --git a/src/destroy/batch.js b/src/destroy/batch.js index 8e2a3af..65e23ed 100644 --- a/src/destroy/batch.js +++ b/src/destroy/batch.js @@ -2,10 +2,11 @@ * @private * @module destroy/batch */ +let util = require('util') let waterfall = require('run-waterfall') let getTableName = require('../helpers/_get-table-name') let getKey = require('../helpers/_get-key') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') /** * Destroy an array of documents @@ -33,7 +34,8 @@ module.exports = function batch (params, callback) { let batch = params.map(getKey).map(req) let query = { RequestItems: {} } query.RequestItems[TableName] = batch - doc.batchWrite(query, callback) + let batchWrite = util.callbackify(doc.BatchWriteItem) + batchWrite(query, callback) } ], function destroyed (err) { diff --git a/src/destroy/one.js b/src/destroy/one.js index afde77c..5a6c417 100644 --- a/src/destroy/one.js +++ b/src/destroy/one.js @@ -2,10 +2,11 @@ * @private * @module destroy/one */ +let util = require('util') let waterfall = require('run-waterfall') let getTableName = require('../helpers/_get-table-name') let getKey = require('../helpers/_get-key') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') /** * Destroy a document @@ -22,8 +23,9 @@ module.exports = function one (params, callback) { }) }, function destroys (TableName, doc, callback) { + let del = util.callbackify(doc.DeleteItem) let Key = getKey(params) - doc.delete({ + del({ TableName, Key, }, callback) diff --git a/src/get/batch.js b/src/get/batch.js index 1236673..2d9bddb 100644 --- a/src/get/batch.js +++ b/src/get/batch.js @@ -2,11 +2,12 @@ * @private * @module get/batch */ +let util = require('util') let waterfall = require('run-waterfall') let getTableName = require('../helpers/_get-table-name') let getKey = require('../helpers/_get-key') let unfmt = require('../helpers/_unfmt') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') let badKey = i => !(i['table'] && i['key']) @@ -39,9 +40,10 @@ module.exports = function batch (Keys, callback) { }) }, function gets (table, doc, callback) { + let batchGet = util.callbackify(doc.BatchGetItem) let query = { RequestItems: {} } query.RequestItems[table] = { Keys: Keys.map(getKey) } - doc.batchGet(query, function gots (err, result) { + batchGet(query, function gots (err, result) { if (err) callback(err) else { callback(null, result.Responses[table].map(unfmt)) diff --git a/src/get/one.js b/src/get/one.js index 03cc024..6a68311 100644 --- a/src/get/one.js +++ b/src/get/one.js @@ -6,7 +6,8 @@ let waterfall = require('run-waterfall') let getTableName = require('../helpers/_get-table-name') let getKey = require('../helpers/_get-key') let unfmt = require('../helpers/_unfmt') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') +let util = require('util') /** * Read a document @@ -32,8 +33,9 @@ module.exports = function one (params, callback) { }) }, function gets (TableName, doc, callback) { + let getItem = util.callbackify(doc.GetItem) let Key = getKey(params) - doc.get({ + getItem({ TableName, Key }, callback) diff --git a/src/get/page.js b/src/get/page.js index fd0d8a4..e1dd898 100644 --- a/src/get/page.js +++ b/src/get/page.js @@ -6,7 +6,8 @@ let waterfall = require('run-waterfall') let getTableName = require('../helpers/_get-table-name') let getKey = require('../helpers/_get-key') let unfmt = require('../helpers/_unfmt') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') +let util = require('util') /** * Read documents @@ -55,7 +56,8 @@ module.exports = function page (params, callback) { if (params.cursor) { query.ExclusiveStartKey = JSON.parse(Buffer.from(params.cursor, 'base64').toString('utf8')) } - doc.query(query, callback) + let runQuery = util.callbackify(doc.Query) + runQuery(query, callback) }, ], function paged (err, result) { diff --git a/src/helpers/_atomic-counters.js b/src/helpers/_atomic-counters.js index 6d89823..171afa4 100644 --- a/src/helpers/_atomic-counters.js +++ b/src/helpers/_atomic-counters.js @@ -4,7 +4,8 @@ * @module decr */ let waterfall = require('run-waterfall') -let dynamo = require('./_dynamo').doc +let util = require('util') +let dynamo = require('./_dynamo') let getTableName = require('./_get-table-name') let getKey = require('./_get-key') let unfmt = require('./_unfmt') @@ -41,9 +42,10 @@ function atomic (isIncr, params, callback) { else callback(null, TableName, doc) }) }, - function update (TableName, doc, callback) { + function _update (TableName, doc, callback) { // perform the atomic update and callback w the updated values - doc.update({ + let update = util.callbackify(doc.UpdateItem) + update({ TableName, Key: getKey({ table, key }), UpdateExpression: `SET ${prop} = if_not_exists(${prop}, :zero) ${isIncr ? '+' : '-'} :val`, diff --git a/src/helpers/_dynamo.js b/src/helpers/_dynamo.js index e3cde23..4c78d50 100644 --- a/src/helpers/_dynamo.js +++ b/src/helpers/_dynamo.js @@ -1,70 +1,50 @@ -let https = require('https') let getPorts = require('./_get-ports') -let isNode18 = require('./_is-node-18') -let db, doc +let util = require('util') +let awsLite = require('@aws-lite/client') +let aws = util.callbackify(awsLite) +let db = false /** * Instantiates Dynamo service interfaces */ -function getDynamo (type, callback) { +module.exports = function getDynamo (callback) { let { ARC_ENV, AWS_REGION, ARC_LOCAL } = process.env - if (db && type === 'db') { - return callback(null, db) - } - - if (doc && type === 'doc') { - return callback(null, doc) - } - - let DB, Doc - if (isNode18) { - let dynamo = require('@aws-sdk/client-dynamodb') - let docclient = require('@aws-sdk/lib-dynamodb') - DB = dynamo.DynamoDB - Doc = docclient.DynamoDBDocument - } - else { - let dynamo = require('aws-sdk/clients/dynamodb') - DB = dynamo - Doc = dynamo.DocumentClient - } + if (db) return callback(null, db) let local = ARC_ENV === 'testing' || ARC_LOCAL if (!local) { - let config = { - agent: new https.Agent({ - keepAlive: true, - maxSockets: 50, // Node can set to Infinity; AWS maxes at 50 - rejectUnauthorized: true, - }) - } - db = isNode18 ? new DB : new DB(config) - doc = isNode18 ? Doc.from(db) : new Doc(config) - return callback(null, type === 'db' ? db : doc) + aws(function gotClient (err, { ddb }) { + if (err) callback(err) + else { + db = ddb + callback(null, db) + } + }) } else { - getPorts((err, ports) => { + getPorts(function gotPorts (err, ports) { if (err) callback(err) else { let port = ports.tables if (!port) { return callback(ReferenceError('Sandbox tables port not found')) } - let config = { - endpoint: `http://localhost:${port}`, - region: AWS_REGION || 'us-west-2' // Do not assume region is set! - } - db = new DB(config) - doc = isNode18 ? Doc.from(db) : new Doc(config) - return callback(null, type === 'db' ? db : doc) + aws({ + protocol: 'http', + host: 'localhost', + port, + region: AWS_REGION || 'us-west-2' + }, + function gotClient (err, { dynamodb }) { + if (err) callback(err) + else { + db = dynamodb + callback(null, db) + } + }) } }) } } - -module.exports = { - db: getDynamo.bind({}, 'db'), - doc: getDynamo.bind({}, 'doc'), -} diff --git a/src/helpers/_get-ports.js b/src/helpers/_get-ports.js index eecde6b..62cf679 100644 --- a/src/helpers/_get-ports.js +++ b/src/helpers/_get-ports.js @@ -1,5 +1,7 @@ -let isNode18 = require('./_is-node-18') let toLogicalID = require('./_to-logical-id') +let util = require('util') +let awsLite = require('@aws-lite/client') +let aws = util.callbackify(awsLite) let configuredPorts module.exports = function getPorts (callback) { @@ -28,29 +30,36 @@ module.exports = function getPorts (callback) { app = 'arc-app' } let Name = `/${toLogicalID(`${app}-${env}`)}/ARC_SANDBOX/ports` - let port = 2222 let config = { - endpoint: `http://localhost:${port}/_arc/ssm`, + protocol: 'http', + host: 'localhost', + port: 2222, + endpointPrefix: '_arc/ssm', + // endpoint: `http://localhost:${port}/_arc/ssm`, region: AWS_REGION || 'us-west-2', } - let SSM = isNode18 ? require('@aws-sdk/client-ssm').SSM : require('aws-sdk/clients/ssm') - let ssm = new SSM(config) - ssm.getParameter({ Name }, function done (err, result) { + aws(config, function gotClient (err, { ssm }) { if (err) callback(err) else { - if (!result.Parameter.Value) { - callback(ReferenceError('@begin/data requires Sandbox to be running')) - } - else { - try { - configuredPorts = JSON.parse(result.Parameter.Value) - callback(null, configuredPorts) + let getParameter = util.callbackify(ssm.GetParameter) + getParameter({ Name }, function done (err, result) { + if (err) callback(err) + else { + if (!result.Parameter.Value) { + callback(ReferenceError('@begin/data requires Sandbox to be running')) + } + else { + try { + configuredPorts = JSON.parse(result.Parameter.Value) + callback(null, configuredPorts) + } + catch (err) { + callback(err) + } + } } - catch (err) { - callback(err) - } - } + }) } }) } diff --git a/src/helpers/_get-table-name.js b/src/helpers/_get-table-name.js index c78bc18..ca057a3 100644 --- a/src/helpers/_get-table-name.js +++ b/src/helpers/_get-table-name.js @@ -1,4 +1,6 @@ -let isNode18 = require('./_is-node-18') +let util = require('util') +let awsLite = require('@aws-lite/client') +let aws = util.callbackify(awsLite) let toLogicalID = require('./_to-logical-id') let getPorts = require('./_get-ports') let tablename = false @@ -29,28 +31,39 @@ module.exports = function getTableName (callback) { getPorts((err, ports) => { if (err) callback(err) else go({ - endpoint: `http://localhost:${ports._arc}/_arc/ssm`, + protocol: 'http', + host: 'localhost', + port: ports._arc, + endpointPrefix: '_arc/ssm', + // endpoint: `http://localhost:${ports._arc}/_arc/ssm`, region: AWS_REGION || 'us-west-2', }) }) } - else go() + else { + go() + } function go (config) { - - let SSM = isNode18 ? require('@aws-sdk/client-ssm').SSM : require('aws-sdk/clients/ssm') - let ssm = new SSM(config) - ssm.getParameter({ Name }, function done (err, result) { + aws(config, function gotClient (err, { ssm }) { if (err) callback(err) else { - let table = result.Parameter - if (!table) { - callback(ReferenceError('@begin/data requires a table named data')) - } - else { - tablename = table.Value - callback(null, tablename) - } + let getParameter = util.callbackify(ssm.GetParameter) + getParameter({ Name }, function done (err, result) { + // let ssm = new SSM(config) + // ssm.getParameter({ Name }, function done (err, result) { + if (err) callback(err) + else { + let table = result.Parameter + if (!table) { + callback(ReferenceError('@begin/data requires a table named data')) + } + else { + tablename = table.Value + callback(null, tablename) + } + } + }) } }) } diff --git a/src/helpers/_is-node-18.js b/src/helpers/_is-node-18.js deleted file mode 100644 index 47e653a..0000000 --- a/src/helpers/_is-node-18.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = Number(process.version.replace('v', '').split('.')[0]) >= 18 diff --git a/src/set/batch.js b/src/set/batch.js index 97d175d..fc00ebf 100644 --- a/src/set/batch.js +++ b/src/set/batch.js @@ -10,7 +10,8 @@ let createKey = require('../helpers/_create-key') let validate = require('../helpers/_validate') let unfmt = require('../helpers/_unfmt') let fmt = require('../helpers/_fmt') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') +let util = require('util') /** * Write an array of documents @@ -43,10 +44,11 @@ module.exports = function batch (params, callback) { }, function writeKeys (TableName, items, doc, callback) { validate.size(items) + let batchWrite = util.callbackify(doc.BatchWriteItem) let batch = items.map(Item => ({ PutRequest: { Item } })) let query = { RequestItems: {} } query.RequestItems[TableName] = batch - doc.batchWrite(query, function done (err) { + batchWrite(query, function done (err) { if (err) callback(err) else { let clean = item => unfmt(item.PutRequest.Item) diff --git a/src/set/one.js b/src/set/one.js index 5c54a69..f65d410 100644 --- a/src/set/one.js +++ b/src/set/one.js @@ -8,7 +8,8 @@ let createKey = require('../helpers/_create-key') let validate = require('../helpers/_validate') let unfmt = require('../helpers/_unfmt') let fmt = require('../helpers/_fmt') -let dynamo = require('../helpers/_dynamo').doc +let dynamo = require('../helpers/_dynamo') +let util = require('util') /** * Write a document @@ -37,7 +38,8 @@ module.exports = function one (params, callback) { }, function write (TableName, Item, doc, callback) { validate.size(Item) - doc.put({ + let put = util.callbackify(doc.PutItem) + put({ TableName, Item }, From 7d7de6f467a21b05fc06ea853cfa3e77fd7a36c4 Mon Sep 17 00:00:00 2001 From: brianleroux Date: Fri, 26 Jan 2024 13:32:26 -0800 Subject: [PATCH 02/10] wip --- package.json | 2 +- src/get/page.js | 7 +- src/helpers/_create-key.js | 25 +++--- src/helpers/_dynamo.js | 1 + test/ddb.js | 158 ------------------------------------- test/integration-test.js | 18 +++-- test/namespace-test.js | 3 +- 7 files changed, 33 insertions(+), 181 deletions(-) delete mode 100644 test/ddb.js diff --git a/package.json b/package.json index a46cd49..4398b28 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "Begin Data is a durable and fast key/value document store built on top of DynamoDB", "main": "src/index.js", "scripts": { + "t": "tape test/integration-test.js | tap-spec", "lint": "eslint --fix .", - "t": "tape test/one.js | tap-spec", "test": "npm run lint && tape test/*-test.js | tap-spec", "rc": "npm version prerelease --preid RC" }, diff --git a/src/get/page.js b/src/get/page.js index e1dd898..dc343c5 100644 --- a/src/get/page.js +++ b/src/get/page.js @@ -55,6 +55,9 @@ module.exports = function page (params, callback) { } if (params.cursor) { query.ExclusiveStartKey = JSON.parse(Buffer.from(params.cursor, 'base64').toString('utf8')) + //console.log('received =======>', query.ExclusiveStartKey) + // query.ExclusiveStartKey.scopeID = {S: query.ExclusiveStartKey.scopeID} + //query.ExclusiveStartKey.dataID = {S: query.ExclusiveStartKey.dataID} } let runQuery = util.callbackify(doc.Query) runQuery(query, callback) @@ -65,8 +68,10 @@ module.exports = function page (params, callback) { else { let exact = item => item.table === params.table let returns = result.Items.map(unfmt).filter(exact) - if (result.LastEvaluatedKey) + if (result.LastEvaluatedKey) { + // console.log('sending =======>', result.LastEvaluatedKey) returns.cursor = Buffer.from(JSON.stringify(result.LastEvaluatedKey)).toString('base64') + } callback(null, returns) } }) diff --git a/src/helpers/_create-key.js b/src/helpers/_create-key.js index cc5bc79..ed0234a 100644 --- a/src/helpers/_create-key.js +++ b/src/helpers/_create-key.js @@ -4,8 +4,9 @@ */ let waterfall = require('run-waterfall') let getTableName = require('./_get-table-name') -let dynamo = require('./_dynamo').db +let dynamo = require('./_dynamo') let Hashids = require('@begin/hashid') +let util = require('util') let hash = new Hashids module.exports = function createKey (table, callback) { @@ -19,27 +20,27 @@ module.exports = function createKey (table, callback) { }) }, function update (TableName, db, callback) { - db.updateItem({ + let updateItem = util.callbackify(db.UpdateItem) + let query = { TableName, Key: { - 'scopeID': { S: BEGIN_DATA_SCOPE_ID || ARC_APP_NAME }, - 'dataID': { S: `${table}-seq` } + 'scopeID': BEGIN_DATA_SCOPE_ID || ARC_APP_NAME, + 'dataID': `${table}-seq` }, - AttributeUpdates: { - idx: { - Action: 'ADD', - Value: { N: '1' } - } + UpdateExpression: `SET idx = if_not_exists(idx, :one) + :one`, + ExpressionAttributeValues: { + ':one': 1 }, - ReturnValues: 'UPDATED_NEW' - }, callback) + ReturnValues: 'ALL_NEW' + } + updateItem(query, callback) } ], function done (err, result) { if (err) callback(err) else { let epoc = Date.now() - 1544909702376 // hbd - let seed = Number(result.Attributes.idx.N) + let seed = Number(result.Attributes.idx) let val = hash.encode([ epoc, seed ]) callback(null, val) } diff --git a/src/helpers/_dynamo.js b/src/helpers/_dynamo.js index 4c78d50..7c3d7f7 100644 --- a/src/helpers/_dynamo.js +++ b/src/helpers/_dynamo.js @@ -32,6 +32,7 @@ module.exports = function getDynamo (callback) { return callback(ReferenceError('Sandbox tables port not found')) } aws({ + debug: true, protocol: 'http', host: 'localhost', port, diff --git a/test/ddb.js b/test/ddb.js deleted file mode 100644 index 42760a1..0000000 --- a/test/ddb.js +++ /dev/null @@ -1,158 +0,0 @@ -let test = require('tape') -let file = '../src/helpers/_dynamo' -let dynamo - -function reset (t) { - delete process.env.AWS_REGION - delete require.cache[require.resolve(file)] - dynamo = undefined - - if (process.env.AWS_REGION) t.fail('Did not unset AWS_REGION') - if (require.cache[require.resolve(file)]) t.fail('Did not reset require cache') - if (dynamo) t.fail('Did not unset module') -} - -test('Set up env', t => { - t.plan(2) - process.env.ARC_ENV = 'testing' - - // eslint-disable-next-line - dynamo = require(file) - - // DB x callback - dynamo.db((err, db) => { - if (err) t.fail(err) - t.ok(db, 'Got DynamoDB object (callback)') - }) - - // Doc x callback - dynamo.doc((err, doc) => { - if (err) t.fail(err) - t.ok(doc, 'Got DynamoDB document object (callback)') - }) - - reset(t) -}) - -test('Local port + region configuration', t => { - t.plan(20) - - /** - * Defaults - */ - let localhost = 'localhost' - let defaultPort = 5555 - let defaultRegion = 'us-west-2' - let host = `${localhost}:${defaultPort}` - - // eslint-disable-next-line - dynamo = require(file) - - // DB x callback - dynamo.db((err, db) => { - if (err) t.fail(err) - t.equal(db.endpoint.host, host, `DB configured 'host' property is ${host}`) - t.equal(db.endpoint.hostname, localhost, `DB configured 'hostname' property is ${localhost}`) - t.equal(db.endpoint.href, `http://${host}/`, `DB configured 'href' property is http://${host}/`) - t.equal(db.endpoint.port, defaultPort, `DB configured 'port' property is ${defaultPort}`) - t.equal(db.config.region, defaultRegion, `DB configured 'region' property is ${defaultRegion}`) - }) - - // Doc x callback - dynamo.doc((err, doc) => { - if (err) t.fail(err) - t.equal(doc.options.endpoint.host, host, `Doc configured 'host' property is ${host}`) - t.equal(doc.options.endpoint.hostname, localhost, `Doc configured 'hostname' property is ${localhost}`) - t.equal(doc.options.endpoint.href, `http://${host}/`, `Doc configured 'href' property is http://${host}/`) - t.equal(doc.options.endpoint.port, defaultPort, `Doc configured 'port' property is ${defaultPort}`) - t.equal(doc.service.config.region, defaultRegion, `Doc configured 'region' property is ${defaultRegion}`) - }) - - reset(t) - - /** - * Custom - */ - let customPort = 5555 - let customRegion = 'us-east-1' - process.env.ARC_TABLES_PORT = customPort - process.env.AWS_REGION = customRegion - host = `${localhost}:${customPort}` - - // eslint-disable-next-line - dynamo = require(file) - - // DB x callback - dynamo.db((err, db) => { - if (err) t.fail(err) - t.equal(db.endpoint.host, host, `DB configured 'host' property is ${host}`) - t.equal(db.endpoint.hostname, localhost, `DB configured 'hostname' property is ${localhost}`) - t.equal(db.endpoint.href, `http://${host}/`, `DB configured 'href' property is http://${host}/`) - t.equal(db.endpoint.port, customPort, `DB configured 'port' property is ${customPort}`) - t.equal(db.config.region, customRegion, `DB configured 'region' property is ${customRegion}`) - }) - - // Doc x callback - dynamo.doc((err, doc) => { - if (err) t.fail(err) - t.equal(doc.options.endpoint.host, host, `Doc configured 'host' property is ${host}`) - t.equal(doc.options.endpoint.hostname, localhost, `Doc configured 'hostname' property is ${localhost}`) - t.equal(doc.options.endpoint.href, `http://${host}/`, `Doc configured 'href' property is http://${host}/`) - t.equal(doc.options.endpoint.port, customPort, `Doc configured 'port' property is ${customPort}`) - t.equal(doc.service.config.region, customRegion, `Doc configured 'region' property is ${customRegion}`) - }) - - reset(t) -}) - -test('Live AWS infra config', t => { - t.plan(4) - - // Defaults - process.env.ARC_ENV = 'testing' - - // eslint-disable-next-line - dynamo = require(file) - - // DB x callback - dynamo.db((err, db) => { - if (err) t.fail(err) - t.notOk(db.config.httpOptions.agent, 'DB HTTP agent options not set') - }) - - // Doc x callback - dynamo.doc((err, doc) => { - if (err) t.fail(err) - t.notOk(doc.service.config.httpOptions.agent, 'Doc HTTP agent options not set') - }) - - reset(t) - - // Defaults - process.env.ARC_ENV = 'staging' - process.env.AWS_REGION = 'us-west-1' - - // eslint-disable-next-line - dynamo = require(file) - - // DB x callback - dynamo.db((err, db) => { - if (err) t.fail(err) - t.ok(db.config.httpOptions.agent.options, 'DB HTTP agent options set') - }) - - // Doc x callback - dynamo.doc((err, doc) => { - if (err) t.fail(err) - t.ok(doc.service.config.httpOptions.agent.options, 'Doc HTTP agent options set') - }) - - reset(t) -}) - -test('Tear down env', t => { - t.plan(1) - delete process.env.ARC_ENV - reset(t) - t.pass('Tore down env') -}) diff --git a/test/integration-test.js b/test/integration-test.js index 758dbb3..1de8b51 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -14,10 +14,10 @@ test('env', t => { test('Start sandbox', async t => { t.plan(1) - await sandbox.start({ cwd: __dirname }) + await sandbox.start({ cwd: __dirname, quiet: true }) t.pass('started') }) - + /* test('get a key that does not exist returns null', async t => { t.plan(1) let result = await data.get({ table: 'foo', key: 'nooo' }) @@ -134,9 +134,9 @@ test('incr/decr', async t => { t.ok(decrRes.scopeID === undefined, 'decrRes.scopeID is undefined') t.ok(decrRes.dataID === undefined, 'decrRes.dataID is undefined') }) - + */ test('Node 8+ cursor style pagination', async t => { - t.plan(4) + t.plan(2) // add 100 ppl (25 at a time) console.time('add 100 ppl') @@ -158,13 +158,15 @@ test('Node 8+ cursor style pagination', async t => { let result = await data.get({ table, limit: 100 }) t.ok(result.length === 100, 'got the first 100 ppl') t.ok(result.cursor, `got cursor ${result.cursor}`) + console.log(result) // get the last two let result2 = await data.get({ table, cursor: result.cursor }) - t.ok(result2.length === 2, 'got last two ppl') - t.ok(typeof result2.cursor === 'undefined', 'and no cursor') + console.log(result2) + //t.ok(result2.length === 2, 'got last two ppl') + //t.ok(typeof result2.cursor === 'undefined', 'and no cursor') }) - +/* test('implementing a scan', t => { t.plan(1) @@ -250,7 +252,7 @@ test('paginate ten at a time', async t => { t.ok(count === 5, 'counted five pages (102 records at 25 per page)') console.log(count) }) - +*/ // fin test('shutdown sandbox', async t => { t.plan(1) diff --git a/test/namespace-test.js b/test/namespace-test.js index 8b9c54a..ae05580 100644 --- a/test/namespace-test.js +++ b/test/namespace-test.js @@ -4,10 +4,11 @@ let test = require('tape') test('start sandbox', async t => { t.plan(1) - await sandbox.start({ cwd: __dirname }) + await sandbox.start({ quiet: true, cwd: __dirname }) t.pass('sandbox.start') }) + test('tables with similar namespacing', async t => { t.plan(2) // write some data From 5890cfb714ae26bc39f1a9f54e629377f05f3150 Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Fri, 26 Jan 2024 19:26:07 -0800 Subject: [PATCH 03/10] Update deps, fix pagination Fix tests Remove some commented out stuff --- package.json | 14 +++++++------- src/get/page.js | 4 ---- src/helpers/_dynamo.js | 1 - test/integration-test.js | 16 +++++++--------- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 4398b28..9c99c68 100644 --- a/package.json +++ b/package.json @@ -25,20 +25,20 @@ "src/*" ], "dependencies": { + "@aws-lite/client": "^0.14.3", + "@aws-lite/dynamodb": "^0.3.2", + "@aws-lite/ssm": "^0.2.2", "@begin/hashid": "^1.0.0", "run-parallel": "^1.2.0", "run-waterfall": "^1.1.7" }, "devDependencies": { - "@architect/eslint-config": "^2.0.1", + "@architect/eslint-config": "^2.1.2", "@architect/sandbox": "^6.0.0-RC.1", - "@aws-lite/client": "^0.14.2", - "@aws-lite/dynamodb": "^0.3.1", - "@aws-lite/ssm": "^0.2.2", - "eslint": "^8.10.0", + "eslint": "^8.56.0", "tap-spec": "^5.0.0", - "tape": "^5.5.2", - "tiny-json-http": "^7.3.1" + "tape": "^5.7.4", + "tiny-json-http": "^7.5.1" }, "eslintConfig": { "extends": "@architect/eslint-config", diff --git a/src/get/page.js b/src/get/page.js index dc343c5..ced68a2 100644 --- a/src/get/page.js +++ b/src/get/page.js @@ -55,9 +55,6 @@ module.exports = function page (params, callback) { } if (params.cursor) { query.ExclusiveStartKey = JSON.parse(Buffer.from(params.cursor, 'base64').toString('utf8')) - //console.log('received =======>', query.ExclusiveStartKey) - // query.ExclusiveStartKey.scopeID = {S: query.ExclusiveStartKey.scopeID} - //query.ExclusiveStartKey.dataID = {S: query.ExclusiveStartKey.dataID} } let runQuery = util.callbackify(doc.Query) runQuery(query, callback) @@ -69,7 +66,6 @@ module.exports = function page (params, callback) { let exact = item => item.table === params.table let returns = result.Items.map(unfmt).filter(exact) if (result.LastEvaluatedKey) { - // console.log('sending =======>', result.LastEvaluatedKey) returns.cursor = Buffer.from(JSON.stringify(result.LastEvaluatedKey)).toString('base64') } callback(null, returns) diff --git a/src/helpers/_dynamo.js b/src/helpers/_dynamo.js index 7c3d7f7..4c78d50 100644 --- a/src/helpers/_dynamo.js +++ b/src/helpers/_dynamo.js @@ -32,7 +32,6 @@ module.exports = function getDynamo (callback) { return callback(ReferenceError('Sandbox tables port not found')) } aws({ - debug: true, protocol: 'http', host: 'localhost', port, diff --git a/test/integration-test.js b/test/integration-test.js index 1de8b51..de41544 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -17,7 +17,7 @@ test('Start sandbox', async t => { await sandbox.start({ cwd: __dirname, quiet: true }) t.pass('started') }) - /* + test('get a key that does not exist returns null', async t => { t.plan(1) let result = await data.get({ table: 'foo', key: 'nooo' }) @@ -134,9 +134,9 @@ test('incr/decr', async t => { t.ok(decrRes.scopeID === undefined, 'decrRes.scopeID is undefined') t.ok(decrRes.dataID === undefined, 'decrRes.dataID is undefined') }) - */ + test('Node 8+ cursor style pagination', async t => { - t.plan(2) + t.plan(4) // add 100 ppl (25 at a time) console.time('add 100 ppl') @@ -158,15 +158,13 @@ test('Node 8+ cursor style pagination', async t => { let result = await data.get({ table, limit: 100 }) t.ok(result.length === 100, 'got the first 100 ppl') t.ok(result.cursor, `got cursor ${result.cursor}`) - console.log(result) // get the last two let result2 = await data.get({ table, cursor: result.cursor }) - console.log(result2) - //t.ok(result2.length === 2, 'got last two ppl') - //t.ok(typeof result2.cursor === 'undefined', 'and no cursor') + t.ok(result2.length === 2, 'got last two ppl') + t.ok(typeof result2.cursor === 'undefined', 'and no cursor') }) -/* + test('implementing a scan', t => { t.plan(1) @@ -252,7 +250,7 @@ test('paginate ten at a time', async t => { t.ok(count === 5, 'counted five pages (102 records at 25 per page)') console.log(count) }) -*/ + // fin test('shutdown sandbox', async t => { t.plan(1) From 0642dfd6e86d5ce079877e2f0a376d7eeba10c2a Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Fri, 26 Jan 2024 19:31:34 -0800 Subject: [PATCH 04/10] Update CI --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5f6eeb..b90b896 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [ 14.x, 16.x, 18.x ] + node-version: [ 14.x, 16.x, 18.x, 20.x ] os: [windows-latest, ubuntu-latest, macOS-latest] # Go @@ -42,7 +42,7 @@ jobs: - name: Notify uses: homoluctus/slatify@master # Only fire alert once - if: github.ref == 'refs/heads/master' && failure() && matrix.node-version == '14.x' && matrix.os == 'ubuntu-latest' + if: github.ref == 'refs/heads/master' && failure() && matrix.node-version == '20.x' && matrix.os == 'ubuntu-latest' with: type: ${{ job.status }} job_name: '*Build*' @@ -67,7 +67,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v1 with: - node-version: 12 + node-version: lts/* registry-url: https://registry.npmjs.org/ - name: Install @@ -89,7 +89,7 @@ jobs: # Set up Node again, this time using GitHub as the publish target - uses: actions/setup-node@v1 with: - node-version: 12 + node-version: lts/* registry-url: https://npm.pkg.github.com/ # Change package org (@smallwins on GitHub vs @begin on NPM) From 7c3d9f7c5845e080f2f4cd9826ba5d28a355887b Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Fri, 26 Jan 2024 19:49:44 -0800 Subject: [PATCH 05/10] Fix issue where aws-lite client instantiation failure leads to red herring destructuring error Fix missing credentials for CI tests --- src/helpers/_dynamo.js | 4 ++-- src/helpers/_get-ports.js | 3 ++- src/helpers/_get-table-name.js | 3 ++- test/integration-test.js | 4 ++++ test/namespace-test.js | 4 ++++ 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/helpers/_dynamo.js b/src/helpers/_dynamo.js index 4c78d50..2c4dffe 100644 --- a/src/helpers/_dynamo.js +++ b/src/helpers/_dynamo.js @@ -37,10 +37,10 @@ module.exports = function getDynamo (callback) { port, region: AWS_REGION || 'us-west-2' }, - function gotClient (err, { dynamodb }) { + function gotClient (err, client) { if (err) callback(err) else { - db = dynamodb + db = client.dynamodb callback(null, db) } }) diff --git a/src/helpers/_get-ports.js b/src/helpers/_get-ports.js index 62cf679..930a1e4 100644 --- a/src/helpers/_get-ports.js +++ b/src/helpers/_get-ports.js @@ -39,9 +39,10 @@ module.exports = function getPorts (callback) { region: AWS_REGION || 'us-west-2', } - aws(config, function gotClient (err, { ssm }) { + aws(config, function gotClient (err, client) { if (err) callback(err) else { + let { ssm } = client let getParameter = util.callbackify(ssm.GetParameter) getParameter({ Name }, function done (err, result) { if (err) callback(err) diff --git a/src/helpers/_get-table-name.js b/src/helpers/_get-table-name.js index ca057a3..001f6ba 100644 --- a/src/helpers/_get-table-name.js +++ b/src/helpers/_get-table-name.js @@ -45,9 +45,10 @@ module.exports = function getTableName (callback) { } function go (config) { - aws(config, function gotClient (err, { ssm }) { + aws(config, function gotClient (err, client) { if (err) callback(err) else { + let { ssm } = client let getParameter = util.callbackify(ssm.GetParameter) getParameter({ Name }, function done (err, result) { // let ssm = new SSM(config) diff --git a/test/integration-test.js b/test/integration-test.js index de41544..a3141fe 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -14,6 +14,8 @@ test('env', t => { test('Start sandbox', async t => { t.plan(1) + process.env.AWS_ACCESS_KEY_ID = 'arc_dummy_access_key', + process.env.AWS_SECRET_ACCESS_KEY = 'arc_dummy_secret_key', await sandbox.start({ cwd: __dirname, quiet: true }) t.pass('started') }) @@ -254,6 +256,8 @@ test('paginate ten at a time', async t => { // fin test('shutdown sandbox', async t => { t.plan(1) + delete process.env.AWS_ACCESS_KEY_ID + delete process.env.AWS_SECRET_ACCESS_KEY await sandbox.end() t.pass('done') }) diff --git a/test/namespace-test.js b/test/namespace-test.js index ae05580..95e32bb 100644 --- a/test/namespace-test.js +++ b/test/namespace-test.js @@ -4,6 +4,8 @@ let test = require('tape') test('start sandbox', async t => { t.plan(1) + process.env.AWS_ACCESS_KEY_ID = 'arc_dummy_access_key', + process.env.AWS_SECRET_ACCESS_KEY = 'arc_dummy_secret_key', await sandbox.start({ quiet: true, cwd: __dirname }) t.pass('sandbox.start') }) @@ -28,6 +30,8 @@ test('tables with similar namespacing', async t => { test('shutdown sandbox', async t => { t.plan(1) + delete process.env.AWS_ACCESS_KEY_ID + delete process.env.AWS_SECRET_ACCESS_KEY await sandbox.end() t.pass('sandbox.end') }) From 83e3649ceb87dbecb6c6e6a5bc9c283c271aa276 Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Mon, 29 Jan 2024 11:13:59 -0800 Subject: [PATCH 06/10] Update deps --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9c99c68..090f568 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ ], "dependencies": { "@aws-lite/client": "^0.14.3", - "@aws-lite/dynamodb": "^0.3.2", + "@aws-lite/dynamodb": "^0.3.3", "@aws-lite/ssm": "^0.2.2", "@begin/hashid": "^1.0.0", "run-parallel": "^1.2.0", From 8b29f76062cb1dd8f24281b45f4d956b236b337d Mon Sep 17 00:00:00 2001 From: brianleroux Date: Mon, 29 Jan 2024 11:16:01 -0800 Subject: [PATCH 07/10] 5.0.0-RC.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 090f568..366f506 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@begin/data", - "version": "4.0.2", + "version": "5.0.0-RC.0", "description": "Begin Data is a durable and fast key/value document store built on top of DynamoDB", "main": "src/index.js", "scripts": { From 672584a01e2060f40be148ab8e78d5ec5bdc85dd Mon Sep 17 00:00:00 2001 From: brianleroux Date: Mon, 29 Jan 2024 16:16:38 -0800 Subject: [PATCH 08/10] 5.0.0-RC.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 366f506..5bfdcc5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@begin/data", - "version": "5.0.0-RC.0", + "version": "5.0.0-RC.1", "description": "Begin Data is a durable and fast key/value document store built on top of DynamoDB", "main": "src/index.js", "scripts": { From 1ec13c0dbbf2db404848c67b4f07bb0ce94d6223 Mon Sep 17 00:00:00 2001 From: brianleroux Date: Wed, 7 Feb 2024 16:12:30 -0800 Subject: [PATCH 09/10] update to latest aws-lite explicit style and pathPrefix changes --- package.json | 6 +++--- src/helpers/_dynamo.js | 4 +++- src/helpers/_get-ports.js | 4 ++-- src/helpers/_get-table-name.js | 8 ++++---- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 5bfdcc5..66d2195 100644 --- a/package.json +++ b/package.json @@ -25,9 +25,9 @@ "src/*" ], "dependencies": { - "@aws-lite/client": "^0.14.3", - "@aws-lite/dynamodb": "^0.3.3", - "@aws-lite/ssm": "^0.2.2", + "@aws-lite/client": "^0.17.1", + "@aws-lite/dynamodb": "^0.3.4", + "@aws-lite/ssm": "^0.2.3", "@begin/hashid": "^1.0.0", "run-parallel": "^1.2.0", "run-waterfall": "^1.1.7" diff --git a/src/helpers/_dynamo.js b/src/helpers/_dynamo.js index 2c4dffe..ee2c5ad 100644 --- a/src/helpers/_dynamo.js +++ b/src/helpers/_dynamo.js @@ -15,7 +15,8 @@ module.exports = function getDynamo (callback) { let local = ARC_ENV === 'testing' || ARC_LOCAL if (!local) { - aws(function gotClient (err, { ddb }) { + let plugins = [ import('@aws-lite/dynamodb') ] + aws({ plugins }, function gotClient (err, { ddb }) { if (err) callback(err) else { db = ddb @@ -35,6 +36,7 @@ module.exports = function getDynamo (callback) { protocol: 'http', host: 'localhost', port, + plugins: [ import('@aws-lite/dynamodb') ], region: AWS_REGION || 'us-west-2' }, function gotClient (err, client) { diff --git a/src/helpers/_get-ports.js b/src/helpers/_get-ports.js index 930a1e4..6d67d9d 100644 --- a/src/helpers/_get-ports.js +++ b/src/helpers/_get-ports.js @@ -31,14 +31,14 @@ module.exports = function getPorts (callback) { } let Name = `/${toLogicalID(`${app}-${env}`)}/ARC_SANDBOX/ports` let config = { + plugins: [ import('@aws-lite/ssm') ], protocol: 'http', host: 'localhost', port: 2222, - endpointPrefix: '_arc/ssm', + pathPrefix: '_arc/ssm', // endpoint: `http://localhost:${port}/_arc/ssm`, region: AWS_REGION || 'us-west-2', } - aws(config, function gotClient (err, client) { if (err) callback(err) else { diff --git a/src/helpers/_get-table-name.js b/src/helpers/_get-table-name.js index 001f6ba..25af996 100644 --- a/src/helpers/_get-table-name.js +++ b/src/helpers/_get-table-name.js @@ -34,8 +34,7 @@ module.exports = function getTableName (callback) { protocol: 'http', host: 'localhost', port: ports._arc, - endpointPrefix: '_arc/ssm', - // endpoint: `http://localhost:${ports._arc}/_arc/ssm`, + pathPrefix: '_arc/ssm', region: AWS_REGION || 'us-west-2', }) }) @@ -44,8 +43,9 @@ module.exports = function getTableName (callback) { go() } - function go (config) { - aws(config, function gotClient (err, client) { + function go (config = {}) { + let plugins = [ import('@aws-lite/ssm') ] + aws({ plugins, ...config }, function gotClient (err, client) { if (err) callback(err) else { let { ssm } = client From 11dafe698be1a9148140adf3e2da574d601210dd Mon Sep 17 00:00:00 2001 From: brianleroux Date: Wed, 7 Feb 2024 16:12:35 -0800 Subject: [PATCH 10/10] 5.0.0-RC.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 66d2195..02c1114 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@begin/data", - "version": "5.0.0-RC.1", + "version": "5.0.0-RC.2", "description": "Begin Data is a durable and fast key/value document store built on top of DynamoDB", "main": "src/index.js", "scripts": {