Merge pull request #1521 from okTurtles/e2e-protocol

E2E protocol

.babelrc

@@ -3,8 +3,8 @@
   "plugins": [
     "babel-plugin-root-import",
     "@babel/plugin-transform-runtime",
-    "@babel/plugin-proposal-class-properties",
-    "@babel/plugin-proposal-object-rest-spread",
+    "@babel/plugin-transform-class-properties",
+    "@babel/plugin-transform-object-rest-spread",
     "@babel/plugin-syntax-dynamic-import"
   ]
 }

.flowconfig

@@ -12,10 +12,10 @@
 .*/dist-dashboard/.*
 <PROJECT_ROOT>/contracts/.*
 .*/backend/dashboard/.*
+.*/shared/multiformats/.*
 .*/frontend/assets/.*
 .*/frontend/controller/service-worker.js
 .*/frontend/utils/blockies.js
-.*/frontend/utils/crypto.js
 .*/frontend/utils/vuexQueue.js
 .*/frontend/model/contracts/misc/flowTyper.js
 .*/historical/.*

Gruntfile.js

@@ -1,5 +1,8 @@
 'use strict'
 
+// TODO: REMOVEME. This prevents tests from running
+// if (process.env['CI']) process.exit(1)
+
 // =======================
 // Entry point.
 //
@@ -39,7 +42,8 @@ const {
   CI = '',
   LIGHTWEIGHT_CLIENT = 'true',
   NODE_ENV = 'development',
-  EXPOSE_SBP = ''
+  EXPOSE_SBP = '',
+  ENABLE_UNSAFE_NULL_CRYPTO = 'false'
 } = process.env
 
 if (!['development', 'production'].includes(NODE_ENV)) {
@@ -205,7 +209,8 @@ module.exports = (grunt) => {
         'process.env.GI_VERSION': `'${GI_VERSION}'`,
         'process.env.LIGHTWEIGHT_CLIENT': `'${LIGHTWEIGHT_CLIENT}'`,
         'process.env.NODE_ENV': `'${NODE_ENV}'`,
-        'process.env.EXPOSE_SBP': `'${EXPOSE_SBP}'`
+        'process.env.EXPOSE_SBP': `'${EXPOSE_SBP}'`,
+        'process.env.ENABLE_UNSAFE_NULL_CRYPTO': `'${ENABLE_UNSAFE_NULL_CRYPTO}'`
       },
       external: ['crypto', '*.eot', '*.ttf', '*.woff', '*.woff2'],
       format: 'esm',
@@ -232,7 +237,7 @@ module.exports = (grunt) => {
     },
     // Native options used when building our service worker(s).
     serviceWorkers: {
-      entryPoints: ['./frontend/controller/serviceworkers/primary.js']
+      entryPoints: ['./frontend/controller/serviceworkers/sw-primary.js']
     }
   }
   esbuildOptionBags.contracts = {
@@ -246,7 +251,7 @@ module.exports = (grunt) => {
     // },
     splitting: false,
     outdir: distContracts,
-    entryPoints: [`${contractsDir}/group.js`, `${contractsDir}/chatroom.js`, `${contractsDir}/identity.js`, `${contractsDir}/mailbox.js`],
+    entryPoints: [`${contractsDir}/group.js`, `${contractsDir}/chatroom.js`, `${contractsDir}/identity.js`],
     external: ['@sbp/sbp']
   }
   // prevent contract hash from changing each time we build them
@@ -375,7 +380,7 @@ module.exports = (grunt) => {
       // The `--require` flag ensures custom Babel support in our test files.
       test: {
         cmd: 'node node_modules/mocha/bin/mocha --require ./scripts/mocha-helper.js --exit -R spec --bail "./{test/,!(node_modules|ignored|dist|historical|test)/**/}*.test.js"',
-        options: { env: process.env }
+        options: { env: { ...process.env, ENABLE_UNSAFE_NULL_CRYPTO: 'true' } }
       },
       chelDeployAll: 'find contracts -iname "*.manifest.json" | xargs -r ./node_modules/.bin/chel deploy ./data'
     }
@@ -461,7 +466,14 @@ module.exports = (grunt) => {
       }
     }[command]
     grunt.log.writeln(`cypress: running in "${command}" mode...`)
-    cypress[command]({ config: { baseUrl: process.env.API_URL }, ...options })
+    cypress[command]({
+      config: { baseUrl: process.env.API_URL },
+      // Exclude some spec files for CI runs
+      // group-large|group-proposals are excluded because they take
+      // comparatively long
+      ...process.env.CI && { spec: 'test/cypress/integration/!(group-large|group-proposals).spec.js' },
+      ...options
+    })
       .then(r => done(r.totalFailed === 0)).catch(done)
   })
 
@@ -648,6 +660,7 @@ module.exports = (grunt) => {
 
   grunt.registerTask('test', ['build', 'exec:chelDeployAll', 'backend:launch', 'exec:test', 'cypress'])
   grunt.registerTask('test:unit', ['backend:launch', 'exec:test'])
+  grunt.registerTask('test:cypress', ['build', 'exec:chelDeployAll', 'backend:launch', 'cypress'])
 
   // -------------------------------------------------------------------------
   //  Process event handlers

backend/dashboard/views/pages/Contracts.vue

@@ -54,7 +54,6 @@ export default {
         contractFilter: { id: 'all-contracts', name: L('All contracts') },
         filterOptions: [
           { id: 'all-contracts', name: L('All contracts') },
-          { id: 'mailbox', name: L('Mailbox') },
           { id: 'chatroom', name: L('Chatroom') },
           { id: 'identity', name: L('Identity') },
           { id: 'group', name: L('Group') }

backend/dashboard/views/utils/dummy-data.js

@@ -1,7 +1,7 @@
 /* eslint-disable */
 
 // dummy placeholder data to be used in various pages
-import { blake32Hash } from '@common/functions.js'
+import { createCID } from '@common/functions.js'
 import { addTimeToDate, MONTHS_MILLIS } from '@common/cdTimeUtils.js'
 import L from '@common/translations.js'
 
@@ -10,21 +10,20 @@ const fakeUsers = ['Flex Kubin', 'Attila Hun', 'Childish Gambino', 'Ken M', 'Mar
 const PAST_THREE_MONTHS = -3 * MONTHS_MILLIS
 const randomPastDate = () => addTimeToDate(new Date(), Math.floor(Math.random() * PAST_THREE_MONTHS))
 const randomFromArray = arr => arr[Math.floor(Math.random() * arr.length)]
-const randomHash = () => blake32Hash(Math.random().toString(16).slice(2))
+const randomHash = () => createCID(Math.random().toString(16).slice(2))
 
 // Contracts.vue //
 const contractDummyData = []
 const manifestDummy = {
   'gi.contracts/identity': {"head":{"manifestVersion":"1.0.0"},"body":"{\"version\":\"0.0.1\",\"contract\":{\"hash\":\"21XWnNVpDxBCmdcQvko21FoGWpTXhNydrWfsCxB1VdfWg2wtxA\",\"file\":\"identity.js\"},\"authors\":[{\"cipher\":\"algo\",\"key\":\"<pubkey from deploy-key.json>\"},{\"cipher\":\"algo\",\"key\":\"<pubkey from alex.json>\"}],\"contractSlim\":{\"file\":\"identity-slim.js\",\"hash\":\"21XWnNSeKaQFg61hqcw9FvNV3TPZpqdBnNYQSVz3tKPVWygToL\"}}","signature":{"key":"<which of the \'authors\' keys was used to sign \'body\'>","signature":"<signature>"}},
-  'gi.contracts/mailbox': {"head":{"manifestVersion":"1.0.0"},"body":"{\"version\":\"0.0.1\",\"contract\":{\"hash\":\"21XWnNQjQwf4fPBBqBJtPee3SMVbHrQSju4tu6SRjHNPfDBo8Q\",\"file\":\"mailbox.js\"},\"authors\":[{\"cipher\":\"algo\",\"key\":\"<pubkey from deploy-key.json>\"},{\"cipher\":\"algo\",\"key\":\"<pubkey from alex.json>\"}],\"contractSlim\":{\"file\":\"mailbox-slim.js\",\"hash\":\"21XWnNXUFWQDPrqeADPHPyYueAtVRoAdmwsMT9XBCWfPtP8DLb\"}}","signature":{"key":"<which of the \'authors\' keys was used to sign \'body\'>","signature":"<signature>"}},
   'gi.contracts/chatroom': {"head":{"manifestVersion":"1.0.0"},"body":"{\"version\":\"0.0.1\",\"contract\":{\"hash\":\"21XWnNPTjY6GrUyqb7GzsLstF5JZhs9b5jADnSVufZipPvtQeA\",\"file\":\"chatroom.js\"},\"authors\":[{\"cipher\":\"algo\",\"key\":\"<pubkey from deploy-key.json>\"},{\"cipher\":\"algo\",\"key\":\"<pubkey from alex.json>\"}],\"contractSlim\":{\"file\":\"chatroom-slim.js\",\"hash\":\"21XWnNRT58PL4Hj1BUv2SaJYZHVFkFAF8bS2FTwQy7XvuXB1QE\"}}","signature":{"key":"<which of the \'authors\' keys was used to sign \'body\'>","signature":"<signature>"}},
   'gi.contracts/group': {"head":{"manifestVersion":"1.0.0"},"body":"{\"version\":\"0.0.1\",\"contract\":{\"hash\":\"21XWnNXo4eU77dCQuWkPZtNTXST4hxd1DGnGMiBsaBB6vkdTZk\",\"file\":\"group.js\"},\"authors\":[{\"cipher\":\"algo\",\"key\":\"<pubkey from deploy-key.json>\"},{\"cipher\":\"algo\",\"key\":\"<pubkey from alex.json>\"}],\"contractSlim\":{\"file\":\"group-slim.js\",\"hash\":\"21XWnNSmnNSZZ6oZfsRmss2KKCSQS5QqVh62Ub7iojRAYwcxRr\"}}","signature":{"key":"<which of the \'authors\' keys was used to sign \'body\'>","signature":"<signature>"}}
 }
 
 for (let i=0; i<20; i++) {
   const item = {
     contractId: randomHash(),
-    type: randomFromArray(['gi.contracts/identity', 'gi.contracts/mailbox', 'gi.contracts/chatroom', 'gi.contracts/group']),
+    type: randomFromArray(['gi.contracts/identity', 'gi.contracts/chatroom', 'gi.contracts/group']),
     size: 3 + 10 * Math.random(),
     spaceUsed: 10 + 20 * Math.random(),
     createdDate: randomPastDate()

backend/database.js

@@ -38,10 +38,11 @@ if (!fs.existsSync(dataFolder)) {
 
 sbp('sbp/selectors/register', {
   'backend/db/streamEntriesAfter': async function (contractID: string, hash: string): Promise<*> {
-    let currentHEAD = await sbp('chelonia/db/latestHash', contractID)
-    if (!currentHEAD) {
+    const latestHEADinfo = await sbp('chelonia/db/latestHEADinfo', contractID)
+    if (!latestHEADinfo) {
       throw Boom.notFound(`contractID ${contractID} doesn't exist!`)
     }
+    let { HEAD: currentHEAD } = latestHEADinfo
     let prefix = '['
     // NOTE: if this ever stops working you can also try Readable.from():
     // https://nodejs.org/api/stream.html#stream_stream_readable_from_iterable_options
@@ -52,7 +53,7 @@ sbp('sbp/selectors/register', {
           const json = `"${strToB64(entry.serialize())}"`
           if (currentHEAD !== hash) {
             this.push(prefix + json)
-            currentHEAD = entry.message().previousHEAD
+            currentHEAD = entry.head().previousHEAD
             prefix = ','
           } else {
             this.push(prefix + json + ']')
@@ -88,7 +89,7 @@ sbp('sbp/selectors/register', {
             this.push(prefix + json)
             prefix = ','
             limit--
-            currentHEAD = entry.message().previousHEAD
+            currentHEAD = entry.head().previousHEAD
           }
         } catch (e) {
           // TODO: properly return an error to caller, see https://nodejs.org/api/stream.html#errors-while-reading
@@ -123,7 +124,7 @@ sbp('sbp/selectors/register', {
             offset--
           }
 
-          currentHEAD = entry.message().previousHEAD
+          currentHEAD = entry.head().previousHEAD
           if (!currentHEAD || (isMet && !offset)) {
             this.push(']')
             this.push(null)
@@ -200,18 +201,18 @@ export default async () => {
     })
     sbp('sbp/selectors/lock', ['chelonia/db/get', 'chelonia/db/set', 'chelonia/db/delete'])
   }
-  // TODO: Update this to only run when persistence is disabled when `¢hel deploy` can target SQLite.
+  // TODO: Update this to only run when persistence is disabled when `chel deploy` can target SQLite.
   if (persistence !== 'fs' || options.fs.dirname !== './data') {
-    const HASH_LENGTH = 50
     // Remember to keep these values up-to-date.
+    const HASH_LENGTH = 52
     const CONTRACT_MANIFEST_MAGIC = '{"head":{"manifestVersion"'
     const CONTRACT_SOURCE_MAGIC = '"use strict";'
     // Preload contract source files and contract manifests into Chelonia DB.
     // Note: the data folder may contain other files if the `fs` persistence mode
     // has been used before. We won't load them here; that's the job of `chel migrate`.
     // Note: our target files are currently deployed with unprefixed hashes as file names.
     // We can take advantage of this to recognize them more easily.
-    // TODO: Update this code when `¢hel deploy` no longer generates unprefixed keys.
+    // TODO: Update this code when `chel deploy` no longer generates unprefixed keys.
     const keys = (await readdir(dataFolder))
       // Skip some irrelevant files.
       .filter(k => k.length === HASH_LENGTH)