Key Management Refactoring (#1247)

* feat: add key management service and migration service

* chore: refactor webId service to use controlled container mixin and dereference webIds

* feat: add controlled container dereference mixin

CHANGELOG.md

@@ -7,31 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [middleware-0.4.2] - 2023-01-30
 
-**@semapps/signature**
-- Move proxy service to signature package, split keypair service [ab62eebc3](https://github.com/assemblee-virtuelle/semapps/commit/ab62eebc3807f21218384b24e5b743be33471247) (*:warning: Breaking changes*)
+**@semapps/crypto**
+
+- Move proxy service to signature package, split keypair service [ab62eebc3](https://github.com/assemblee-virtuelle/semapps/commit/ab62eebc3807f21218384b24e5b743be33471247) (_:warning: Breaking changes_)
 - Handle non-GET methods on proxy endpoint through multipart/form-data [81793af4](https://github.com/assemblee-virtuelle/semapps/commit/81793af42b96a2e42d868d8a4e33f7dc35ab8eec)
 
 **@semapps/notifications**
+
 - Add formatLink method to single-mail notifications [a5dda0e2](https://github.com/assemblee-virtuelle/semapps/commit/a5dda0e2f6a6121d1ec422a244aaa6d6e3d0e658)
 - Allow to add delay before processing notifications [42a82ebf](https://github.com/assemblee-virtuelle/semapps/commit/42a82ebf7c363ff448a815103517b317985f85b9)
 
 **@semapps/auth**
-- Local login: allow to redirect /auth to a given frontend URL (formUrl) [22e5f1cc](https://github.com/assemblee-virtuelle/semapps/commit/22e5f1ccc5d34f669ac5a869a2e438057120916f) 
+
+- Local login: allow to redirect /auth to a given frontend URL (formUrl) [22e5f1cc](https://github.com/assemblee-virtuelle/semapps/commit/22e5f1ccc5d34f669ac5a869a2e438057120916f)
 - Fix reset password mails if frontendUrl has a trailing slash [4b16b950](https://github.com/assemblee-virtuelle/semapps/commit/4b16b9502467c88c7ad3977e0174945666dab3f8)
 
 **@semapps/webacl**
+
 - Fix getAclUriFromResourceUri if ending slash is missing from baseUrl [923de06f](https://github.com/assemblee-virtuelle/semapps/commit/923de06f2b4045fa3e66903942b09586365f6b89)
 
 ## [frontend-0.4.2] - 2023-01-30
 
 **@semapps/semantic-data-provider**
-- Pod login & proxy authentication [#1091](https://github.com/assemblee-virtuelle/semapps/pull/1091) (*:warning: Breaking changes*)
+
+- Pod login & proxy authentication [#1091](https://github.com/assemblee-virtuelle/semapps/pull/1091) (_:warning: Breaking changes_)
 - Convert sparqlwhere parameter to object if it is a string [#1104](https://github.com/assemblee-virtuelle/semapps/pull/1104)
 
 **@semapps/auth-provider**
+
 - Don't throw error if permissions cannot be fetched [54500de0](https://github.com/assemblee-virtuelle/semapps/commit/54500de024b4e0e9ef0ec357282086a6bebefb65)
 
 **@semapps/interop-components**
+
 - Control lexiconImportForm to avoid creating empty items [#1090](https://github.com/assemblee-virtuelle/semapps/pull/1090)
 - fetchESCO: allow to choose a type of classification ](https://github.com/assemblee-virtuelle/semapps/commit/29784572924524579540eacf87741e3be952d370)
 
@@ -40,27 +47,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Use named routes for moleculer-web [#1099](https://github.com/assemblee-virtuelle/semapps/pull/1099)
 
 **@semapps/core**
+
 - Fix bug created datasets are always secure [d181bfbc](https://github.com/assemblee-virtuelle/semapps/commit/d181bfbc0a6a94c666cc96f4cc1a78e15f111372)
 
 **@semapps/activitypub**
+
 - Fix activitypub.actor.getProfile action [41fa295f](https://github.com/assemblee-virtuelle/semapps/commit/41fa295f9613f9e54b681aacf5f10aabc8a8b6a2)
 
 **@semapps/auth**
+
 - Fix auth.local.login action: return newUser false [6603dc86](https://github.com/assemblee-virtuelle/semapps/commit/6603dc865d70e62c57800d120a8c0de1f2e16a84)
 
 **@semapps/ldp**
+
 - ldp.resource.upload: fix files type [caddbf43](https://github.com/assemblee-virtuelle/semapps/commit/caddbf43a4a93b5d88a9465dc0fd548467d3acfb)
 
 **@semapps/inference**
+
 - InferenceService: handle cases where triples are undefined [fab3aaef](https://github.com/assemblee-virtuelle/semapps/commit/fab3aaef81089f9b132e53f5c2b33485c502154e)
 
 ## [frontend-0.4.1] - 2022-12-08
 
 **@semapps/activitypub-components**
+
 - CommentsField: prevent page to scroll to input on load [1183b6f3](https://github.com/assemblee-virtuelle/semapps/commit/1183b6f31ccf5ce4fe68794acadab266395c22af)
 - CommentsField: add placeholder prop [abebb823](https://github.com/assemblee-virtuelle/semapps/commit/abebb82396afbfa0a655f8ba42327432370bf731)
 
 **@semapps/interop-components**
+
 - LexiconAutocompleteInput: fix no suggestion to add to dictionary on modal show [a3902b68](https://github.com/assemblee-virtuelle/semapps/commit/a3902b68bf19bd910199a321ec535ad81856fd9e)
 
 ## [0.4.0] - 2022-11-23

README.md

@@ -19,7 +19,7 @@ This is where all the semantic web standards are implemented.
 - [WAC / WebACL](https://semapps.org/docs/middleware/webacl)
 - [Webfinger](https://semapps.org/docs/middleware/webfinger)
 - [WebId](https://semapps.org/docs/middleware/webid)
-- [HTTP Signature](https://semapps.org/docs/middleware/signature)
+- [HTTP Signature](https://semapps.org/docs/middleware/crypto/signature)
 
 We use the [Moleculer](https://moleculer.services/) micro-service framework to help create modular backends.
 

src/middleware/.eslintrc.json

@@ -1,11 +1,12 @@
 {
   "env": {
-    "node": true
+    "node": true,
+    "jest": true
   },
   "extends": [
     "plugin:jsdoc/recommended-typescript-flavor",
     "airbnb/base",
-    //  "plugin:@typescript-eslint/recommended-type-checked",
+    // "plugin:@typescript-eslint/recommended-type-checked",
     "plugin:node/recommended",
     "plugin:jest/recommended",
     "prettier"

src/middleware/babel.config.js

@@ -0,0 +1,13 @@
+module.exports = {
+  presets: [
+    [
+      '@babel/preset-env',
+      {
+        targets: {
+          node: 'current'
+        }
+      }
+    ],
+    '@babel/preset-typescript'
+  ]
+};

src/middleware/package.json

@@ -14,16 +14,20 @@
     "preinstall": "git config core.hooksPath .git-hooks"
   },
   "devDependencies": {
-    "@typescript-eslint/eslint-plugin": "^6.0.0",
-    "@typescript-eslint/parser": "^6.0.0",
+    "@babel/core": "^7.24.3",
+    "@babel/preset-env": "^7.24.3",
+    "@babel/preset-typescript": "^7.24.1",
+    "@typescript-eslint/eslint-plugin": "^7.1.1",
+    "@typescript-eslint/parser": "^7.5.0",
     "eslint": "^8.2.0",
-    "eslint-config-airbnb": "19.0.4",
-    "eslint-config-prettier": "^8.8.0",
-    "eslint-plugin-import": "^2.25.3",
-    "eslint-plugin-jest": "^27.2.2",
-    "eslint-plugin-jsdoc": "^46.5.0",
+    "eslint-config-airbnb": "^19.0.4",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-import-resolver-typescript": "^3.6.1",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-jest": "^27.9.0",
+    "eslint-plugin-jsdoc": "^48.2.1",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-prettier": "^5.0.0",
+    "eslint-plugin-prettier": "^5.1.3",
     "lerna": "^8.0.1",
     "nx": "17.2.5",
     "prettier": "^3.1.1",
@@ -37,5 +41,5 @@
   "engines": {
     "node": ">=14"
   },
-  "version": "0.6.0-alpha.3"
+  "version": "0.7.0"
 }

src/middleware/packages/activitypub/services/activitypub/subservices/actor.js

@@ -5,6 +5,7 @@ const { MIME_TYPES } = require('@semapps/mime-types');
 const { ACTOR_TYPES, AS_PREFIX } = require('../../../constants');
 const { defaultToArray, getSlugFromUri, waitForResource } = require('../../../utils');
 
+/** @type {import('moleculer').ServiceSchema} */
 const ActorService = {
   name: 'activitypub.actor',
   dependencies: ['activitypub.collection', 'ldp', 'signature'],
@@ -158,19 +159,6 @@ const ActorService = {
         async () => await this.actions.get({ actorUri, webId: 'system' }, { parentCtx: ctx, meta: { $cache: false } })
       );
     },
-    async generateMissingActorsData(ctx) {
-      for (const containerUri of this.settings.actorsContainers) {
-        const containerData = await ctx.call('ldp.container.get', { containerUri, accept: MIME_TYPES.JSON });
-        for (const actor of containerData['ldp:contains']) {
-          const actorUri = actor.id || actor['@id'];
-          await this.actions.appendActorData({ actorUri, userData: actor }, { parentCtx: ctx });
-          if (!actor.publicKey) {
-            await this.actions.generateKeyPair({ actorUri }, { parentCtx: ctx });
-          }
-          this.broker.info(`Generated missing data for actor ${actorUri}`);
-        }
-      }
-    },
     getCollectionUri: {
       cache: true,
       async handler(ctx) {

src/middleware/packages/activitypub/services/activitypub/subservices/outbox.js

@@ -31,7 +31,7 @@ const OutboxService = {
 
       const collectionExists = await ctx.call('activitypub.collection.exist', { resourceUri: collectionUri });
       if (!collectionExists) {
-        throw E.NotFoundError();
+        throw new E.NotFoundError();
       }
 
       const actorUri = await ctx.call('activitypub.collection.getOwner', { collectionUri, collectionKey: 'outbox' });

src/middleware/packages/activitypub/utils.js

@@ -85,15 +85,23 @@ const arrayOf = value => {
  * @type {import("./utilTypes").waitForResource}
  */
 const waitForResource = async (delayMs, fieldNames, maxTries, callback) => {
+  let result;
   for (let i = 0; i < maxTries; i += 1) {
-    const result = await callback();
+    result = await callback();
     // If a result (and the expected field, if required) is present, return.
     if (result !== undefined && arrayOf(fieldNames).every(fieldName => Object.keys(result).includes(fieldName))) {
       return result;
     }
     await delay(delayMs);
   }
-  throw new Error(`Waiting for resource failed. No results after ${maxTries} tries`);
+
+  const missingProperties = result && fieldNames.filter(fieldName => !Object.keys(result).includes(fieldName));
+
+  throw new Error(
+    `Waiting for resource failed. No results after ${maxTries} tries. The resource is ${
+      result === undefined ? 'undefined' : `missing the following properties: ${JSON.stringify(missingProperties)}`
+    }.`
+  );
 };
 
 const objectDepth = o => (Object(o) === o ? 1 + Math.max(-1, ...Object.values(o).map(objectDepth)) : 0);

src/middleware/packages/auth/mixins/auth.js

@@ -19,7 +19,7 @@ const AuthMixin = {
     accountsDataset: 'settings',
     podProvider: false
   },
-  dependencies: ['api', 'webid'],
+  dependencies: ['api'],
   async created() {
     const { jwtPath, reservedUsernames, accountsDataset, podProvider } = this.settings;
 

src/middleware/packages/auth/mixins/auth.sso.js

@@ -33,8 +33,7 @@ const AuthSSOMixin = {
         webId = accountData.webId;
         newUser = false;
 
-        // TODO update account with recent information
-        // await ctx.call('webid.edit', profileData, { meta: { webId } });
+        // TODO update account with recent profileData information
 
         ctx.emit('auth.connected', { webId, accountData, ssoData }, { meta: { webId: null, dataset: null } });
       } else {
@@ -47,7 +46,7 @@ const AuthSSOMixin = {
           email: profileData.email,
           username: profileData.username
         });
-        webId = await ctx.call('webid.create', this.pickWebIdData({ nick: accountData.username, ...profileData }));
+        webId = await ctx.call('webid.createWebId', this.pickWebIdData({ nick: accountData.username, ...profileData }));
         newUser = true;
 
         // Link the webId with the account

src/middleware/packages/auth/services/auth.local.js

@@ -4,6 +4,7 @@ const sendToken = require('../middlewares/sendToken');
 const { MoleculerError } = require('moleculer').Errors;
 const AuthMailService = require('./mail');
 
+/** @type {import('moleculer').ServiceSchema} */
 const AuthLocalService = {
   name: 'auth',
   mixins: [AuthMixin],
@@ -27,6 +28,7 @@ const AuthLocalService = {
       }
     }
   },
+  dependencies: ['webid'],
   async created() {
     const { mail } = this.settings;
 
@@ -43,6 +45,8 @@ const AuthLocalService = {
   actions: {
     async signup(ctx) {
       const { username, email, password, interactionId, ...rest } = ctx.params;
+      // This is going to get in our way otherwise when waiting for completions.
+      ctx.meta.skipObjectsWatcher = true;
 
       if (username && username.length < 2) {
         throw new Error('The username must be at least 2 characters long');
@@ -56,7 +60,7 @@ const AuthLocalService = {
       });
 
       const profileData = { nick: username, email, ...rest };
-      const webId = await ctx.call('webid.create', this.pickWebIdData(profileData));
+      const webId = await ctx.call('webid.createWebId', this.pickWebIdData(profileData));
 
       // Link the webId with the account
       accountData = await ctx.call('auth.account.attachWebId', { accountUri: accountData['@id'], webId });

src/middleware/packages/core/package.json

@@ -9,12 +9,13 @@
     "@semapps/jsonld": "0.7.0",
     "@semapps/ldp": "0.7.0",
     "@semapps/ontologies": "0.7.0",
-    "@semapps/signature": "0.7.0",
+    "@semapps/crypto": "0.7.0",
     "@semapps/sparql-endpoint": "0.7.0",
     "@semapps/triplestore": "0.7.0",
     "@semapps/void": "0.7.0",
     "@semapps/webacl": "0.7.0",
     "@semapps/webfinger": "0.7.0",
+    "@semapps/webid": "0.7.0",
     "moleculer": "^0.14.19",
     "moleculer-web": "^0.10.0-beta1",
     "url-join": "^4.0.1"

src/middleware/packages/core/service.js

@@ -5,12 +5,13 @@ const { ActivityPubService, FULL_ACTOR_TYPES } = require('@semapps/activitypub')
 const { JsonLdService } = require('@semapps/jsonld');
 const { LdpService, DocumentTaggerMixin } = require('@semapps/ldp');
 const { OntologiesService } = require('@semapps/ontologies');
-const { SignatureService } = require('@semapps/signature');
 const { SparqlEndpointService } = require('@semapps/sparql-endpoint');
 const { TripleStoreService } = require('@semapps/triplestore');
 const { VoidService } = require('@semapps/void');
 const { WebAclService } = require('@semapps/webacl');
 const { WebfingerService } = require('@semapps/webfinger');
+const { KeysService, SignatureService } = require('@semapps/crypto');
+const { WebIdService } = require('@semapps/webid');
 
 const botsContainer = {
   path: '/as/application',
@@ -48,7 +49,8 @@ const CoreService = {
     sparqlEndpoint: {},
     void: {},
     webacl: {},
-    webfinger: {}
+    webfinger: {},
+    webid: {}
   },
   created() {
     const { baseUrl, baseDir, triplestore, containers, ontologies } = this.settings;
@@ -134,12 +136,27 @@ const CoreService = {
     if (this.settings.signature !== false) {
       this.broker.createService(SignatureService, {
         settings: {
-          actorsKeyPairsDir: path.resolve(baseDir, './actors'),
           ...this.settings.signature
         }
       });
     }
 
+    if (this.settings.webId !== false) {
+      this.broker.createService(WebIdService, {
+        settings: {
+          baseUrl,
+          ...this.settings.webid
+        }
+      });
+    }
+
+    this.broker.createService(KeysService, {
+      settings: {
+        actorsKeyPairsDir: path.resolve(baseDir, './actors'),
+        ...this.settings.keys
+      }
+    });
+
     if (this.settings.sparqlEndpoint !== false) {
       this.broker.createService(SparqlEndpointService, {
         settings: {

src/middleware/packages/crypto/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./keys'),
+  ...require('./signature')
+};

src/middleware/packages/crypto/keys/index.js

@@ -0,0 +1,5 @@
+module.exports = {
+  KEY_TYPES: require('./keyTypes'),
+  MigrationService: require('./migration-service'),
+  KeysService: require('./keys-service')
+};