Feat(tenant): Management of multiple instances via tenants (#1703)

* WIP(tenant): Add multi-tenant instances * WIP(tenant): Still work on multi-tenant * WIP(tenant): add multi-tenant for front * WIP(tenant): Work on tests * WIP(tenant): Stil on test * WIP(tenant): Stil on test * WIP(tenant): Make instance concurrency * WIP(e2e): Fix tests * Feat(tenant): End of tests corrections * Fix(tenant): Update e2e-api tests * Fix(tenant): Update e2e-api tests * Fix(tenant): Update e2e tests * Fix(tenant): Update e2e tests * Feat(tenant): Update connection string * Feat(tenant): Update test-e2e-api * Feat(tenant): Update socket function
Inist-CNRS · Sep 15, 2023 · 0ccddc6 · 0ccddc6
1 parent 0b0209c
commit 0ccddc6
Show file tree

Hide file tree

Showing 54 changed files with 1,142 additions and 819 deletions.
diff --git a/cypress/integration/formats.spec.js b/cypress/integration/formats.spec.js
@@ -24,6 +24,7 @@ describe('Transformers & Formats', () => {
                     format: 'list',
                 },
                 searchable: true,
+                syndication: 1,
             });
 
             datasetImportPage.publish();

diff --git a/cypress/support/authentication.js b/cypress/support/authentication.js
@@ -3,7 +3,7 @@ export const login = (username = 'admin', password = 'secret') => {
     cy.get('input[name="username"]').type(username);
     cy.get('input[name="password"]').type(password);
     cy.get('button').click();
-    cy.location('pathname').should('equal', '/');
+    cy.location('pathname').should('equal', '/instance/lodex_test/');
 };
 
 export const logout = () => {
@@ -27,5 +27,9 @@ export const teardown = (withoutLogin = false) => {
         return;
     }
 
+    cy.window().then(win => {
+        win.sessionStorage.setItem('tenant', 'lodex_test');
+    });
+
     logoutAndLoginAs('admin');
 };
diff --git a/cypress/support/datasetImportPage.js b/cypress/support/datasetImportPage.js
@@ -8,10 +8,7 @@ export const openImport = () => {
 
 export const importDataset = (filename, mimeType = 'text/csv') => {
     addFile(filename, mimeType);
-
-    cy.get('.progress-container', { timeout: 500 }).should('be.visible');
-    cy.wait(300);
-    cy.get('[role="grid"]', { timeout: 6000 }).should('exist');
+    cy.get('[role="grid"]', { timeout: 12000 }).should('exist');
 };
 
 export const importOtherDataset = (filename, mimeType = 'text/csv') => {
@@ -23,8 +20,7 @@ export const importOtherDataset = (filename, mimeType = 'text/csv') => {
     cy.get('#confirm-upload', { timeout: 3000 }).should('be.visible');
     cy.wait(300);
     cy.contains('Accept').click({ force: true });
-    cy.get('.progress-container', { timeout: 500 }).should('be.visible');
-    cy.get('[role="grid"]', { timeout: 6000 }).should('exist');
+    cy.get('[role="grid"]', { timeout: 12000 }).should('exist');
 };
 
 export const importMoreDataset = (filename, mimeType = 'text/csv') => {
@@ -133,7 +129,7 @@ export const publish = () => {
 
 export const goToPublishedResources = () => {
     cy.get('.go-published-button', { timeout: 1000 }).click();
-    cy.location('pathname').should('equal', '/');
+    cy.location('pathname').should('equal', '/instance/lodex_test');
 };
 
 export const goToModel = () => {

diff --git a/cypress/support/enrichmentFormPage.js b/cypress/support/enrichmentFormPage.js
@@ -35,10 +35,10 @@ export const fillAdvancedEnrichment = () => {
 };
 
 export const runEnrichment = () => {
-    cy.contains('Run', { timeout: 500 }).click({
+    cy.contains('Run', { timeout: 1500 }).click({
         force: true,
     });
-    cy.get('.progress-container', { timeout: 500 }).should('be.visible');
+    cy.get('.progress-container', { timeout: 4000 }).should('be.visible');
 };
 
 export const checkIfEnrichmentExistInDataset = () => {

diff --git a/cypress/support/menu.js b/cypress/support/menu.js
@@ -72,5 +72,5 @@ export const signOut = () => {
     cy.get('.advanced-page a')
         .contains('Sign out')
         .click();
-    cy.location('pathname').should('equal', '/login');
+    cy.location('pathname').should('equal', '/instance/lodex_test/login');
 };
diff --git a/globalConfig.json b/globalConfig.json
@@ -1 +1 @@
-{"mongoUri":"mongodb://127.0.0.1:40015/jest?","mongoDBName":"jest"}
+{"mongoUri":"mongodb://127.0.0.1:33417/jest?","mongoDBName":"jest"}
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/api/controller/api/enrichment.js b/src/api/controller/api/enrichment.js
@@ -63,7 +63,7 @@ export const putEnrichment = async (ctx, id) => {
 export const deleteEnrichment = async (ctx, id) => {
     try {
         const enrichment = await ctx.enrichment.findOneById(id);
-        const activeJob = await getActiveJob();
+        const activeJob = await getActiveJob(ctx.tenant);
         if (
             activeJob?.data?.jobType === ENRICHER &&
             activeJob?.data?.id === id
@@ -96,7 +96,15 @@ export const enrichmentAction = async (ctx, action, id) => {
 
     if (action === 'launch') {
         await workerQueue
-            .add({ id, jobType: ENRICHER }, { jobId: uuid() })
+            .add(
+                ctx.tenant, // Name of the job
+                {
+                    id,
+                    jobType: ENRICHER,
+                    tenant: ctx.tenant,
+                },
+                { jobId: uuid() },
+            )
             .then(job => {
                 setEnrichmentJobId(ctx, id, job);
             });
@@ -109,7 +117,15 @@ export const enrichmentAction = async (ctx, action, id) => {
         const enrichment = await ctx.enrichment.findOneById(id);
         await ctx.dataset.removeAttribute(enrichment.name);
         await workerQueue
-            .add({ id, jobType: ENRICHER }, { jobId: uuid() })
+            .add(
+                ctx.tenant, // Name of the job
+                {
+                    id,
+                    jobType: ENRICHER,
+                    tenant: ctx.tenant,
+                },
+                { jobId: uuid() },
+            )
             .then(job => {
                 setEnrichmentJobId(ctx, id, job);
             });

diff --git a/src/api/controller/api/export.js b/src/api/controller/api/export.js
@@ -71,7 +71,7 @@ const middlewareScript = async (ctx, scriptNameCalledParam, fieldsParams) => {
         field: parseFieldsParams(fieldsParams),
         match: match,
         ...facetsWithoutId,
-        connectionStringURI: mongoConnectionString,
+        connectionStringURI: mongoConnectionString(ctx.tenant),
         host,
     };
 
@@ -94,7 +94,8 @@ const middlewareScript = async (ctx, scriptNameCalledParam, fieldsParams) => {
             );
         }
     };
-    const workers_url = `${process.env.WORKERS_URL || 'http://localhost:31976'}/exporters/${exporterName}`;
+    const workers_url = `${process.env.WORKERS_URL ||
+        'http://localhost:31976'}/exporters/${exporterName}`;
     console.error('Connecting to workers', workers_url, 'with', query);
     ctx.body = input
         .pipe(

diff --git a/src/api/controller/api/field.js b/src/api/controller/api/field.js
@@ -50,7 +50,7 @@ export const restoreFields = (fileStream, ctx) => {
         dropJobs(ENRICHER);
         return new Promise((resolve, reject) =>
             restore({
-                uri: mongoConnectionString,
+                uri: mongoConnectionString(ctx.tenant),
                 stream: fileStream,
                 parser: 'json',
                 dropCollections: ['field', 'subresource', 'enrichment'],
@@ -164,7 +164,7 @@ export const postField = async ctx => {
     const result = await ctx.field.create(newField);
 
     if (searchable) {
-        await indexSearchableFields();
+        await indexSearchableFields(ctx);
     }
 
     if (result) {
@@ -180,7 +180,7 @@ export const patchField = async (ctx, id) => {
 
     try {
         ctx.body = await ctx.field.updateOneById(id, newField);
-        await indexSearchableFields();
+        await indexSearchableFields(ctx);
     } catch (error) {
         ctx.status = 403;
         ctx.body = { error: error.message };
@@ -230,7 +230,7 @@ export const patchSearchableFields = async ctx => {
             { $set: { searchable: false } },
         );
 
-        await indexSearchableFields();
+        await indexSearchableFields(ctx);
         ctx.body = 'ok';
     } catch (error) {
         ctx.status = 403;
@@ -245,7 +245,7 @@ export const patchSearchableFields = async ctx => {
 
 export const removeField = async (ctx, id) => {
     ctx.body = await ctx.field.removeById(id);
-    await indexSearchableFields();
+    await indexSearchableFields(ctx);
 };
 
 export const exportFields = async ctx => {

diff --git a/src/api/controller/api/job.js b/src/api/controller/api/job.js
@@ -18,12 +18,12 @@ export const getJobLogs = async (ctx, id) => {
 };
 
 export const postCancelJob = async (ctx, type) => {
-    cancelJob(type);
+    cancelJob(ctx, type);
     ctx.status = 200;
 };
 
 export const postClearJobs = async ctx => {
-    clearJobs();
+    clearJobs(ctx);
     ctx.body = { status: 'success' };
     ctx.status = 200;
 };

diff --git a/src/api/controller/api/progress.js b/src/api/controller/api/progress.js
@@ -1,5 +1,5 @@
 import progress from '../../services/progress';
 
 export default async ctx => {
-    ctx.body = progress.getProgress();
+    ctx.body = progress.getProgress(ctx.tenant);
 };
diff --git a/src/api/controller/api/publish.js b/src/api/controller/api/publish.js
@@ -10,7 +10,11 @@ import { PUBLISHER } from '../../workers/publisher';
 const app = new Koa();
 
 export const doPublish = async ctx => {
-    await workerQueue.add({ jobType: PUBLISHER }, { jobId: uuid() });
+    await workerQueue.add(
+        ctx.tenant, // Name of the job
+        { jobType: PUBLISHER, tenant: ctx.tenant },
+        { jobId: uuid() },
+    );
     ctx.status = 200;
     ctx.body = {
         status: 'publishing',
@@ -24,7 +28,7 @@ export const handleClearPublished = async ctx => {
             status: 'success',
         };
     } catch (error) {
-        logger.error('handle clear published error', {
+        logger.error(`handle clear published error - ${ctx.tenant}`, {
             error,
         });
         ctx.body = {

diff --git a/src/api/controller/api/publishFacets.js b/src/api/controller/api/publishFacets.js
@@ -5,17 +5,17 @@ import { jobLogger } from '../../workers/tools';
 export default async (ctx, fields, withProgress = false) => {
     const facetFields = fields.filter(c => c.isFacet);
     if (!facetFields.length) {
-        withProgress && progress.finish();
+        withProgress && progress.finish(ctx.tenant);
         return;
     }
     withProgress &&
-        progress.start({
+        progress.start(ctx.tenant, {
             status: PUBLISH_FACET,
             target: facetFields.length,
             label: 'publishing',
             type: 'publisher',
         });
-    jobLogger.info(ctx.job, 'Publishing facets');
+    jobLogger.info(ctx.job, `Publishing facets - ${ctx.tenant}`);
 
     const names = fields.map(({ name }) => name);
     await ctx.publishedFacet.remove({ field: { $in: names } });
@@ -31,16 +31,16 @@ export default async (ctx, fields, withProgress = false) => {
                         return null;
                     }
                     await ctx.publishedFacet.insertMany(facets);
-                    withProgress && progress.incrementProgress(1);
+                    withProgress && progress.incrementProgress(ctx.tenant, 1);
                 }),
             Promise.resolve(),
         )
         .catch(error => {
             if (!withProgress) {
                 throw error;
             }
-            progress.throw(error);
+            progress.throw(ctx.tenant, error);
         });
-    withProgress && progress.finish();
-    jobLogger.info(ctx.job, 'Facets published');
+    withProgress && progress.finish(ctx.tenant);
+    jobLogger.info(ctx.job, `Facets published - ${ctx.tenant}`);
 };
diff --git a/src/api/controller/api/run.js b/src/api/controller/api/run.js
@@ -49,7 +49,7 @@ const middlewareScript = async (ctx, scriptNameCalledParam, fieldsParams) => {
             orderBy,
             field,
             ...ctx.query,
-            connectionStringURI: mongoConnectionString,
+            connectionStringURI: mongoConnectionString(ctx.tenant),
             host,
         };
         const input = new PassThrough({ objectMode: true });
@@ -76,7 +76,10 @@ const middlewareScript = async (ctx, scriptNameCalledParam, fieldsParams) => {
                 );
             }
         };
-        const workers_url = `${process.env.WORKERS_URL || 'http://localhost:31976'}/routines/${routineName}?${ctx.querystring}`;
+        const workers_url = `${process.env.WORKERS_URL ||
+            'http://localhost:31976'}/routines/${routineName}?${
+            ctx.querystring
+        }`;
         console.error('Connecting to workers', workers_url, 'with', query);
         ctx.body = input
             .pipe(

diff --git a/src/api/controller/api/upload.js b/src/api/controller/api/upload.js
@@ -44,7 +44,7 @@ export const prepareUpload = async (ctx, next) => {
     try {
         await next();
     } catch (error) {
-        progress.throw(error);
+        progress.throw(ctx.tenant, error);
         ctx.status = 500;
         ctx.body = error.message;
     }
@@ -90,9 +90,8 @@ export async function uploadChunkMiddleware(ctx, loaderName) {
         extension,
         customLoader,
     } = ctx.resumable;
-
-    if (progress.getProgress().status === PENDING) {
-        progress.start({
+    if (progress.getProgress(ctx.tenant).status === PENDING) {
+        progress.start(ctx.tenant, {
             status: UPLOADING_DATASET,
             target: 100,
             symbol: '%',
@@ -109,18 +108,23 @@ export async function uploadChunkMiddleware(ctx, loaderName) {
     );
 
     const progression = Math.round((uploadedFileSize * 100) / totalSize);
-    if (progress.getProgress().status === UPLOADING_DATASET) {
-        progress.setProgress(progression === 100 ? 99 : progression);
+    if (progress.getProgress(ctx.tenant).status === UPLOADING_DATASET) {
+        progress.setProgress(
+            ctx.tenant,
+            progression === 100 ? 99 : progression,
+        );
     }
 
     if (uploadedFileSize >= totalSize) {
         await workerQueue.add(
+            ctx.tenant, // Name of the job
             {
                 loaderName,
                 filename,
                 totalChunks,
                 extension,
                 customLoader,
+                tenant: ctx.tenant,
                 jobType: IMPORT,
             },
             { jobId: uuid() },
@@ -137,7 +141,15 @@ export const uploadUrl = async ctx => {
     const { url, loaderName, customLoader } = ctx.request.body;
     const [extension] = url.match(/[^.]*$/);
     await workerQueue.add(
-        { loaderName, url, extension, customLoader, jobType: IMPORT },
+        ctx.tenant, // Name of the job
+        {
+            loaderName,
+            url,
+            extension,
+            customLoader,
+            tenant: ctx.tenant,
+            jobType: IMPORT,
+        },
         { jobId: uuid() },
     );
     ctx.body = {
@@ -164,7 +176,15 @@ export const checkChunkMiddleware = async (ctx, loaderName) => {
 
     if (exists && chunkNumber === totalChunks) {
         await workerQueue.add(
-            { loaderName, filename, totalChunks, extension, jobType: IMPORT },
+            ctx.tenant, // Name of the job
+            {
+                loaderName,
+                filename,
+                totalChunks,
+                extension,
+                tenant: ctx.tenant,
+                jobType: IMPORT,
+            },
             { jobId: uuid() },
         );
     }