Feat(precomputed): Export precomputed as part of model

src/api/controller/api/field.js

@@ -23,6 +23,7 @@ import { ENRICHER } from '../../workers/enricher';
 import { ObjectID } from 'mongodb';
 import generateUid from '../../services/generateUid';
 import { restoreEnrichments } from '../../services/enrichment/enrichment';
+import { PRECOMPUTER } from '../../workers/precomputer';
 
 const sortByFieldUri = (a, b) =>
     (a.name === 'uri' ? -1 : a.position) - (b.name === 'uri' ? -1 : b.position);
@@ -48,12 +49,18 @@ export const restoreFields = (fileStream, ctx) => {
 
     const restoreTask = () => {
         dropJobs(ctx.tenant, ENRICHER);
+        dropJobs(ctx.tenant, PRECOMPUTER);
         return new Promise((resolve, reject) =>
             restore({
                 uri: mongoConnectionString(ctx.tenant),
                 stream: fileStream,
                 parser: 'json',
-                dropCollections: ['field', 'subresource', 'enrichment'],
+                dropCollections: [
+                    'field',
+                    'subresource',
+                    'enrichment',
+                    'precomputed',
+                ],
                 callback: function(err) {
                     err ? reject(err) : resolve();
                 },
@@ -253,13 +260,23 @@ export const exportFields = async ctx => {
     ctx.set('Content-disposition', `attachment; filename=${filename}`);
     ctx.set('Content-type', 'application/x-tar');
     try {
-        const collections = ['field', 'subresource', 'enrichment'];
+        const collections = [
+            'field',
+            'subresource',
+            'enrichment',
+            'precomputed',
+        ];
         const pack = tar.pack();
 
         for (const collection of collections) {
+            let projection = {};
+            if (collection === 'precomputed') {
+                projection = { data: 0 };
+            }
+
             const docs = await ctx.db
                 .collection(collection)
-                .find()
+                .find({}, { projection })
                 .toArray();
 
             for (const doc of docs) {
@@ -285,19 +302,14 @@ export const importFields = asyncBusboyImpl => async ctx => {
 
     try {
         await ctx.restoreFields(fileStream, ctx);
-        // check if model contains an enrichment
         const enrichments = await ctx.enrichment.findAll();
-        if (enrichments.length > 0) {
-            ctx.body = {
-                message: 'Model imported successfully',
-                hasEnrichments: true,
-            };
-        } else {
-            ctx.body = {
-                message: 'Model imported successfully',
-                hasEnrichments: false,
-            };
-        }
+        const precomputed = await ctx.precomputed.findAll();
+
+        ctx.body = {
+            message: 'Model imported successfully',
+            hasEnrichments: enrichments?.length > 0,
+            hasPrecomputed: precomputed?.length > 0,
+        };
         ctx.status = 200;
     } catch (e) {
         ctx.status = 500;

src/api/controller/api/field.spec.js

@@ -122,11 +122,15 @@ describe('field routes', () => {
             enrichment: {
                 findAll: jest.fn(),
             },
+            precomputed: {
+                findAll: jest.fn(),
+            },
         };
         beforeEach(() => {
             ctx.restoreFields.mockClear();
             ctx.set.mockClear();
             ctx.enrichment.findAll.mockClear();
+            ctx.precomputed.findAll.mockClear();
         });
 
         it('should import fields without enrichments', async () => {
@@ -137,6 +141,9 @@ describe('field routes', () => {
             ctx.enrichment.findAll.mockImplementation(() =>
                 Promise.resolve([]),
             );
+            ctx.precomputed.findAll.mockImplementation(() =>
+                Promise.resolve([]),
+            );
 
             await importFields(asyncBusboyImpl)(ctx);
             expect(asyncBusboyImpl).toHaveBeenCalledWith('request');
@@ -161,6 +168,43 @@ describe('field routes', () => {
             expect(ctx.body.hasEnrichments).toEqual(true);
         });
 
+        it('should import fields with precomputed', async () => {
+            const asyncBusboyImpl = jest.fn().mockImplementation(() => ({
+                files: ['file0'],
+            }));
+
+            ctx.precomputed.findAll.mockImplementation(() =>
+                Promise.resolve(['id: 1']),
+            );
+
+            await importFields(asyncBusboyImpl)(ctx);
+            expect(asyncBusboyImpl).toHaveBeenCalledWith('request');
+            expect(ctx.restoreFields).toHaveBeenCalledWith('file0', ctx);
+            expect(ctx.status).toEqual(200);
+            expect(ctx.body.hasPrecomputed).toEqual(true);
+        });
+
+        it('should import fields with precomputed and enrichment', async () => {
+            const asyncBusboyImpl = jest.fn().mockImplementation(() => ({
+                files: ['file0'],
+            }));
+
+            ctx.precomputed.findAll.mockImplementation(() =>
+                Promise.resolve(['id: 1']),
+            );
+
+            ctx.enrichment.findAll.mockImplementation(() =>
+                Promise.resolve(['id: 1']),
+            );
+
+            await importFields(asyncBusboyImpl)(ctx);
+            expect(asyncBusboyImpl).toHaveBeenCalledWith('request');
+            expect(ctx.restoreFields).toHaveBeenCalledWith('file0', ctx);
+            expect(ctx.status).toEqual(200);
+            expect(ctx.body.hasPrecomputed).toEqual(true);
+            expect(ctx.body.hasEnrichments).toEqual(true);
+        });
+
         it('should return 500 in ctx.status and error message in ctx.body on error', async () => {
             const asyncBusboyImpl = jest.fn().mockImplementation(() => ({
                 files: ['file0'],

src/api/workers/tools.js

@@ -87,7 +87,7 @@ export const cancelJob = async (ctx, jobType, subLabel = null) => {
 export const dropJobs = async (tenant, jobType) => {
     const jobs = await workerQueues[tenant].getJobs();
     jobs.forEach(job => {
-        if (!jobType || job.data.jobType === jobType) job.remove();
+        if (!jobType || job?.data?.jobType === jobType) job.remove();
     });
 };
 

src/app/custom/translations.tsv

@@ -896,6 +896,10 @@
 "resource_detail_third"	"Resource detail third"	"Troisième detail de la ressource"
 "dialog_import_has_enrichments"	"Import has enrichments"	"L'import a des enrichissements"
 "dialog_import_has_enrichments_description"     	"Enrichments are present in the model. Do not forget to run them again if they apply to a new dataset."	"Des enrichissements sont présents dans le modèle. Ne pas oublier de les relancer s'ils s'appliquent à un nouveau jeu de données."
+"dialog_import_has_precomputed"	"Import has precomputed"	"L'import a des précalculs"
+"dialog_import_has_precomputed_description"     	"Precomputed are present in the model. Do not forget to run them again if they apply to a new dataset."	"Des précalculs sont présents dans le modèle. Ne pas oublier de les relancer s'ils s'appliquent à un nouveau jeu de données."
+"dialog_import_has_enrichments_and_precomputed"	"Import has enrichments"	"L'import a des enrichissements et des précalculs"
+"dialog_import_has_enrichments_and_precomputed_description"     	"Enrichments and Precomputed are present in the model. Do not forget to run them again if they apply to a new dataset."	"Des enrichissements et des précalculs sont présents dans le modèle. Ne pas oublier de les relancer s'ils s'appliquent à un nouveau jeu de données."
 "see_logs"	"See logs"	"Voir les logs"
 "see_data"	"See precomputed data"	"Voir les données précalculées"
 "download_logs"	"Download logs"	"Télécharger logs"

src/app/js/admin/Appbar/ImportHasRelaunchDialog.js

@@ -0,0 +1,69 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import compose from 'recompose/compose';
+import translate from 'redux-polyglot/translate';
+
+import { polyglot as polyglotPropTypes } from '../../propTypes';
+import {
+    Button,
+    Dialog,
+    DialogActions,
+    DialogContent,
+    DialogTitle,
+} from '@mui/material';
+
+const ImportHasRelaunchDialog = ({ p: polyglot, onClose, data }) => {
+    const actions = [
+        <Button
+            raised
+            key="submit"
+            className="btn-save"
+            onClick={onClose}
+            color="primary"
+            variant="contained"
+        >
+            {polyglot.t('confirm')}
+        </Button>,
+    ];
+
+    const returnTitleTranslationKey = () => {
+        if (data.hasEnrichments && data.hasPrecomputed) {
+            return 'dialog_import_has_enrichments_and_precomputed';
+        } else if (data.hasEnrichments) {
+            return 'dialog_import_has_enrichments';
+        } else if (data.hasPrecomputed) {
+            return 'dialog_import_has_precomputed';
+        }
+    };
+
+    const returnDescriptionTranslationKey = () => {
+        if (data.hasEnrichments && data.hasPrecomputed) {
+            return 'dialog_import_has_enrichments_and_precomputed_description';
+        } else if (data.hasEnrichments) {
+            return 'dialog_import_has_enrichments_description';
+        } else if (data.hasPrecomputed) {
+            return 'dialog_import_has_precomputed_description';
+        }
+    };
+
+    return (
+        <Dialog open onClose={onClose}>
+            <DialogTitle>{polyglot.t(returnTitleTranslationKey())}</DialogTitle>
+            <DialogContent>
+                <b>{polyglot.t(returnDescriptionTranslationKey())}</b>
+            </DialogContent>
+            <DialogActions>{actions}</DialogActions>
+        </Dialog>
+    );
+};
+
+ImportHasRelaunchDialog.propTypes = {
+    p: polyglotPropTypes.isRequired,
+    onClose: PropTypes.func.isRequired,
+    data: PropTypes.shape({
+        hasEnrichments: PropTypes.bool.isRequired,
+        hasPrecomputed: PropTypes.bool.isRequired,
+    }).isRequired,
+};
+
+export default compose(translate)(ImportHasRelaunchDialog);

src/app/js/admin/Appbar/Menu.js

@@ -33,7 +33,7 @@ import ClearDialog from './ClearDialog';
 import jobsApi from '../api/job';
 import { toast } from '../../../../common/tools/toast';
 import ImportModelDialog from '../ImportModelDialog';
-import ImportHasEnrichmentsDialog from './ImportHasEnrichmentsDialog';
+import ImportHasRelaunchDialog from './ImportHasRelaunchDialog';
 import { withRouter } from 'react-router';
 import { DEFAULT_TENANT } from '../../../../common/tools/tenantTools';
 import SettingsIcon from '@mui/icons-material/Settings';
@@ -47,7 +47,8 @@ const MenuComponent = ({
     importFields,
     nbFields,
     importSucceeded,
-    importHasEnrichment,
+    importHasEnrichments,
+    importHasPrecomputed,
     importFailed,
     p: polyglot,
     history,
@@ -61,10 +62,8 @@ const MenuComponent = ({
         showImportFieldsConfirmation,
         setShowImportFieldsConfirmation,
     ] = useState(false);
-    const [
-        showImportedFieldsHasEnrichmentsDialog,
-        setShowImportedFieldsHasEnrichmentsDialog,
-    ] = useState(false);
+    const [showRelaunchDialog, setShowRelaunchDialog] = useState(false);
+    const [dataRelaunchDialog, setDataRelaunchDialog] = useState(null);
 
     const [applyUploadInput, setApplyUploadInput] = useState(false);
 
@@ -89,10 +88,14 @@ const MenuComponent = ({
     }, [nbFields]);
 
     useEffect(() => {
-        if (importHasEnrichment) {
-            setShowImportedFieldsHasEnrichmentsDialog(true);
+        if (importHasEnrichments || importHasPrecomputed) {
+            setShowRelaunchDialog(true);
+            setDataRelaunchDialog({
+                hasEnrichments: importHasEnrichments,
+                hasPrecomputed: importHasPrecomputed,
+            });
         }
-    }, [importHasEnrichment]);
+    }, [importHasEnrichments, importHasPrecomputed]);
 
     const open = !!anchorEl;
 
@@ -378,11 +381,10 @@ const MenuComponent = ({
                     onClose={() => setShowImportFieldsConfirmation(false)}
                 />
             )}
-            {showImportedFieldsHasEnrichmentsDialog && (
-                <ImportHasEnrichmentsDialog
-                    onClose={() =>
-                        setShowImportedFieldsHasEnrichmentsDialog(false)
-                    }
+            {showRelaunchDialog && (
+                <ImportHasRelaunchDialog
+                    onClose={() => setShowRelaunchDialog(false)}
+                    data={dataRelaunchDialog}
                 />
             )}
         </>
@@ -398,7 +400,8 @@ MenuComponent.propTypes = {
     importFields: PropTypes.func.isRequired,
     nbFields: PropTypes.number.isRequired,
     importSucceeded: PropTypes.bool.isRequired,
-    importHasEnrichment: PropTypes.bool.isRequired,
+    importHasEnrichments: PropTypes.bool.isRequired,
+    importHasPrecomputed: PropTypes.bool.isRequired,
     importFailed: PropTypes.bool.isRequired,
     p: polyglotPropTypes.isRequired,
     history: PropTypes.object.isRequired,
@@ -419,7 +422,8 @@ const mapStateToProps = state => ({
     nbFields: fromFields.getAllListFields(state).filter(f => f.name !== 'uri')
         .length,
     importSucceeded: fromImport.hasImportSucceeded(state),
-    importHasEnrichment: fromImport.hasEnrichment(state),
+    importHasEnrichments: fromImport.hasEnrichments(state),
+    importHasPrecomputed: fromImport.hasPrecomputed(state),
     importFailed: fromImport.hasImportFailed(state),
 });
 export default compose(