feature getodk#668: Entities integrity URL

lib/formats/openrosa.js

@@ -87,5 +87,18 @@ const openRosaErrorTemplate = openRosaMessageBase('error');
 parse(openRosaErrorTemplate);
 const openRosaError = (message) => render(openRosaErrorTemplate, { message });
 
-module.exports = { createdMessage, formList, formManifest, openRosaError };
+// Takes forms: [Form] and optional basePath: String, returns an OpenRosa xformsList
+// response. If basePath is given, it is inserted after domain in the downloadUrl.
+const entityListTemplate = template(200, `<?xml version="1.0" encoding="UTF-8"?>
+  <data>
+    <entities>
+    {{#entities}}
+      <entity id="{{uuid}}">
+        <deleted>{{deleted}}</deleted>
+      </entity>
+    {{/entities}}
+    </entities>
+  </data>`);
+const entityList = (data) => entityListTemplate(data);
+module.exports = { createdMessage, formList, formManifest, openRosaError, entityList };
 

lib/model/query/datasets.js

@@ -428,19 +428,18 @@ const getPublishedBySimilarName = (projectId, name) => ({ maybeOne }) => {
 
 // Gets the dataset information, properties (including which forms each property comes from),
 // and which forms consume the dataset via CSV attachment.
+const _getLinkedForms = (datasetName, projectId) => sql`
+SELECT DISTINCT f."xmlFormId", coalesce(current_def.name, f."xmlFormId") "name" FROM form_attachments fa
+JOIN form_defs fd ON fd.id = fa."formDefId" AND fd."publishedAt" IS NOT NULL
+JOIN forms f ON f.id = fd."formId" AND f."deletedAt" IS NULL
+JOIN form_defs current_def ON f."currentDefId" = current_def.id
+JOIN datasets ds ON ds.id = fa."datasetId"
+WHERE ds.name = ${datasetName}
+  AND ds."projectId" = ${projectId}
+  AND ds."publishedAt" IS NOT NULL
+`;
 const getMetadata = (dataset) => async ({ all, Datasets }) => {
 
-  const _getLinkedForms = (datasetName, projectId) => sql`
-    SELECT DISTINCT f."xmlFormId", coalesce(current_def.name, f."xmlFormId") "name" FROM form_attachments fa
-    JOIN form_defs fd ON fd.id = fa."formDefId" AND fd."publishedAt" IS NOT NULL
-    JOIN forms f ON f.id = fd."formId" AND f."deletedAt" IS NULL
-    JOIN form_defs current_def ON f."currentDefId" = current_def.id
-    JOIN datasets ds ON ds.id = fa."datasetId"
-    WHERE ds.name = ${datasetName}
-      AND ds."projectId" = ${projectId}
-      AND ds."publishedAt" IS NOT NULL
-  `;
-
   const _getSourceForms = (datasetName, projectId) => sql`
     SELECT DISTINCT f."xmlFormId", coalesce(fd.name, f."xmlFormId") "name" FROM datasets ds
     JOIN dataset_form_defs dfd ON ds.id = dfd."datasetId"    
@@ -489,7 +488,6 @@ const getMetadata = (dataset) => async ({ all, Datasets }) => {
   };
 };
 
-
 ////////////////////////////////////////////////////////////////////////////
 // DATASET PROPERTY GETTERS
 
@@ -665,6 +663,28 @@ const getLastUpdateTimestamp = (datasetId) => ({ maybeOne }) =>
     .then((t) => t.orNull())
     .then((t) => (t ? t.loggedAt : null));
 
+
+const canReadForOpenRosa = (auth, datasetName, projectId) => ({ oneFirst }) => oneFirst(sql`
+      WITH linked_forms AS (
+        ${_getLinkedForms(datasetName, projectId)}
+      )
+      SELECT count(1) FROM linked_forms
+      INNER JOIN (
+        SELECT forms."xmlFormId" FROM forms
+        INNER JOIN projects ON projects.id=forms."projectId"
+        INNER JOIN (
+          SELECT "acteeId" FROM assignments
+          INNER JOIN (
+            SELECT id FROM roles WHERE verbs ? 'form.read' OR verbs ? 'open_form.read'
+          ) AS role ON role.id=assignments."roleId"
+          WHERE "actorId"=${auth.actor.map((actor) => actor.id).orElse(-1)}
+        ) AS assignment ON assignment."acteeId" IN ('*', 'form', projects."acteeId", forms."acteeId")
+        WHERE forms.state != 'closed'
+        GROUP BY forms."xmlFormId"
+      ) AS users_forms ON users_forms."xmlFormId" = linked_forms."xmlFormId"
+    `)
+  .then(count => count > 0);
+
 module.exports = {
   createPublishedDataset, createPublishedProperty,
   createOrMerge, publishIfExists,
@@ -674,5 +694,5 @@ module.exports = {
   getProperties, getFieldsByFormDefId,
   getDiff, update, countUnprocessedSubmissions,
   getUnprocessedSubmissions,
-  getLastUpdateTimestamp
+  getLastUpdateTimestamp, canReadForOpenRosa
 };

lib/model/query/entities.js

@@ -851,6 +851,25 @@ const del = (entity) => ({ run }) =>
 
 del.audit = (entity, dataset) => (log) => log('entity.delete', entity.with({ acteeId: dataset.acteeId }), { uuid: entity.uuid });
 
+////////////////////////////////////////////////////////////////////////////////
+// INTEGRITY CHECK
+
+const idFilter = (options) => {
+  const query = options.ifArg('id', ids => sql`uuid IN (${sql.join(ids.split(',').map(id => sql`${id.trim()}`), sql`, `)})`);
+  return query.sql ? query : sql`TRUE`;
+};
+
+const _getAllEntitiesState = (datasetId, options) => sql`
+  SELECT uuid, "deletedAt" IS NOT NULL as deleted 
+  FROM entities 
+  WHERE "datasetId" = ${datasetId} AND ${idFilter(options)}
+  -- union with purged
+  -- union with not approved
+`;
+
+const getEntitiesState = (datasetId, options = QueryOptions.none) =>
+  ({ all }) => all(_getAllEntitiesState(datasetId, options));
+
 module.exports = {
   createNew, _processSubmissionEvent,
   createSource,
@@ -867,5 +886,5 @@ module.exports = {
   countByDatasetId, getById, getDef,
   getAll, getAllDefs, del,
   createEntitiesFromPendingSubmissions,
-  resolveConflict
+  resolveConflict, getEntitiesState
 };

lib/resources/entities.js

@@ -13,6 +13,7 @@ const { Entity } = require('../model/frames');
 const Problem = require('../util/problem');
 const { diffEntityData, extractBulkSource, getWithConflictDetails } = require('../data/entity');
 const { QueryOptions } = require('../util/db');
+const { entityList } = require('../formats/openrosa');
 
 module.exports = (service, endpoint) => {
 
@@ -25,6 +26,22 @@ module.exports = (service, endpoint) => {
     return Entities.getAll(dataset.id, queryOptions);
   }));
 
+  service.get('/projects/:projectId/datasets/:name/integrity', endpoint.openRosa(async ({ Datasets, Entities }, { params, auth, queryOptions }) => {
+    const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound);
+
+    // Anyone with the verb `entity.list` or anyone with read access on a Form
+    // that consumes this dataset can call this endpoint.
+    const canAccessEntityList = await auth.can('entity.list', dataset);
+    if (!canAccessEntityList) {
+      await Datasets.canReadForOpenRosa(auth, params.name, params.projectId)
+        .then(canAccess => canAccess || reject(Problem.user.insufficientRights()));
+    }
+
+    const entities = await Entities.getEntitiesState(dataset.id, queryOptions.allowArgs('id'));
+
+    return entityList({ entities });
+  }));
+
   service.get('/projects/:projectId/datasets/:name/entities/:uuid', endpoint(async ({ Datasets, Entities }, { params, auth, queryOptions }) => {
 
     const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound);

package.json

@@ -68,6 +68,7 @@
     "should": "~13",
     "streamtest": "~1.2",
     "supertest": "^6.3.3",
-    "tmp": "~0.2"
+    "tmp": "~0.2",
+    "xml2js": "^0.5.0"
   }
 }

test/integration/api/entities.js

@@ -2,6 +2,7 @@ const appRoot = require('app-root-path');
 const { testService, testServiceFullTrx } = require('../setup');
 const testData = require('../../data/xml');
 const { sql } = require('slonik');
+const xml2js = require('xml2js');
 const should = require('should');
 const { QueryOptions, queryFuncs } = require('../../../lib/util/db');
 const { getById, createVersion } = require('../../../lib/model/query/entities');
@@ -240,6 +241,106 @@ describe('Entities API', () => {
     }));
   });
 
+  describe('GET /datasets/:name/integrity', () => {
+    it('should return notfound if the dataset does not exist', testEntities(async (service) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.get('/v1/projects/1/datasets/nonexistent/integrity')
+        .set('X-OpenRosa-Version', '1.0')
+        .expect(404);
+    }));
+
+    it('should reject if the user cannot read', testEntities(async (service) => {
+      const asChelsea = await service.login('chelsea');
+
+      await asChelsea.get('/v1/projects/1/datasets/people/integrity')
+        .set('X-OpenRosa-Version', '1.0')
+        .expect(403);
+    }));
+
+    it('should happily return given no entities', testService(async (service) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.post('/v1/projects/1/forms?publish=true')
+        .send(testData.forms.simpleEntity)
+        .expect(200);
+
+      await asAlice.get('/v1/projects/1/datasets/people/integrity')
+        .set('X-OpenRosa-Version', '1.0')
+        .expect(200)
+        .then(async ({ text }) => {
+          const result = await xml2js.parseStringPromise(text, { explicitArray: false });
+          result.data.entities.should.not.have.property('entity');
+        });
+    }));
+
+    it('should return data for app-user with access to consuming Form', testEntities(async (service) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.post('/v1/projects/1/forms?publish=true')
+        .send(testData.forms.withAttachments.replace(/goodone/g, 'people'))
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      const appUser = await asAlice.post('/v1/projects/1/app-users')
+        .send({ displayName: 'test' })
+        .then(({ body }) => body);
+
+      await asAlice.post(`/v1/projects/1/forms/withAttachments/assignments/app-user/${appUser.id}`);
+
+      await service.get(`/v1/key/${appUser.token}/projects/1/datasets/people/integrity`)
+        .set('X-OpenRosa-Version', '1.0')
+        .expect(200)
+        .then(async ({ text }) => {
+          const result = await xml2js.parseStringPromise(text, { explicitArray: false });
+          result.data.entities.entity.length.should.be.eql(2);
+        });
+    }));
+
+    it('should reject for app-user if consuming Form is closed', testEntities(async (service) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.post('/v1/projects/1/forms?publish=true')
+        .send(testData.forms.withAttachments.replace(/goodone/g, 'people'))
+        .set('Content-Type', 'application/xml')
+        .expect(200);
+
+      const appUser = await asAlice.post('/v1/projects/1/app-users')
+        .send({ displayName: 'test' })
+        .then(({ body }) => body);
+
+      await asAlice.post(`/v1/projects/1/forms/withAttachments/assignments/app-user/${appUser.id}`);
+
+      await asAlice.patch('/v1/projects/1/forms/withAttachments')
+        .send({ state: 'closed' })
+        .expect(200);
+
+      await service.get(`/v1/key/${appUser.token}/projects/1/datasets/people/integrity`)
+        .set('X-OpenRosa-Version', '1.0')
+        .expect(403);
+    }));
+
+    it('should return with correct deleted value', testEntities(async (service) => {
+      const asAlice = await service.login('alice');
+
+      await asAlice.delete('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc')
+        .expect(200);
+
+      await asAlice.get(`/v1/projects/1/datasets/people/integrity`)
+        .set('X-OpenRosa-Version', '1.0')
+        .expect(200)
+        .then(async ({ text }) => {
+          const result = await xml2js.parseStringPromise(text, { explicitArray: false });
+          result.data.entities.entity.length.should.be.eql(2);
+          const [first, second] = result.data.entities.entity;
+          first.$.id.should.be.eql('12345678-1234-4123-8234-123456789abc');
+          first.deleted.should.be.eql('true');
+          second.$.id.should.be.eql('12345678-1234-4123-8234-123456789aaa');
+          second.deleted.should.be.eql('false');
+        });
+    }));
+  });
+
   describe('GET /datasets/:name/entities/:uuid', () => {
 
     it('should return notfound if the dataset does not exist', testEntities(async (service) => {