Merge branch 'master' into run-migrations-logging

docs/api.yaml

@@ -46,6 +46,7 @@ info:
     - [RESTORE](/central-api-entity-management/#restoring-a-deleted-entity) endpoint for Entities.
     - Entities that have been soft-deleted for 30 days will automatically be purged. 
     - [Entities Odata](/central-api-odata-endpoints/#id3) now returns `__system/deletedAt`. It can also be used in $filter, $sort and $select query parameters.
+    - [Integrity](/central-api-openrosa-endpoints/#openrosa-dataset-integrity-api) endpoint for the Entity list.
 
     ## ODK Central v2024.3
 
@@ -9800,6 +9801,12 @@ paths:
         * The Manifest will only output information for files the server actually has in its possession. Any missing expected files will be omitted, as we cannot provide a `hash` or `downloadUrl` for them.
 
         * For Attachments that are linked to a Dataset, the value of `hash` is calculated using the MD5 of the last updated timestamp of the Dataset, instead of the content of the Dataset.
+
+        [Offline Entities support](https://forum.getodk.org/t/openrosa-spec-proposal-support-offline-entities/48052):
+
+        * If an attachment is linked to a Dataset, then `type="entityList"` attribute is added to the `mediaFile` element.
+
+        * `integrityUrl` is also returned for the attachments that are linked to a Dataset.
       operationId: OpenRosa Form Manifest API
       parameters:
       - name: projectId
@@ -9844,6 +9851,12 @@ paths:
                     <hash>md5:a6fdc426037143cf71cced68e2532e3c</hash>
                     <downloadUrl>https://your.odk.server/v1/projects/7/forms/basic/attachments/question2.jpg</downloadUrl>
                   </mediaFile>
+                  <mediaFile type="entityList">
+                    <filename>people.csv</filename>
+                    <hash>md5:9fd39ac868eccdc0c134b3b7a6a25eb7</hash>
+                    <downloadUrl>https://your.odk.server/v1/projects/7/forms/basic/attachments/people.csv</downloadUrl>
+                    <integrityUrl>https://your.odk.server/v1/projects/7/datasets/people/integrity</integrityUrl>
+                  </mediaFile>
                 </manifest>
         403:
           description: Forbidden
@@ -9857,6 +9870,79 @@ paths:
                 <OpenRosaResponse xmlns="http://openrosa.org/http/response" items="0">
                   <message nature="error">The authenticated actor does not have rights to perform that action.</message>
                 </OpenRosaResponse>
+  /v1/projects/{projectId}/datasets/{name}/integrity?id={UUIDs}:
+    get:
+      tags:
+      - OpenRosa Endpoints
+      summary: OpenRosa Dataset Integrity API
+      description: |-
+        _(introduced: version 2025.1)_
+
+        This is the fully standards-compliant implementation of the Entities Integrity API as described in [OpenRosa spec proposal: support offline Entities](https://forum.getodk.org/t/openrosa-spec-proposal-support-offline-entities/48052).
+
+        This returns the `deleted` flag of the Entities requested through `id` query parameter. If no `id` is provided then all Entities are return.
+      operationId: OpenRosa Form Manifest API
+      parameters:
+      - name: projectId
+        in: path
+        description: The numeric ID of the Project
+        required: true
+        schema:
+          type: number
+        example: "7"
+      - name: name
+        in: path
+        description: The `name` of the dataset being referenced.
+        required: true
+        schema:
+          type: string
+        example: people
+      - name: id
+        in: query
+        description: The comma separated UUIDs of the Entities
+        required: true
+        schema:
+          type: string
+        example: 6fdfa3b6-64fb-46cf-b98c-c92b57f914b1,97717278-2bf8-4565-88b2-711c88d66e75
+      - name: X-OpenRosa-Version
+        in: header
+        description: e.g. 1.0
+        schema:
+          type: string
+        example: "1.0"
+      responses:
+        200:
+          description: OK
+          headers:
+            X-OpenRosa-Version:
+              schema:
+                type: string
+          content:
+            text/xml:
+              example: |
+                <?xml version="1.0" encoding="UTF-8"?>
+                <data>
+                  <entities>
+                    <entity id="6fdfa3b6-64fb-46cf-b98c-c92b57f914b1">
+                      <deleted>true</deleted>
+                    </entity>
+                    <entity id="97717278-2bf8-4565-88b2-711c88d66e75">
+                      <deleted>false</deleted>
+                    </entity>
+                  </entities>
+                </data>
+        403:
+          description: Forbidden
+          headers:
+            X-OpenRosa-Version:
+              schema:
+                type: string
+          content:
+            text/xml:
+              example: |
+                <OpenRosaResponse xmlns="http://openrosa.org/http/response" items="0">
+                  <message nature="error">The authenticated actor does not have rights to perform that action.</message>
+                </OpenRosaResponse>
   /v1/test/{token}/projects/{projectId}/forms/{xmlFormId}/draft/formList:
     get:
       tags:

lib/data/submission.js

@@ -43,6 +43,11 @@ const submissionXmlToFieldStream = (fields, xml, includeStructuralAttrs = false,
   const stack = new SchemaStack(fields, true);
   let textBuffer = ''; // agglomerates text nodes that come as multiple events.
   const parser = new hparser.Parser({
+    onend: () => {
+      if (!stack.hasExited()) {
+        outStream.destroy(new Error('Stream ended before stack was exhausted.'));
+      }
+    },
     onopentag: (name, attrs) => {
       const field = stack.push(name);
       if (field != null) {

lib/external/sentry.js

@@ -109,8 +109,7 @@ const init = (config) => {
       },
       captureException(err) {
         process.stderr.write('attempted to log Sentry exception in development:\n');
-        process.stderr.write(inspect(err));
-        process.stderr.write('\n');
+        process.stderr.write(inspect(err) + '\n');
       },
       configureScope: noop
     };

lib/formats/openrosa.js

@@ -12,6 +12,7 @@
 
 const { mergeRight } = require('ramda');
 const { parse, render } = require('mustache');
+const { attachmentToDatasetName } = require('../util/util');
 
 ////////////////////////////////////////////////////////////////////////////////
 // SETUP
@@ -66,6 +67,9 @@ const formManifestTemplate = template(200, `<?xml version="1.0" encoding="UTF-8"
       <filename>{{name}}</filename>
       <hash>md5:{{openRosaHash}}</hash>
       <downloadUrl>{{{domain}}}{{{basePath}}}/attachments/{{urlName}}</downloadUrl>
+      {{#integrityUrl}}
+      <integrityUrl>{{{integrityUrl}}}</integrityUrl>
+      {{/integrityUrl}}
     </mediaFile>
     {{/hasSource}}
   {{/attachments}}
@@ -77,7 +81,10 @@ const formManifest = (data) => formManifestTemplate(mergeRight(data, {
     attachment.with({
       hasSource: attachment.blobId || attachment.datasetId,
       urlName: encodeURIComponent(attachment.name),
-      isDataset: attachment.datasetId != null
+      isDataset: attachment.datasetId != null,
+      integrityUrl: attachment.datasetId ?
+        `${data.domain}${data.projectPath}/datasets/${encodeURIComponent(attachmentToDatasetName(attachment.name))}/integrity`
+        : null
     }))
 }));
 
@@ -87,5 +94,16 @@ const openRosaErrorTemplate = openRosaMessageBase('error');
 parse(openRosaErrorTemplate);
 const openRosaError = (message) => render(openRosaErrorTemplate, { message });
 
-module.exports = { createdMessage, formList, formManifest, openRosaError };
+const entityListTemplate = template(200, `<?xml version="1.0" encoding="UTF-8"?>
+  <data>
+    <entities>
+    {{#entities}}
+      <entity id="{{uuid}}">
+        <deleted>{{deleted}}</deleted>
+      </entity>
+    {{/entities}}
+    </entities>
+  </data>`);
+const entityList = (data) => entityListTemplate(data);
+module.exports = { createdMessage, formList, formManifest, openRosaError, entityList };
 

lib/http/endpoint.js

@@ -246,7 +246,7 @@ const defaultErrorWriter = (error, request, response) => {
     writeProblemJson(response, error);
   } else {
     debugger; // trip debugger if attached.
-    process.stderr.write(inspect(error));
+    process.stderr.write(inspect(error) + '\n');
     response.status(500).type('application/json').send({
       message: 'Internal Server Error',
     });

lib/model/migrations/20180727-03-add-form-attachments-table.js

@@ -31,7 +31,7 @@ const up = (knex) =>
       Promise.all(forms.map((form) => expectedFormAttachments(form.xml)
         .then((expected) => {
           if (uniq(pluck('name', expected)).length < expected.length) {
-            process.stderr.write(`WARNING: a form ${form.xmlFormId} contains an attachment filename collision. It will not correctly support form attachments.`);
+            process.stderr.write(`WARNING: a form ${form.xmlFormId} contains an attachment filename collision. It will not correctly support form attachments.\n`);
             return Promise.resolve();
           }
           return knex.insert(Object.assign({ formId: form.id }, expected))

lib/model/migrations/20250221-01-deletedAt-index-entity.js

@@ -0,0 +1,21 @@
+// Copyright 2025 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+
+const up = async (db) => {
+  await db.raw('CREATE INDEX entities_deletedat_index ON public.entities USING btree ("deletedAt")');
+  await db.raw('CREATE INDEX entity_defs_sourceid_index ON public.entity_defs USING btree ("sourceId");');
+};
+
+const down = async (db) => {
+  await db.raw('DROP INDEX entities_deletedat_index');
+  await db.raw('DROP INDEX entity_defs_sourceid_index');
+};
+
+module.exports = { up, down };

lib/model/query/datasets.js

@@ -426,21 +426,21 @@ const getPublishedBySimilarName = (projectId, name) => ({ maybeOne }) => {
 ////////////////////////////////////////////////////////////////////////////////
 // DATASET METADATA GETTERS
 
+const _getLinkedForms = (datasetName, projectId) => sql`
+SELECT DISTINCT f."xmlFormId", coalesce(current_def.name, f."xmlFormId") "name" FROM form_attachments fa
+JOIN form_defs fd ON fd.id = fa."formDefId" AND fd."publishedAt" IS NOT NULL
+JOIN forms f ON f.id = fd."formId" AND f."deletedAt" IS NULL
+JOIN form_defs current_def ON f."currentDefId" = current_def.id
+JOIN datasets ds ON ds.id = fa."datasetId"
+WHERE ds.name = ${datasetName}
+  AND ds."projectId" = ${projectId}
+  AND ds."publishedAt" IS NOT NULL
+`;
+
 // Gets the dataset information, properties (including which forms each property comes from),
 // and which forms consume the dataset via CSV attachment.
 const getMetadata = (dataset) => async ({ all, Datasets }) => {
 
-  const _getLinkedForms = (datasetName, projectId) => sql`
-    SELECT DISTINCT f."xmlFormId", coalesce(current_def.name, f."xmlFormId") "name" FROM form_attachments fa
-    JOIN form_defs fd ON fd.id = fa."formDefId" AND fd."publishedAt" IS NOT NULL
-    JOIN forms f ON f.id = fd."formId" AND f."deletedAt" IS NULL
-    JOIN form_defs current_def ON f."currentDefId" = current_def.id
-    JOIN datasets ds ON ds.id = fa."datasetId"
-    WHERE ds.name = ${datasetName}
-      AND ds."projectId" = ${projectId}
-      AND ds."publishedAt" IS NOT NULL
-  `;
-
   const _getSourceForms = (datasetName, projectId) => sql`
     SELECT DISTINCT f."xmlFormId", coalesce(fd.name, f."xmlFormId") "name" FROM datasets ds
     JOIN dataset_form_defs dfd ON ds.id = dfd."datasetId"    
@@ -489,7 +489,6 @@ const getMetadata = (dataset) => async ({ all, Datasets }) => {
   };
 };
 
-
 ////////////////////////////////////////////////////////////////////////////
 // DATASET PROPERTY GETTERS
 
@@ -665,6 +664,28 @@ const getLastUpdateTimestamp = (datasetId) => ({ maybeOne }) =>
     .then((t) => t.orNull())
     .then((t) => (t ? t.loggedAt : null));
 
+
+const canReadForOpenRosa = (auth, datasetName, projectId) => ({ oneFirst }) => oneFirst(sql`
+      WITH linked_forms AS (
+        ${_getLinkedForms(datasetName, projectId)}
+      )
+      SELECT count(1) FROM linked_forms
+      INNER JOIN (
+        SELECT forms."xmlFormId" FROM forms
+        INNER JOIN projects ON projects.id=forms."projectId"
+        INNER JOIN (
+          SELECT "acteeId" FROM assignments
+          INNER JOIN (
+            SELECT id FROM roles WHERE verbs ? 'form.read' OR verbs ? 'open_form.read'
+          ) AS role ON role.id=assignments."roleId"
+          WHERE "actorId"=${auth.actor.map((actor) => actor.id).orElse(-1)}
+        ) AS assignment ON assignment."acteeId" IN ('*', 'form', projects."acteeId", forms."acteeId")
+        WHERE forms.state != 'closed'
+        GROUP BY forms."xmlFormId"
+      ) AS users_forms ON users_forms."xmlFormId" = linked_forms."xmlFormId"
+    `)
+  .then(count => count > 0);
+
 module.exports = {
   createPublishedDataset, createPublishedProperty,
   createOrMerge, publishIfExists,
@@ -674,5 +695,5 @@ module.exports = {
   getProperties, getFieldsByFormDefId,
   getDiff, update, countUnprocessedSubmissions,
   getUnprocessedSubmissions,
-  getLastUpdateTimestamp
+  getLastUpdateTimestamp, canReadForOpenRosa
 };

lib/model/query/entities.js

@@ -964,6 +964,34 @@ const purge = (force = false, projectId = null, datasetName = null, entityUuid =
     SELECT COUNT(*) FROM deleted_entities`);
 };
 
+////////////////////////////////////////////////////////////////////////////////
+// INTEGRITY CHECK
+
+const idFilter = (options) => {
+  const query = options.ifArg('id', ids => sql`uuid IN (${sql.join(ids.split(',').map(id => sql`${id.trim()}`), sql`, `)})`);
+  return query.sql ? query : sql`TRUE`;
+};
+
+const _getAllEntitiesState = (datasetId, options) => sql`
+  SELECT uuid, "deletedAt" IS NOT NULL as deleted 
+  FROM entities 
+  WHERE "datasetId" = ${datasetId}
+  AND ${idFilter(options)}
+  UNION
+  SELECT uuid, deleted FROM (
+    SELECT jsonb_array_elements_text(details -> 'entityUuids') AS uuid, TRUE as deleted
+    FROM audits
+    JOIN datasets ON datasets."acteeId" = audits."acteeId"
+    WHERE action = 'entity.purge' 
+    AND datasets.id = ${datasetId}
+  ) purged
+  WHERE ${idFilter(options)}
+  -- union with not approved
+`;
+
+const getEntitiesState = (datasetId, options = QueryOptions.none) =>
+  ({ all }) => all(_getAllEntitiesState(datasetId, options));
+
 module.exports = {
   createNew, _processSubmissionEvent,
   createSource,
@@ -980,5 +1008,5 @@ module.exports = {
   countByDatasetId, getById, getDef,
   getAll, getAllDefs, del,
   createEntitiesFromPendingSubmissions,
-  resolveConflict, restore, purge
+  resolveConflict, restore, purge, getEntitiesState
 };

lib/model/query/users.js

@@ -12,8 +12,6 @@ const { map } = require('ramda');
 const { Actor, User } = require('../frames');
 const { hashPassword } = require('../../util/crypto');
 const { unjoiner, page, sqlEquals, QueryOptions } = require('../../util/db');
-const { reject } = require('../../util/promise');
-const Problem = require('../../util/problem');
 
 const create = (user) => ({ Actors }) => Actors.createSubtype(user);
 create.audit = (user) => (log) => log('user.create', user.actor, { data: user });
@@ -34,11 +32,7 @@ const update = (user, data) => ({ run, one }) => {
 update.audit = (user, data) => (log) => log('user.update', user.actor, { data: data.with(data.actor) });
 
 const updatePassword = (user, cleartext) => ({ run }) =>
-  (cleartext.length < 10
-    ? reject(Problem.user.passwordTooShort())
-    : Buffer.from(cleartext).length > 72
-      ? reject(Problem.user.passwordTooLong())
-      : hashPassword(cleartext))
+  hashPassword(cleartext)
     .then((hash) => run(sql`update users set password=${hash} where "actorId"=${user.actor.id}`));
 updatePassword.audit = (user) => (log) => log('user.update', user.actor, { data: { password: true } });