Skip to content

Commit

Permalink
Expose md5 hash of file attachments through api (getodk#1292)
Browse files Browse the repository at this point in the history 
* Expose md5 hash of file attachments through api

* updated names of functions and hash/md5 attribute

* code review fixes
  • Loading branch information
@ktuite
ktuite authored Nov 14, 2024
1 parent ca1652e commit 0c3ec3d
Show file tree
Hide file tree
Showing 6 changed files with 42 additions and 31 deletions.
9 changes: 8 additions & 1 deletion lib/model/frames.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,14 @@ Form.Attachment = class extends Frame.define(
fieldTypes(['int4', 'int4', 'int4', 'int4', 'text', 'text', 'timestamptz'])
) {
forApi() {
const data = { name: this.name, type: this.type, exists: (this.blobId != null || this.datasetId != null), blobExists: this.blobId != null, datasetExists: this.datasetId != null };
const data = {
name: this.name,
type: this.type,
hash: this.aux.blob?.md5,
exists: (this.blobId != null || this.datasetId != null),
blobExists: this.blobId != null,
datasetExists: this.datasetId != null
};
if (this.updatedAt != null) data.updatedAt = this.updatedAt;
return data;
}
Expand Down
24 changes: 14 additions & 10 deletions lib/model/query/form-attachments.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,23 +130,27 @@ update.audit = (form, fa, blobId, datasetId = null) => (log) => log('form.attach
////////////////////////////////////////////////////////////////////////////////
// GETTERS

// This unjoiner pulls md5 from blob table (if it exists) and adds it to attachment frame
const _unjoinMd5 = unjoiner(Form.Attachment, Frame.define(into('blob'), 'md5'));

const getAllByFormDefId = (formDefId) => ({ all }) =>
all(sql`select * from form_attachments where "formDefId"=${formDefId} order by name asc`)
.then(map(construct(Form.Attachment)));
all(sql`select ${_unjoinMd5.fields} from form_attachments
left outer join (select id, md5 from blobs) as blobs on form_attachments."blobId"=blobs.id
where "formDefId"=${formDefId} order by name asc`)
.then(map(_unjoinMd5));

// Does not need to be joined with blobs table due to how it is used
const getByFormDefIdAndName = (formDefId, name) => ({ maybeOne }) => maybeOne(sql`
select * from form_attachments where "formDefId"=${formDefId} and "name"=${name}`)
.then(map(construct(Form.Attachment)));

// This unjoiner pulls md5 from blob table (if it exists) and adds it to attachment frame
const _unjoinMd5 = unjoiner(Form.Attachment, Frame.define(into('openRosa'), 'md5'));

// This function decides on the openrosa hash (functionally equivalent to an http Etag)
// This function decides on the OpenRosa hash (functionally equivalent to an http Etag)
// It uses the blob md5 directly if it exists.
// If the attachment is actually an entity list, it looks up the last modified time
// in the database, which is computed from the latest dataset/entity audit timestamp.
const _chooseHash = (attachment) => async ({ Datasets }) => {
if (attachment.blobId) return attachment.with({ openRosaHash: attachment.aux.openRosa.md5 });
// It is dynamic because it can change when a dataset's data is updated.
const _chooseOpenRosaHash = (attachment) => async ({ Datasets }) => {
if (attachment.blobId) return attachment.with({ openRosaHash: attachment.aux.blob.md5 });

if (attachment.datasetId) {
const lastTimestamp = await Datasets.getLastUpdateTimestamp(attachment.datasetId);
Expand All @@ -161,14 +165,14 @@ select ${_unjoinMd5.fields} from form_attachments
left outer join (select id, md5 from blobs) as blobs on form_attachments."blobId"=blobs.id
where "formDefId"=${formDefId}`)
.then(map(_unjoinMd5))
.then((attachments) => Promise.all(attachments.map(FormAttachments._chooseHash)));
.then((attachments) => Promise.all(attachments.map(FormAttachments._chooseOpenRosaHash)));


module.exports = {
createNew, createVersion,
update,
getAllByFormDefId, getByFormDefIdAndName,
getAllByFormDefIdForOpenRosa,
_chooseHash
_chooseOpenRosaHash
};

16 changes: 8 additions & 8 deletions test/integration/api/forms/draft.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,8 +538,8 @@ describe('api: /projects/:id/forms (drafts)', () => {
.expect(200)
.then(({ body }) => {
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: false, blobExists: false, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: false, blobExists: false, datasetExists: false, hash: null },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand Down Expand Up @@ -572,8 +572,8 @@ describe('api: /projects/:id/forms (drafts)', () => {
// eslint-disable-next-line no-param-reassign
delete body[0].updatedAt;
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'greattwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2af2751b79eccfaa8f452331e76e679e' },
{ name: 'greattwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand Down Expand Up @@ -604,8 +604,8 @@ describe('api: /projects/:id/forms (drafts)', () => {
// eslint-disable-next-line no-param-reassign
delete body[0].updatedAt;
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'greattwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2af2751b79eccfaa8f452331e76e679e' },
{ name: 'greattwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand Down Expand Up @@ -1139,8 +1139,8 @@ describe('api: /projects/:id/forms (drafts)', () => {
// eslint-disable-next-line no-param-reassign
delete body[0].updatedAt;
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2af2751b79eccfaa8f452331e76e679e' },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand Down
8 changes: 4 additions & 4 deletions test/integration/api/forms/forms.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1226,8 +1226,8 @@ describe('api: /projects/:id/forms (create, read, update)', () => {
.expect(200)
.then(({ body }) => {
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: false, blobExists: false, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: false, blobExists: false, datasetExists: false, hash: null },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand All @@ -1252,8 +1252,8 @@ describe('api: /projects/:id/forms (create, read, update)', () => {
delete body[0].updatedAt;

body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2241de57bbec8144c8ad387e69b3a3ba' },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand Down
4 changes: 2 additions & 2 deletions test/integration/api/forms/versions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,8 @@ describe('api: /projects/:id/forms (versions)', () => {
delete body[0].updatedAt;

body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2af2751b79eccfaa8f452331e76e679e' },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
})))));

Expand Down
12 changes: 6 additions & 6 deletions test/integration/other/form-entities-version.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,8 @@ describe('Update / migrate entities-version within form', () => {
// eslint-disable-next-line no-param-reassign
delete body[0].updatedAt;
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2241de57bbec8144c8ad387e69b3a3ba' },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
});

Expand Down Expand Up @@ -318,8 +318,8 @@ describe('Update / migrate entities-version within form', () => {
// eslint-disable-next-line no-param-reassign
delete body[0].updatedAt;
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2241de57bbec8144c8ad387e69b3a3ba' },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
});

Expand All @@ -329,8 +329,8 @@ describe('Update / migrate entities-version within form', () => {
// eslint-disable-next-line no-param-reassign
delete body[0].updatedAt;
body.should.eql([
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false }
{ name: 'goodone.csv', type: 'file', exists: true, blobExists: true, datasetExists: false, hash: '2241de57bbec8144c8ad387e69b3a3ba' },
{ name: 'goodtwo.mp3', type: 'audio', exists: false, blobExists: false, datasetExists: false, hash: null }
]);
});
}));
Expand Down

0 comments on commit 0c3ec3d

Please sign in to comment.