From 31182b454b7838fab01dc15b167fe655d7fa5aad Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 21 Apr 2020 12:30:11 +0200 Subject: [PATCH 01/16] To search store added: score to results + all() + paging support --- api/src/store/enrichments.ts | 1 + api/src/store/search.ts | 57 ++++++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/api/src/store/enrichments.ts b/api/src/store/enrichments.ts index aa259fdf..ec8aef38 100644 --- a/api/src/store/enrichments.ts +++ b/api/src/store/enrichments.ts @@ -9,6 +9,7 @@ export interface EnrichedProjectDocument { _id: string; project: ProjectDocument; enrichments?: ProjectEnrichments; + score?: number; } export class ProjectEnrichmentStore { diff --git a/api/src/store/search.ts b/api/src/store/search.ts index ab0d04f7..96422442 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -1,6 +1,7 @@ import { Client } from '@elastic/elasticsearch'; import { EnrichedProjectDocument } from './enrichments'; import { loadSchema, Lookups } from '../util/schema'; +import { Search } from '@elastic/elasticsearch/api/requestParams'; interface Hit { _id: string; @@ -34,7 +35,7 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, if (mode_title) { d.mode_title = mode_title; } - const ionization_type_title = lookups.ionization_type.get(d.ionization.ionization_type); + const ionization_type_title = lookups.ionization_type.get(d.ionization.ionization_type); if (ionization_type_title) { d.ionization_type_title = ionization_type_title; } @@ -51,7 +52,7 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, if (doc.enrichments && doc.enrichments.genomes) { doc.enrichments.genomes = Object.entries(doc.enrichments.genomes).map((keyval: any) => { - return {...keyval[1], label: keyval[0]}; + return { ...keyval[1], label: keyval[0] }; }); } // TODO species label fallback for unenriched project @@ -92,12 +93,15 @@ export function collapseHit(hit: Hit): EnrichedProjectDocument { } project._id = hit._id; + project.score = hit._score; return project; } export type FilterField = 'principal_investigator' | 'submitter' | 'genome_type' | 'species' | 'metagenomic_environment' | 'instrument_type' | 'ionization_mode' | 'ionization_type' | 'growth_medium' | 'solvent'; +const DEFAULT_PAGE_SIZE = 10; + export class SearchEngine { private schema: any; private client: Client; @@ -174,22 +178,23 @@ export class SearchEngine { }); } - async search(query: string) { - const { body } = await this.client.search({ - index: this.index, - body: { - 'query': { - 'simple_query_string': { - query - } - } + async all(size = DEFAULT_PAGE_SIZE, from = 0) { + const query = { + match_all: {} + }; + return await this._search(query, size, from, true); + } + + async search(query: string, size = DEFAULT_PAGE_SIZE, from = 0) { + const qbody = { + 'simple_query_string': { + query } - }); - const hits: EnrichedProjectDocument[] = body.hits.hits.map(collapseHit); - return hits; + }; + return await this._search(qbody, size, from); } - async filter(key: FilterField, value: string) { + async filter(key: FilterField, value: string, size = DEFAULT_PAGE_SIZE, from = 0) { const query: any = {}; if (key === 'submitter') { query.bool = { @@ -220,13 +225,27 @@ export class SearchEngine { query.match = {}; query.match[eskey] = value; } - const { body } = await this.client.search({ + return await this._search(query, size, from); + } + + private async _search(query: any, size: number, from: number, sortbymetaboliteid = false) { + const request: any = { index: this.index, + size: size, + from, body: { query } - }); - const hits: EnrichedProjectDocument[] = body.hits.hits.map(collapseHit); - return hits; + }; + if (sortbymetaboliteid) { + request.sort = [ + { 'project.metabolomics.project.metabolights_study_id.keyword': 'desc' }, + { 'project.metabolomics.project.GNPSMassIVE_ID.keyword': 'desc' } + ]; + } + const response = await this.client.search(request); + const data: EnrichedProjectDocument[] = response.body.hits.hits.map(collapseHit); + const total: number = response.body.hits.total.value; + return { data, total }; } } \ No newline at end of file From 49b165ba6eb65b2240b81a63ae498f3750b337da Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 21 Apr 2020 16:55:17 +0200 Subject: [PATCH 02/16] Made GET /projects work paged --- api/src/app.test.ts | 80 ++++++++++++++++++++++++---- api/src/controller.ts | 72 +++++++++++++++++-------- api/src/projectdocumentstore.test.ts | 27 +++++++--- api/src/projectdocumentstore.ts | 24 +++++++-- api/src/store/search.test.ts | 77 ++++++++++++++++++++++---- api/src/store/search.ts | 18 +++++-- api/src/testhelpers.ts | 5 +- app/public/openapi.yaml | 29 +++++++++- 8 files changed, 271 insertions(+), 61 deletions(-) diff --git a/api/src/app.test.ts b/api/src/app.test.ts index 237aa53a..75e5c36d 100644 --- a/api/src/app.test.ts +++ b/api/src/app.test.ts @@ -20,6 +20,13 @@ describe('app', () => { listProjects: async () => { return [] as EnrichedProjectDocument[]; }, + searchProjects: jest.fn().mockImplementation(async () => { + const data: EnrichedProjectDocument[] = []; + return { + data, + total: 0 + }; + }), createProject: async () => { return 'projectid1.1'; }, @@ -89,10 +96,55 @@ describe('app', () => { expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { - data: [] + data: [], + total: 0 }; expect(body).toEqual(expected); }); + + describe.each([ + ['/api/projects', {}], + ['/api/projects?size=12', { size: 12 }], + ['/api/projects?offset=34', { from: 34 }], + ['/api/projects?size=12&offset=34', { size: 12, from: 34 }], + ['/api/projects?q=Justin', { query: 'Justin' }], + ['/api/projects?q=Justin&size=12&offset=34', { query: 'Justin', size: 12, from: 34 }], + ['/api/projects?fk=species&fv=somevalue', { filter: { key: 'species', value: 'somevalue' } }], + ['/api/projects?fk=species&fv=somevalue&size=12&offset=34', { filter: { key: 'species', value: 'somevalue' }, size: 12, from: 34 }], + ])('GET %s', (url, expected) => { + it('should call store.searchProjects', async () => { + await supertest(app).get(url); + expect(store.searchProjects).toBeCalledWith(expected); + }); + }); + + describe.each([ + ['/api/projects?fk=wrongfield&fv=somevalue', 'Invalid `fk`'], + ['/api/projects?fk=species', 'Require both `fk` and `fv` to filter'], + ['/api/projects?fv=somevalue', 'Require both `fk` and `fv` to filter'], + ['/api/projects?q=Justin&fk=species&fv=somevalue', 'Either search with `q` or filter with `fk` and `fv`'], + ['/api/projects?size=foo', 'Size is not an integer'], + ['/api/projects?size=-10', 'Size must be between `1` and `1000`'], + ['/api/projects?size=1110', 'Size must be between `1` and `1000`'], + ['/api/projects?offset=foo', 'Offset is not an integer'], + ['/api/projects?offset=-10', 'Offset must be between `0` and `100000`'], + ['/api/projects?offset=100001', 'Offset must be between `0` and `100000`'], + ])('GET %s', (url, expected) => { + let response: any; + + beforeEach(async () => { + response = await supertest(app).get(url); + }); + + it('should return a 400 status', () => { + expect(response.status).toBe(400); + }); + + it('should return an error message', () => { + const body = JSON.parse(response.text); + expect(body.message).toEqual(expected); + }); + }); }); describe('GET /api/pending/projects', () => { @@ -100,7 +152,7 @@ describe('app', () => { const response = await supertest(app) .get('/api/pending/projects') .set('Authorization', 'Bearer ashdfjhasdlkjfhalksdjhflak') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { @@ -115,7 +167,7 @@ describe('app', () => { const response = await supertest(app) .get('/api/pending/projects/projectid1.1') .set('Authorization', 'Bearer ashdfjhasdlkjfhalksdjhflak') - ; + ; expect(response.status).toBe(404); }); }); @@ -125,7 +177,7 @@ describe('app', () => { const response = await supertest(app) .get('/api/pending/projects/projectid1.1') .set('Authorization', 'Bearer incorrecttoken') - ; + ; expect(response.status).toBe(401); }); }); @@ -172,6 +224,12 @@ describe('app', () => { listProjects: async () => { return [eproject]; }, + searchProjects: async () => { + return { + data: [eproject], + total: 1 + }; + }, createProject: async () => { return 'projectid1.1'; }, @@ -209,7 +267,7 @@ describe('app', () => { const response = await supertest(app) .get('/api/pending/projects/projectid1.1') .set('Authorization', 'Bearer ashdfjhasdlkjfhalksdjhflak') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { @@ -225,7 +283,7 @@ describe('app', () => { const response = await supertest(app) .post('/api/pending/projects/projectid1.1') .set('Authorization', 'Bearer ashdfjhasdlkjfhalksdjhflak') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { @@ -241,7 +299,7 @@ describe('app', () => { const response = await supertest(app) .delete('/api/pending/projects/projectid1.1') .set('Authorization', 'Bearer ashdfjhasdlkjfhalksdjhflak') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { @@ -255,7 +313,7 @@ describe('app', () => { it('should return project', async () => { const response = await supertest(app) .get('/api/projects/projectid1.1') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); expect(body).toEqual(project); @@ -266,7 +324,7 @@ describe('app', () => { it('should return enriched project', async () => { const response = await supertest(app) .get('/api/projects/projectid1.1/enriched') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { @@ -281,7 +339,7 @@ describe('app', () => { it('should return enriched project', async () => { const response = await supertest(app) .get('/api/projects/projectid1.1/history') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { @@ -345,7 +403,7 @@ describe('app', () => { it('should return version info', async () => { const response = await supertest(app) .get('/api/version') - ; + ; expect(response.status).toBe(200); const body = JSON.parse(response.text); const expected: any = { diff --git a/api/src/controller.ts b/api/src/controller.ts index 6658f4af..ae6a3a04 100644 --- a/api/src/controller.ts +++ b/api/src/controller.ts @@ -1,12 +1,13 @@ import { Request, Response } from 'express'; -import { ProjectDocumentStore, NotFoundException, ListOptions } from './projectdocumentstore'; +import { ProjectDocumentStore, NotFoundException, SearchOptions } from './projectdocumentstore'; import { Validator } from './validate'; import { Queue } from 'bull'; import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from './schema'; import { computeStats } from './util/stats'; -import { summarizeProject, compareMetaboliteID } from './summarize'; +import { summarizeProject } from './summarize'; import { ZENODO_DEPOSITION_ID } from './util/secrets'; +import { FilterFields } from './store/search'; function getStore(req: Request) { @@ -81,32 +82,59 @@ export async function denyProject(req: Request, res: Response) { res.json({'message': 'Denied pending project'}); } -export async function listProjects(req: Request, res: Response) { - const store = getStore(req); - const options: ListOptions = {}; - if (req.query.q) { - options.query = req.query.q; +function checkRange(svalue: string, label: string, min: number, max: number) { + const value = parseInt(svalue); + if (Number.isNaN(value)) { + throw `${label} is not an integer`; + } + if (value < min || value > max ) { + throw `${label} must be between \`${min}\` and \`${max}\``; } - if (req.query.fk || req.query.fv) { - if (req.query.fk && req.query.fv) { + return value; +} + +function validateSearchOptions(query: any) { + const options: SearchOptions = {}; + + if (query.q) { + options.query = query.q; + } + if (query.fk || query.fv) { + if (query.fk && query.fv) { + if (query.q) { + throw 'Either search with `q` or filter with `fk` and `fv`'; + } + if (!(query.fk in FilterFields)) { + throw 'Invalid `fk`'; + } options.filter = { - key: req.query.fk, - value: req.query.fv + key: query.fk, + value: query.fv }; } else { - res.status(400); - res.json({ message: 'Require both `fk` and `fv` to filter'}); - return; - } - if (req.query.query) { - res.status(400); - res.json({ message: 'Eiter search with `q` or filter with `fk` and `fv`'}); - return; + throw 'Require both `fk` and `fv` to filter'; } } - const projects = await store.listProjects(options); - const data = projects.map(summarizeProject).sort(compareMetaboliteID).reverse(); - res.json({data}); + if (query.offset) { + options.from = checkRange(query.offset, 'Offset', 0, 100000); + } + if (query.size) { + options.size = checkRange(query.size, 'Size', 1, 1000); + } + return options; +} + +export async function listProjects(req: Request, res: Response) { + const store = getStore(req); + try { + const options = validateSearchOptions(req.query); + const { data: projects, total} = await store.searchProjects(options); + const data = projects.map(summarizeProject); + res.json({data, total}); + } catch (message) { + res.status(400); + res.json({message}); + } } export async function getProject(req: Request, res: Response) { diff --git a/api/src/projectdocumentstore.test.ts b/api/src/projectdocumentstore.test.ts index 5c019686..0a368447 100644 --- a/api/src/projectdocumentstore.test.ts +++ b/api/src/projectdocumentstore.test.ts @@ -137,23 +137,23 @@ describe('ProjectDocumentStore', () => { expect(approved_project).toEqual(submitted_project); }); - describe('listProjects()', () => { + describe('searchProjects()', () => { describe('without arguments', () => { - it('should be listed', async () => { + it('should have project1 in list', async () => { expect.assertions(1); - const projects = await store.listProjects(); + const {data: projects} = await store.searchProjects(); const project_ids = new Set(projects.map(p => p._id)); - expect(project_ids).toEqual(new Set([project_id])); + expect(project_ids).toEqual(new Set(['projectid1'])); }); }); describe('with query=Justin', () => { - it('should be listed', async () => { + it('should have project1 in list', async () => { expect.assertions(1); - const projects = await store.listProjects({ + const {data: projects} = await store.searchProjects({ query: 'Justin' }); const project_ids = new Set(projects.map(p => p._id)); @@ -163,10 +163,10 @@ describe('ProjectDocumentStore', () => { }); describe('with filter `principal_investigator=Pieter C. Dorrestein`', () => { - it('should be listed', async () => { + it('should have project1 in list', async () => { expect.assertions(1); - const projects = await store.listProjects({ + const {data: projects} = await store.searchProjects({ filter: { key: 'principal_investigator', value: 'Pieter C. Dorrestein' @@ -179,6 +179,17 @@ describe('ProjectDocumentStore', () => { }); }); + describe('listProjects()', () => { + it('should have project1 in list', async () => { + expect.assertions(1); + + const projects = await store.listProjects(); + const project_ids = new Set(projects.map(p => p._id)); + + expect(project_ids).toEqual(new Set([project_id])); + }); + }); + describe('addEnrichments()', () => { beforeEach(async () => { client.index.mockClear(); diff --git a/api/src/projectdocumentstore.ts b/api/src/projectdocumentstore.ts index da3c47dc..70979a04 100644 --- a/api/src/projectdocumentstore.ts +++ b/api/src/projectdocumentstore.ts @@ -4,17 +4,19 @@ import { ProjectDocumentDiskStore } from './store/Disk'; import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from './schema'; import logger from './util/logger'; import { ProjectEnrichmentStore, EnrichedProjectDocument } from './store/enrichments'; -import { SearchEngine, FilterField } from './store/search'; +import { SearchEngine, FilterField, DEFAULT_PAGE_SIZE } from './store/search'; import { ProjectEnrichments } from './enrich'; export const NotFoundException = MemoryNotFoundException; -export interface ListOptions { +export interface SearchOptions { query?: string; filter?: { key: FilterField; value: string; }; + from?: number; + size?: number; } export class ProjectDocumentStore { @@ -55,12 +57,24 @@ export class ProjectDocumentStore { return new_project_id; } - async listProjects(options: ListOptions = {}) { + async searchProjects(options: SearchOptions = {}) { + let from = 0; + if (options.from) { + from = options.from; + } + let size = DEFAULT_PAGE_SIZE; + if (options.size) { + size = options.size; + } if (options.query) { - return await this.search_engine.search(options.query); + return await this.search_engine.search(options.query, size, from); } else if (options.filter) { - return await this.search_engine.filter(options.filter.key, options.filter.value); + return await this.search_engine.filter(options.filter.key, options.filter.value, size, from); } + return await this.search_engine.all(size, from); + } + + async listProjects() { const entries = this.memory_store.listProjects(); return await this.enrichment_store.mergeMany(entries); } diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index b83321d6..60936e47 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -2,6 +2,7 @@ import { loadJSONDocument } from '../util/io'; import { EXAMPLE_PROJECT_JSON_FN, mockedElasticSearchClient } from '../testhelpers'; import { SearchEngine, FilterField } from './search'; import { Client } from '@elastic/elasticsearch'; +import { EnrichedProjectDocument } from './enrichments'; jest.mock('@elastic/elasticsearch'); const MockedClient: jest.Mock = Client as any; @@ -51,7 +52,10 @@ describe('new SearchEngine()', () => { _id: 'projectid1', _score: 0.5, _source: esproject - }] + }], + total: { + value: 1 + } } } }); @@ -71,7 +75,7 @@ describe('new SearchEngine()', () => { describe('the added document', () => { let doc: any; - beforeAll(() => { + beforeEach(() => { doc = client.index.mock.calls[0][0].body; }); @@ -116,17 +120,53 @@ describe('new SearchEngine()', () => { }); }); + describe('all(1, 2)', () => { + let hits: any; + + beforeEach(async () => { + hits = await searchEngine.all(1, 2); + }); + + it('should have called client.search', () => { + expect(client.search).toHaveBeenCalledWith({ + index: 'podp', + size: 1, + from: 2, + sort: [ + { 'project.metabolomics.project.metabolights_study_id.keyword': 'desc'}, + { 'project.metabolomics.project.GNPSMassIVE_ID.keyword': 'desc'} + ], + body: { + 'query': { + match_all: {} + } + } + }); + }); + + it('should return hits', async () => { + const expected_project = await genomeScoredProject(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); + }); + }); + describe('search(\'Justin\')', () => { const query = 'Justin'; let hits: any; - beforeAll(async () => { + beforeEach(async () => { hits = await searchEngine.search(query); }); it('should have called client.search', () => { expect(client.search).toHaveBeenCalledWith({ index: 'podp', + from: 0, + size: 10, body: { 'query': { simple_query_string: { @@ -138,8 +178,12 @@ describe('new SearchEngine()', () => { }); it('should return hits', async () => { - const expected = await genomeProject(); - expect(hits).toEqual([expected]); + const expected_project = await genomeScoredProject(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); }); }); @@ -177,6 +221,8 @@ describe('new SearchEngine()', () => { const called = client.search.mock.calls[0][0]; let expected = { index: 'podp', + from: 0, + size: 10, body: { query: { match: expect.anything() @@ -185,7 +231,7 @@ describe('new SearchEngine()', () => { }; if (key === 'submitter') { expected = { - index: 'podp', + ...expected, body: { query: { bool: { @@ -196,7 +242,7 @@ describe('new SearchEngine()', () => { { match: expect.anything() } - ], + ], }, }, }, @@ -206,8 +252,12 @@ describe('new SearchEngine()', () => { }); it('should return hits', async () => { - const expected = await genomeProject(); - expect(hits).toEqual([expected]); + const expected_project = await genomeScoredProject(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); }); }); @@ -300,6 +350,7 @@ async function esGenomeProject() { project.experimental.instrumentation_methods[0].ionization_type_title = 'Electrospray Ionization (ESI)'; const esproject = { _id: 'projectid1', + score: 0.5, project, enrichments: { genomes: [{ @@ -334,5 +385,11 @@ async function genomeProject() { } } }; - return eproject; + return eproject as EnrichedProjectDocument; } + +async function genomeScoredProject() { + const project = await genomeProject(); + project.score = 0.5; + return project; +} \ No newline at end of file diff --git a/api/src/store/search.ts b/api/src/store/search.ts index 96422442..f98d9806 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -1,7 +1,6 @@ import { Client } from '@elastic/elasticsearch'; import { EnrichedProjectDocument } from './enrichments'; import { loadSchema, Lookups } from '../util/schema'; -import { Search } from '@elastic/elasticsearch/api/requestParams'; interface Hit { _id: string; @@ -98,9 +97,22 @@ export function collapseHit(hit: Hit): EnrichedProjectDocument { return project; } -export type FilterField = 'principal_investigator' | 'submitter' | 'genome_type' | 'species' | 'metagenomic_environment' | 'instrument_type' | 'ionization_mode' | 'ionization_type' | 'growth_medium' | 'solvent'; +export enum FilterFields { + 'principal_investigator', + 'submitter', + 'genome_type', + 'species', + 'metagenomic_environment', + 'instrument_type', + 'ionization_mode', + 'ionization_type', + 'growth_medium', + 'solvent' +} + +export type FilterField = keyof typeof FilterFields; -const DEFAULT_PAGE_SIZE = 10; +export const DEFAULT_PAGE_SIZE = 10; export class SearchEngine { private schema: any; diff --git a/api/src/testhelpers.ts b/api/src/testhelpers.ts index b015ece9..61e33365 100644 --- a/api/src/testhelpers.ts +++ b/api/src/testhelpers.ts @@ -24,7 +24,10 @@ export async function mockedElasticSearchClient() { _id: 'projectid1', project } - }] + }], + total: { + value: 1 + } } } }); diff --git a/app/public/openapi.yaml b/app/public/openapi.yaml index 49edfe4b..81218688 100644 --- a/app/public/openapi.yaml +++ b/app/public/openapi.yaml @@ -36,6 +36,8 @@ paths: "species", "metagenomic_environment", "instrument_type", + "ionization_mode", + "ionization_type", "growth_medium", "solvent", ] @@ -45,6 +47,24 @@ paths: example: Pieter C. Dorrestein schema: type: string + - name: size + in: query + description: Page size + example: 50 + schema: + type: integer + default: 10 + minimum: 1 + maximum: 1000 + - name: offset + in: query + description: Page offset + example: 25 + schema: + type: integer + default: 0 + minimum: 0 + maximum: 100000 responses: "200": description: Successful Response @@ -135,10 +155,14 @@ components: type: object properties: data: + description: Projects on current page type: array items: $ref: "#/components/schemas/ProjectSummary" - required: [data] + total: + description: Total number of projects + type: integer + required: [data, total] additionalProperties: false ProjectSummary: type: object @@ -193,6 +217,9 @@ components: $ref: "/schema.json" enrichments: $ref: "#/components/schemas/Enrichments" + score: + description: Search score + type: number required: [_id, project] additionalProperties: false Enrichments: From b0068a2441069c505455c0ed5da10fe3fb847c04 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 21 Apr 2020 17:11:49 +0200 Subject: [PATCH 03/16] Sort is treated differently between js client and rest api of elastic search --- api/src/store/search.test.ts | 4 ++-- api/src/store/search.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 60936e47..98c5e155 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -133,8 +133,8 @@ describe('new SearchEngine()', () => { size: 1, from: 2, sort: [ - { 'project.metabolomics.project.metabolights_study_id.keyword': 'desc'}, - { 'project.metabolomics.project.GNPSMassIVE_ID.keyword': 'desc'} + 'project.metabolomics.project.metabolights_study_id.keyword:desc', + 'project.metabolomics.project.GNPSMassIVE_ID.keyword:desc' ], body: { 'query': { diff --git a/api/src/store/search.ts b/api/src/store/search.ts index f98d9806..4f679d05 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -251,8 +251,8 @@ export class SearchEngine { }; if (sortbymetaboliteid) { request.sort = [ - { 'project.metabolomics.project.metabolights_study_id.keyword': 'desc' }, - { 'project.metabolomics.project.GNPSMassIVE_ID.keyword': 'desc' } + 'project.metabolomics.project.metabolights_study_id.keyword:desc', + 'project.metabolomics.project.GNPSMassIVE_ID.keyword:desc' ]; } const response = await this.client.search(request); From 9426236441dc4f2bc3a4241f1adfb10ddae5f598 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Tue, 21 Apr 2020 17:41:38 +0200 Subject: [PATCH 04/16] Raise default page size to 500 --- api/src/store/search.test.ts | 4 ++-- api/src/store/search.ts | 2 +- app/public/openapi.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 98c5e155..7ed2e2f1 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -166,7 +166,7 @@ describe('new SearchEngine()', () => { expect(client.search).toHaveBeenCalledWith({ index: 'podp', from: 0, - size: 10, + size: 500, body: { 'query': { simple_query_string: { @@ -222,7 +222,7 @@ describe('new SearchEngine()', () => { let expected = { index: 'podp', from: 0, - size: 10, + size: 500, body: { query: { match: expect.anything() diff --git a/api/src/store/search.ts b/api/src/store/search.ts index 4f679d05..e32fb27e 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -112,7 +112,7 @@ export enum FilterFields { export type FilterField = keyof typeof FilterFields; -export const DEFAULT_PAGE_SIZE = 10; +export const DEFAULT_PAGE_SIZE = 500; export class SearchEngine { private schema: any; diff --git a/app/public/openapi.yaml b/app/public/openapi.yaml index 81218688..f2171f7f 100644 --- a/app/public/openapi.yaml +++ b/app/public/openapi.yaml @@ -53,7 +53,7 @@ paths: example: 50 schema: type: integer - default: 10 + default: 500 minimum: 1 maximum: 1000 - name: offset From 83a84d5f206e6a96be2754b19da58a628b636f0a Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 09:15:12 +0200 Subject: [PATCH 05/16] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c0546a7..89e7c01c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added ionization modes to stats page ([#132](https://github.com/iomega/paired-data-form/issues/132)) * Search query examples ([#132](https://github.com/iomega/paired-data-form/issues/132)) +* Paging projects ([#137](https://github.com/iomega/paired-data-form/issues/137)) ### Fixed From 3237bbce6e8c5e364d7c3494e0fd29048d3a72a1 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 09:16:00 +0200 Subject: [PATCH 06/16] Switched from offset paging to page paging --- api/src/app.test.ts | 22 +++++++++++----------- api/src/controller.ts | 12 ++++++++---- app/public/openapi.yaml | 8 ++++---- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/api/src/app.test.ts b/api/src/app.test.ts index 75e5c36d..6860e98e 100644 --- a/api/src/app.test.ts +++ b/api/src/app.test.ts @@ -103,14 +103,14 @@ describe('app', () => { }); describe.each([ - ['/api/projects', {}], - ['/api/projects?size=12', { size: 12 }], - ['/api/projects?offset=34', { from: 34 }], - ['/api/projects?size=12&offset=34', { size: 12, from: 34 }], - ['/api/projects?q=Justin', { query: 'Justin' }], - ['/api/projects?q=Justin&size=12&offset=34', { query: 'Justin', size: 12, from: 34 }], - ['/api/projects?fk=species&fv=somevalue', { filter: { key: 'species', value: 'somevalue' } }], - ['/api/projects?fk=species&fv=somevalue&size=12&offset=34', { filter: { key: 'species', value: 'somevalue' }, size: 12, from: 34 }], + ['/api/projects', { size: 100, from: 0 }], + ['/api/projects?size=12', { size: 12, from: 0 }], + ['/api/projects?page=2', { size: 100, from: 200 }], + ['/api/projects?size=12&page=3', { size: 12, from: 36 }], + ['/api/projects?q=Justin', { query: 'Justin', size: 100, from: 0 }], + ['/api/projects?q=Justin&size=12&page=3', { query: 'Justin', size: 12, from: 36 }], + ['/api/projects?fk=species&fv=somevalue', { filter: { key: 'species', value: 'somevalue' }, size: 100, from: 0 }], + ['/api/projects?fk=species&fv=somevalue&size=12&page=3', { filter: { key: 'species', value: 'somevalue' }, size: 12, from: 36 }], ])('GET %s', (url, expected) => { it('should call store.searchProjects', async () => { await supertest(app).get(url); @@ -126,9 +126,9 @@ describe('app', () => { ['/api/projects?size=foo', 'Size is not an integer'], ['/api/projects?size=-10', 'Size must be between `1` and `1000`'], ['/api/projects?size=1110', 'Size must be between `1` and `1000`'], - ['/api/projects?offset=foo', 'Offset is not an integer'], - ['/api/projects?offset=-10', 'Offset must be between `0` and `100000`'], - ['/api/projects?offset=100001', 'Offset must be between `0` and `100000`'], + ['/api/projects?page=foo', 'Page is not an integer'], + ['/api/projects?page=-10', 'Page must be between `0` and `1000`'], + ['/api/projects?page=1111', 'Page must be between `0` and `1000`'], ])('GET %s', (url, expected) => { let response: any; diff --git a/api/src/controller.ts b/api/src/controller.ts index ae6a3a04..cc5d1682 100644 --- a/api/src/controller.ts +++ b/api/src/controller.ts @@ -7,7 +7,7 @@ import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from './schema'; import { computeStats } from './util/stats'; import { summarizeProject } from './summarize'; import { ZENODO_DEPOSITION_ID } from './util/secrets'; -import { FilterFields } from './store/search'; +import { FilterFields, DEFAULT_PAGE_SIZE } from './store/search'; function getStore(req: Request) { @@ -115,11 +115,15 @@ function validateSearchOptions(query: any) { throw 'Require both `fk` and `fv` to filter'; } } - if (query.offset) { - options.from = checkRange(query.offset, 'Offset', 0, 100000); - } if (query.size) { options.size = checkRange(query.size, 'Size', 1, 1000); + } else { + options.size = DEFAULT_PAGE_SIZE; + } + if (query.page) { + options.from = checkRange(query.page, 'Page', 0, 1000) * options.size; + } else { + options.from = 0; } return options; } diff --git a/app/public/openapi.yaml b/app/public/openapi.yaml index f2171f7f..e460d6a0 100644 --- a/app/public/openapi.yaml +++ b/app/public/openapi.yaml @@ -53,18 +53,18 @@ paths: example: 50 schema: type: integer - default: 500 + default: 100 minimum: 1 maximum: 1000 - - name: offset + - name: page in: query - description: Page offset + description: Page number example: 25 schema: type: integer default: 0 minimum: 0 - maximum: 100000 + maximum: 1000 responses: "200": description: Successful Response From dae3a5e123091971f87e0cdd70de416acf8d193d Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 09:30:45 +0200 Subject: [PATCH 07/16] Store project summary in es and return summaries at GET /projects --- api/src/controller.ts | 5 ++- api/src/store/enrichments.ts | 3 +- api/src/store/search.test.ts | 55 +++++++++++++++++++++++--------- api/src/store/search.ts | 52 ++++++++---------------------- api/src/summarize.test.ts | 61 +----------------------------------- api/src/summarize.ts | 25 ++++----------- api/src/testhelpers.ts | 14 +++++++-- 7 files changed, 76 insertions(+), 139 deletions(-) diff --git a/api/src/controller.ts b/api/src/controller.ts index cc5d1682..b33fa4ed 100644 --- a/api/src/controller.ts +++ b/api/src/controller.ts @@ -132,9 +132,8 @@ export async function listProjects(req: Request, res: Response) { const store = getStore(req); try { const options = validateSearchOptions(req.query); - const { data: projects, total} = await store.searchProjects(options); - const data = projects.map(summarizeProject); - res.json({data, total}); + const hits = await store.searchProjects(options); + res.json(hits); } catch (message) { res.status(400); res.json({message}); diff --git a/api/src/store/enrichments.ts b/api/src/store/enrichments.ts index ec8aef38..a497b977 100644 --- a/api/src/store/enrichments.ts +++ b/api/src/store/enrichments.ts @@ -2,6 +2,7 @@ import Keyv from 'keyv'; import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from '../schema'; import { ProjectEnrichments } from '../enrich'; +import { ProjectSummary } from '../summarize'; const PREFIX = 'enrichment:'; @@ -9,7 +10,7 @@ export interface EnrichedProjectDocument { _id: string; project: ProjectDocument; enrichments?: ProjectEnrichments; - score?: number; + summary?: ProjectSummary; } export class ProjectEnrichmentStore { diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 7ed2e2f1..c6923a53 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -61,13 +61,15 @@ describe('new SearchEngine()', () => { }); }); - it('should have called client.index', () => { + it('should have called client.index', async () => { + const project = await esGenomeProject(); expect(client.index).toHaveBeenCalledWith({ index: 'podp', id: 'projectid1', body: { - project: expect.anything(), - enrichments: expect.anything() + project: project.project, + enrichments: project.enrichments, + summary: project.summary } }); }); @@ -132,9 +134,9 @@ describe('new SearchEngine()', () => { index: 'podp', size: 1, from: 2, + _source: 'summary', sort: [ - 'project.metabolomics.project.metabolights_study_id.keyword:desc', - 'project.metabolomics.project.GNPSMassIVE_ID.keyword:desc' + 'summary.metabolite_id.keyword:desc', ], body: { 'query': { @@ -145,7 +147,7 @@ describe('new SearchEngine()', () => { }); it('should return hits', async () => { - const expected_project = await genomeScoredProject(); + const expected_project = await genomeProjectSummary(); const expected = { data: [expected_project], total: 1 @@ -166,7 +168,8 @@ describe('new SearchEngine()', () => { expect(client.search).toHaveBeenCalledWith({ index: 'podp', from: 0, - size: 500, + size: 100, + _source: 'summary', body: { 'query': { simple_query_string: { @@ -178,7 +181,7 @@ describe('new SearchEngine()', () => { }); it('should return hits', async () => { - const expected_project = await genomeScoredProject(); + const expected_project = await genomeProjectSummary(); const expected = { data: [expected_project], total: 1 @@ -222,7 +225,8 @@ describe('new SearchEngine()', () => { let expected = { index: 'podp', from: 0, - size: 500, + size: 100, + _source: 'summary', body: { query: { match: expect.anything() @@ -252,7 +256,7 @@ describe('new SearchEngine()', () => { }); it('should return hits', async () => { - const expected_project = await genomeScoredProject(); + const expected_project = await genomeProjectSummary(); const expected = { data: [expected_project], total: 1 @@ -350,7 +354,6 @@ async function esGenomeProject() { project.experimental.instrumentation_methods[0].ionization_type_title = 'Electrospray Ionization (ESI)'; const esproject = { _id: 'projectid1', - score: 0.5, project, enrichments: { genomes: [{ @@ -362,6 +365,17 @@ async function esGenomeProject() { 'title': 'Streptomyces sp. CNB091, whole genome shotgun sequencing project', 'url': 'https://www.ncbi.nlm.nih.gov/nuccore/ARJI01000000' }] + }, + summary: { + metabolite_id: 'MSV000078839', + PI_name: 'Marnix Medema', + submitters: 'Justin van der Hooft', + nr_extraction_methods: 3, + nr_genecluster_mspectra_links: 3, + nr_genome_metabolmics_links: 21, + nr_genomes: 3, + nr_growth_conditions: 3, + nr_instrumentation_methods: 1, } }; return esproject; @@ -388,8 +402,19 @@ async function genomeProject() { return eproject as EnrichedProjectDocument; } -async function genomeScoredProject() { - const project = await genomeProject(); - project.score = 0.5; - return project; +async function genomeProjectSummary() { + const summary = { + _id: 'projectid1', + metabolite_id: 'MSV000078839', + PI_name: 'Marnix Medema', + submitters: 'Justin van der Hooft', + nr_extraction_methods: 3, + nr_genecluster_mspectra_links: 3, + nr_genome_metabolmics_links: 21, + nr_genomes: 3, + nr_growth_conditions: 3, + nr_instrumentation_methods: 1, + score: 0.5, + }; + return summary; } \ No newline at end of file diff --git a/api/src/store/search.ts b/api/src/store/search.ts index e32fb27e..eae444ce 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -1,6 +1,7 @@ import { Client } from '@elastic/elasticsearch'; import { EnrichedProjectDocument } from './enrichments'; import { loadSchema, Lookups } from '../util/schema'; +import { summarizeProject, ProjectSummary } from '../summarize'; interface Hit { _id: string; @@ -56,45 +57,18 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, } // TODO species label fallback for unenriched project + doc.summary = summarizeProject(project); + delete doc.summary._id; + delete doc._id; return doc; } -export function collapseHit(hit: Hit): EnrichedProjectDocument { - const project = hit._source; - project.project.experimental.sample_preparation.forEach( - (d: any) => { - delete d.medium_details.medium_title; - delete d.medium_details.metagenomic_environment_title; - } - ); - project.project.experimental.instrumentation_methods.forEach( - (d: any) => { - delete d.instrumentation.instrument_title; - delete d.mode_title; - delete d.ionization_type_title; - } - ); - project.project.experimental.extraction_methods.forEach( - (m: any) => m.solvents.forEach( - (d: any) => delete d.solvent_title - ) - ); - - if (project.enrichments && project.enrichments.genomes) { - const array = project.enrichments.genomes; - const object: any = {}; - array.forEach((item: any) => { - object[item.label] = item; - delete item.label; - }); - project.enrichments.genomes = object; - } - - project._id = hit._id; - project.score = hit._score; - - return project; +export function collapseHit(hit: Hit): ProjectSummary { + const summary = hit._source.summary; + summary.score = hit._score; + summary._id = hit._id; + return summary; } export enum FilterFields { @@ -112,7 +86,7 @@ export enum FilterFields { export type FilterField = keyof typeof FilterFields; -export const DEFAULT_PAGE_SIZE = 500; +export const DEFAULT_PAGE_SIZE = 100; export class SearchEngine { private schema: any; @@ -245,18 +219,18 @@ export class SearchEngine { index: this.index, size: size, from, + _source: 'summary', body: { query } }; if (sortbymetaboliteid) { request.sort = [ - 'project.metabolomics.project.metabolights_study_id.keyword:desc', - 'project.metabolomics.project.GNPSMassIVE_ID.keyword:desc' + 'summary.metabolite_id.keyword:desc' ]; } const response = await this.client.search(request); - const data: EnrichedProjectDocument[] = response.body.hits.hits.map(collapseHit); + const data: ProjectSummary[] = response.body.hits.hits.map(collapseHit); const total: number = response.body.hits.total.value; return { data, total }; } diff --git a/api/src/summarize.test.ts b/api/src/summarize.test.ts index 1ab7e3c0..e96e58a1 100644 --- a/api/src/summarize.test.ts +++ b/api/src/summarize.test.ts @@ -1,5 +1,5 @@ import { loadJSONDocument } from './util/io'; -import { summarizeProject, compareMetaboliteID } from './summarize'; +import { summarizeProject } from './summarize'; import { EXAMPLE_PROJECT_JSON_FN } from './testhelpers'; describe('summarizeProject()', () => { @@ -56,62 +56,3 @@ describe('summarizeProject()', () => { }); }); }); - -describe('compareMetaboliteID()', () => { - test.each([ - ['MSV000078839', '', 'MSV000078839', '', 0], - ['MSV000098839', '', 'MSV000078839', '', 1], - ['MSV000078839', '', 'MSV000098839', '', -1], - ['', 'MTBLS1302', '', 'MTBLS1302', 0], - ['', 'MTBLS2302', '', 'MTBLS1302', 1], - ['', 'MTBLS1302', '', 'MTBLS2302', -1], - ])('%s,%s,%s,%s,%i', (a_gnps: string, a_metabolights: string, b_gnps: string, b_metabolights: string, expected: number) => { - const a = { - '_id': 'some-project-id', - 'GNPSMassIVE_ID': '', - 'PI_name': 'Marnix Medema', - 'submitters': 'Justin van der Hooft & Stefan Verhoeven', - 'metabolights_study_id': '', - 'nr_extraction_methods': 3, - 'nr_genecluster_mspectra_links': 3, - 'nr_genome_metabolmics_links': 21, - 'nr_genomes': 3, - 'nr_growth_conditions': 3, - 'nr_instrumentation_methods': 1 - }; - if (a_gnps) { - a['GNPSMassIVE_ID'] = a_gnps; - } else { - delete a['GNPSMassIVE_ID']; - } - if (a_metabolights) { - a['metabolights_study_id'] = a_metabolights; - } else { - delete a['metabolights_study_id']; - } - const b = { - '_id': 'some-project-id', - 'PI_name': 'Marnix Medema', - 'GNPSMassIVE_ID': '', - 'submitters': 'Justin van der Hooft & Stefan Verhoeven', - 'metabolights_study_id': '', - 'nr_extraction_methods': 3, - 'nr_genecluster_mspectra_links': 3, - 'nr_genome_metabolmics_links': 21, - 'nr_genomes': 3, - 'nr_growth_conditions': 3, - 'nr_instrumentation_methods': 1 - }; - if (b_gnps) { - b['GNPSMassIVE_ID'] = b_gnps; - } else { - delete b['GNPSMassIVE_ID']; - } - if (b_metabolights) { - b['metabolights_study_id'] = b_metabolights; - } else { - delete b['metabolights_study_id']; - } - expect(compareMetaboliteID(a, b)).toEqual(expected); - }); -}); \ No newline at end of file diff --git a/api/src/summarize.ts b/api/src/summarize.ts index 1f855a1b..650d05f4 100644 --- a/api/src/summarize.ts +++ b/api/src/summarize.ts @@ -1,10 +1,9 @@ import { GNPSMassIVE, MetaboLights } from './schema'; import { EnrichedProjectDocument } from './store/enrichments'; -interface ProjectSummary { +export interface ProjectSummary { _id: string; - GNPSMassIVE_ID: string; - metabolights_study_id: string; + metabolite_id: string; PI_name: string; submitters: string; nr_genomes: number; @@ -13,6 +12,7 @@ interface ProjectSummary { nr_instrumentation_methods: number; nr_genome_metabolmics_links: number; nr_genecluster_mspectra_links: number; + score?: number; } function isMetaboLights(project: GNPSMassIVE | MetaboLights): project is MetaboLights { @@ -33,8 +33,7 @@ export const summarizeProject = (d: EnrichedProjectDocument): ProjectSummary => const nr_genecluster_mspectra_links = project['BGC_MS2_links'] ? project['BGC_MS2_links']!.length : 0; const summary = { _id: d._id, - metabolights_study_id: '', - GNPSMassIVE_ID: '', + metabolite_id: '', PI_name: project['personal']['PI_name']!, submitters, nr_genomes, @@ -45,21 +44,9 @@ export const summarizeProject = (d: EnrichedProjectDocument): ProjectSummary => nr_genecluster_mspectra_links, }; if (isMetaboLights(project.metabolomics.project)) { - summary.metabolights_study_id = project.metabolomics.project.metabolights_study_id; + summary.metabolite_id = project.metabolomics.project.metabolights_study_id; } else { - summary.GNPSMassIVE_ID = project.metabolomics.project.GNPSMassIVE_ID; + summary.metabolite_id = project.metabolomics.project.GNPSMassIVE_ID; } return summary; }; - -export const compareMetaboliteID = (a: ProjectSummary, b: ProjectSummary): 1 | -1 | 0 => { - const ia = a.GNPSMassIVE_ID ? a.GNPSMassIVE_ID : a.metabolights_study_id; - const ib = b.GNPSMassIVE_ID ? b.GNPSMassIVE_ID : b.metabolights_study_id; - if (ia < ib) { - return -1; - } - if (ia > ib) { - return 1; - } - return 0; -}; \ No newline at end of file diff --git a/api/src/testhelpers.ts b/api/src/testhelpers.ts index 61e33365..95f242a1 100644 --- a/api/src/testhelpers.ts +++ b/api/src/testhelpers.ts @@ -3,7 +3,6 @@ import { loadJSONDocument } from './util/io'; export const EXAMPLE_PROJECT_JSON_FN = '../app/public/examples/paired_datarecord_MSV000078839_example.json'; export async function mockedElasticSearchClient() { - const project = await loadJSONDocument(EXAMPLE_PROJECT_JSON_FN); const client = { indices: { delete: jest.fn(), @@ -14,6 +13,17 @@ export async function mockedElasticSearchClient() { search: jest.fn(), delete: jest.fn(), }; + const summary = { + metabolite_id: 'MSV000078839', + PI_name: 'Marnix Medema', + submitters: 'Justin van der Hooft', + nr_extraction_methods: 3, + nr_genecluster_mspectra_links: 3, + nr_genome_metabolmics_links: 21, + nr_genomes: 3, + nr_growth_conditions: 3, + nr_instrumentation_methods: 1, + }; client.search.mockResolvedValue({ body: { hits: { @@ -22,7 +32,7 @@ export async function mockedElasticSearchClient() { _score: 0.5, _source: { _id: 'projectid1', - project + summary } }], total: { From 578177e6d21218802bfb0db31f406e653673b908 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 12:24:51 +0200 Subject: [PATCH 08/16] Added sort field and sort order to api/projects --- api/src/controller.ts | 16 ++- api/src/projectdocumentstore.ts | 27 +---- api/src/store/search.test.ts | 50 +++------ api/src/store/search.ts | 190 ++++++++++++++++++++------------ api/src/summarize.test.ts | 6 +- app/src/api.ts | 15 ++- app/src/pages/Projects.tsx | 25 ++++- 7 files changed, 188 insertions(+), 141 deletions(-) diff --git a/api/src/controller.ts b/api/src/controller.ts index b33fa4ed..8c16eb99 100644 --- a/api/src/controller.ts +++ b/api/src/controller.ts @@ -1,13 +1,13 @@ import { Request, Response } from 'express'; -import { ProjectDocumentStore, NotFoundException, SearchOptions } from './projectdocumentstore'; +import { ProjectDocumentStore, NotFoundException } from './projectdocumentstore'; import { Validator } from './validate'; import { Queue } from 'bull'; import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from './schema'; import { computeStats } from './util/stats'; import { summarizeProject } from './summarize'; import { ZENODO_DEPOSITION_ID } from './util/secrets'; -import { FilterFields, DEFAULT_PAGE_SIZE } from './store/search'; +import { FilterFields, DEFAULT_PAGE_SIZE, Order, SearchOptions, SortFields } from './store/search'; function getStore(req: Request) { @@ -125,6 +125,18 @@ function validateSearchOptions(query: any) { } else { options.from = 0; } + if (query.sort) { + if (!(query.sort in SortFields)) { + throw 'Invalid `sort`'; + } + options.sort = query.sort; + } + if (query.order) { + if (!(query.order in Order)) { + throw 'Invalid `order`, must be either `desc` or `asc`'; + } + options.order = Order[query.order as 'desc' | 'asc']; + } return options; } diff --git a/api/src/projectdocumentstore.ts b/api/src/projectdocumentstore.ts index 70979a04..f6e73ded 100644 --- a/api/src/projectdocumentstore.ts +++ b/api/src/projectdocumentstore.ts @@ -4,21 +4,11 @@ import { ProjectDocumentDiskStore } from './store/Disk'; import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from './schema'; import logger from './util/logger'; import { ProjectEnrichmentStore, EnrichedProjectDocument } from './store/enrichments'; -import { SearchEngine, FilterField, DEFAULT_PAGE_SIZE } from './store/search'; +import { SearchEngine, SearchOptions } from './store/search'; import { ProjectEnrichments } from './enrich'; export const NotFoundException = MemoryNotFoundException; -export interface SearchOptions { - query?: string; - filter?: { - key: FilterField; - value: string; - }; - from?: number; - size?: number; -} - export class ProjectDocumentStore { memory_store = new ProjectDocumentMemoryStore(); disk_store: ProjectDocumentDiskStore; @@ -58,20 +48,7 @@ export class ProjectDocumentStore { } async searchProjects(options: SearchOptions = {}) { - let from = 0; - if (options.from) { - from = options.from; - } - let size = DEFAULT_PAGE_SIZE; - if (options.size) { - size = options.size; - } - if (options.query) { - return await this.search_engine.search(options.query, size, from); - } else if (options.filter) { - return await this.search_engine.filter(options.filter.key, options.filter.value, size, from); - } - return await this.search_engine.all(size, from); + return await this.search_engine.search(options); } async listProjects() { diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index c6923a53..71a347d7 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -39,7 +39,6 @@ describe('new SearchEngine()', () => { expect(client.bulk).not.toHaveBeenCalled(); }); - describe('with a single genome project', () => { beforeEach(async () => { const eproject = await genomeProject(); @@ -65,8 +64,9 @@ describe('new SearchEngine()', () => { const project = await esGenomeProject(); expect(client.index).toHaveBeenCalledWith({ index: 'podp', - id: 'projectid1', + id: project.project_id, body: { + project_id: project.project_id, project: project.project, enrichments: project.enrichments, summary: project.summary @@ -122,11 +122,11 @@ describe('new SearchEngine()', () => { }); }); - describe('all(1, 2)', () => { + describe('search({size:1, from:2})', () => { let hits: any; beforeEach(async () => { - hits = await searchEngine.all(1, 2); + hits = await searchEngine.search({ size: 1, from: 2 }); }); it('should have called client.search', () => { @@ -135,9 +135,7 @@ describe('new SearchEngine()', () => { size: 1, from: 2, _source: 'summary', - sort: [ - 'summary.metabolite_id.keyword:desc', - ], + sort: 'summary.metabolite_id.keyword:desc', body: { 'query': { match_all: {} @@ -161,7 +159,7 @@ describe('new SearchEngine()', () => { let hits: any; beforeEach(async () => { - hits = await searchEngine.search(query); + hits = await searchEngine.search({ query }); }); it('should have called client.search', () => { @@ -212,17 +210,17 @@ describe('new SearchEngine()', () => { ['solvent', 'Butanol'], ['ionization_mode', 'Positive'], ['ionization_type', 'Electrospray Ionization (ESI)'] - ])('filter(\'%s\', \'%s\')', (key: FilterField, value) => { + ])('search({filter:{key:\'%s\', value:\'%s\'}})', (key: FilterField, value) => { let hits: any; beforeEach(async () => { client.search.mockClear(); - hits = await searchEngine.filter(key, value); + hits = await searchEngine.search({ filter: { key, value } }); }); it('should have called index.search', () => { expect(client.search).toBeCalled(); const called = client.search.mock.calls[0][0]; - let expected = { + const expected = { index: 'podp', from: 0, size: 100, @@ -233,25 +231,6 @@ describe('new SearchEngine()', () => { } } }; - if (key === 'submitter') { - expected = { - ...expected, - body: { - query: { - bool: { - should: [ - { - match: expect.anything() - }, - { - match: expect.anything() - } - ], - }, - }, - }, - } as any; - } expect(called).toEqual(expected); }); @@ -265,11 +244,16 @@ describe('new SearchEngine()', () => { }); }); - describe('filter(invalid field)', () => { + describe('search({filter:{key:`invalid field`}})', () => { it('should throw Error', async () => { expect.assertions(1); try { - await searchEngine.filter('some invalid key' as any, 'somevalue'); + await searchEngine.search({ + filter: { + key: 'some invalid key' as any, + value: 'somevalue' + } + }); } catch (error) { expect(error).toEqual(new Error('Invalid filter field')); } @@ -353,7 +337,7 @@ async function esGenomeProject() { project.experimental.instrumentation_methods[0].mode_title = 'Positive'; project.experimental.instrumentation_methods[0].ionization_type_title = 'Electrospray Ionization (ESI)'; const esproject = { - _id: 'projectid1', + project_id: 'projectid1', project, enrichments: { genomes: [{ diff --git a/api/src/store/search.ts b/api/src/store/search.ts index eae444ce..da6a2c37 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -3,6 +3,55 @@ import { EnrichedProjectDocument } from './enrichments'; import { loadSchema, Lookups } from '../util/schema'; import { summarizeProject, ProjectSummary } from '../summarize'; +export interface SearchOptions { + query?: string; + filter?: { + key: FilterField; + value: string; + }; + from?: number; + size?: number; + sort?: SortField; + order?: Order; +} + +export enum FilterFields { + principal_investigator = 'project.personal.PI_name.keyword', + submitter = 'summary.submitters.keyword', + genome_type = 'project.genomes.genome_ID.genome_type.keyword', + species = 'enrichments.genomes.species.scientific_name.keyword', + metagenomic_environment = 'project.experimental.sample_preparation.medium_details.metagenomic_environment_title.keyword', + instrument_type = 'project.experimental.instrumentation_methods.instrumentation.instrument_title.keyword', + ionization_mode = 'project.experimental.instrumentation_methods.mode_title.keyword', + ionization_type = 'project.experimental.instrumentation_methods.ionization_type_title.keyword', + growth_medium = 'project.experimental.sample_preparation.medium_details.medium_title.keyword', + solvent = 'project.experimental.extraction_methods.solvents.solvent_title.keyword', +} + +export type FilterField = keyof typeof FilterFields; + +export enum SortFields { + score = 'score', + _id = 'project_id.keyword', + met_id = 'summary.metabolite_id.keyword', + PI_name = 'summary.PI_name.keyword', + submitters = 'summary.submitters.keyword', + nr_genomes = 'summary.nr_genomes', + nr_growth_conditions = 'summary.nr_growth_conditions', + nr_extraction_methods = 'summary.nr_extraction_methods', + nr_instrumentation_methods = 'summary.nr_instrumentation_methods', + nr_genome_metabolomics_links = 'summary.nr_genome_metabolomics_links', + nr_genecluster_mspectra_links = 'summary.nr_genecluster_mspectra_links', +} +export type SortField = keyof typeof SortFields; + +export enum Order { + desc = 'desc', + asc = 'asc' +} + +export const DEFAULT_PAGE_SIZE = 100; + interface Hit { _id: string; _score: number; @@ -59,34 +108,76 @@ export function expandEnrichedProjectDocument(project: EnrichedProjectDocument, doc.summary = summarizeProject(project); delete doc.summary._id; - + // To sort on _id we need to store id in es, but es does not like _id name + // so storing as project_id + doc.project_id = doc._id; delete doc._id; return doc; } export function collapseHit(hit: Hit): ProjectSummary { const summary = hit._source.summary; - summary.score = hit._score; + if (hit._score) { + summary.score = hit._score; + } summary._id = hit._id; + delete hit._source.project_id; return summary; } -export enum FilterFields { - 'principal_investigator', - 'submitter', - 'genome_type', - 'species', - 'metagenomic_environment', - 'instrument_type', - 'ionization_mode', - 'ionization_type', - 'growth_medium', - 'solvent' +function buildAll() { + return { + match_all: {} + }; } -export type FilterField = keyof typeof FilterFields; +function buildQuery(query: string) { + return { + 'simple_query_string': { + query + } + }; +} -export const DEFAULT_PAGE_SIZE = 100; +function buildFilter(key: FilterField, value: string) { + const eskey = FilterFields[key]; + if (!eskey) { + throw new Error('Invalid filter field'); + } + const match: { [key: string]: string } = {}; + match[eskey] = value; + return { + match + }; +} + +function buildQueryFilter(query: any, filter: any) { + return { + bool: { + must: { + query + }, + filter: { + filter + } + } + }; +} + +function buildBody(options: SearchOptions) { + if (options.query && options.filter) { + return buildQueryFilter( + buildQuery(options.query), + buildFilter(options.filter.key, options.filter.value) + ); + } else if (options.query) { + return buildQuery(options.query); + } else if (options.filter) { + return buildFilter(options.filter.key, options.filter.value); + } else { + return buildAll(); + } +} export class SearchEngine { private schema: any; @@ -113,7 +204,7 @@ export class SearchEngine { } private async createIndex() { - // Force all detected integers to be floats + // Force all detected integers to be floats except fields that start with nr_ // Due to dynamic mapping `1` will be mapped to a long while other documents will require a float. await this.client.indices.create({ index: this.index, @@ -121,6 +212,7 @@ export class SearchEngine { mappings: { dynamic_templates: [{ floats: { + unmatch: 'nr_*', match_mapping_type: 'long', mapping: { type: 'float' @@ -164,57 +256,19 @@ export class SearchEngine { }); } - async all(size = DEFAULT_PAGE_SIZE, from = 0) { - const query = { - match_all: {} - }; - return await this._search(query, size, from, true); - } - - async search(query: string, size = DEFAULT_PAGE_SIZE, from = 0) { - const qbody = { - 'simple_query_string': { - query - } - }; - return await this._search(qbody, size, from); - } - - async filter(key: FilterField, value: string, size = DEFAULT_PAGE_SIZE, from = 0) { - const query: any = {}; - if (key === 'submitter') { - query.bool = { - should: [{ - 'match': {} - }, { - 'match': {} - }] - }; - query.bool.should[0].match['project.personal.submitter_name.keyword'] = value; - query.bool.should[1].match['project.personal.submitter_name_secondary.keyword'] = value; - } else { - const key2eskey = { - principal_investigator: 'project.personal.PI_name.keyword', - genome_type: 'project.genomes.genome_ID.genome_type.keyword', - species: 'enrichments.genomes.species.scientific_name.keyword', - metagenomic_environment: 'project.experimental.sample_preparation.medium_details.metagenomic_environment_title.keyword', - instrument_type: 'project.experimental.instrumentation_methods.instrumentation.instrument_title.keyword', - ionization_mode: 'project.experimental.instrumentation_methods.mode_title.keyword', - ionization_type: 'project.experimental.instrumentation_methods.ionization_type_title.keyword', - growth_medium: 'project.experimental.sample_preparation.medium_details.medium_title.keyword', - solvent: 'project.experimental.extraction_methods.solvents.solvent_title.keyword', - }; - const eskey = key2eskey[key]; - if (!eskey) { - throw new Error('Invalid filter field'); - } - query.match = {}; - query.match[eskey] = value; - } - return await this._search(query, size, from); + async search(options: SearchOptions = {}) { + const defaultSort = (options.query || options.filter) ? 'score' : 'met_id'; + const { + size = DEFAULT_PAGE_SIZE, + from = 0, + sort = defaultSort, + order = Order.desc + } = options; + const qbody = buildBody(options); + return await this._search(qbody, size, from, sort, order); } - private async _search(query: any, size: number, from: number, sortbymetaboliteid = false) { + private async _search(query: any, size: number, from: number, sort: SortField, order: Order) { const request: any = { index: this.index, size: size, @@ -224,10 +278,8 @@ export class SearchEngine { query } }; - if (sortbymetaboliteid) { - request.sort = [ - 'summary.metabolite_id.keyword:desc' - ]; + if (sort && sort !== 'score') { + request.sort = SortFields[sort] + ':' + order; } const response = await this.client.search(request); const data: ProjectSummary[] = response.body.hits.hits.map(collapseHit); diff --git a/api/src/summarize.test.ts b/api/src/summarize.test.ts index e96e58a1..28abf89f 100644 --- a/api/src/summarize.test.ts +++ b/api/src/summarize.test.ts @@ -14,10 +14,9 @@ describe('summarizeProject()', () => { const expected = { '_id': 'some-project-id', - 'GNPSMassIVE_ID': 'MSV000078839', + 'metabolite_id': 'MSV000078839', 'PI_name': 'Marnix Medema', 'submitters': 'Justin van der Hooft', - 'metabolights_study_id': '', 'nr_extraction_methods': 3, 'nr_genecluster_mspectra_links': 3, 'nr_genome_metabolmics_links': 21, @@ -41,10 +40,9 @@ describe('summarizeProject()', () => { const expected = { '_id': 'some-project-id', - 'GNPSMassIVE_ID': 'MSV000078839', + 'metabolite_id': 'MSV000078839', 'PI_name': 'Marnix Medema', 'submitters': 'Justin van der Hooft & Stefan Verhoeven', - 'metabolights_study_id': '', 'nr_extraction_methods': 3, 'nr_genecluster_mspectra_links': 3, 'nr_genome_metabolmics_links': 21, diff --git a/app/src/api.ts b/app/src/api.ts index 18628e3f..6c64ca6b 100644 --- a/app/src/api.ts +++ b/app/src/api.ts @@ -9,16 +9,19 @@ import { UiSchema } from "react-jsonschema-form"; export const API_BASE_URL = '/api'; -export const useProjects = (query='', filter={ key: '', value: '' }) => { - const searchParams = new URLSearchParams(); +export const useProjects = (query='', filter={ key: '', value: '' }, page=0) => { + const params = new URLSearchParams(); + const pageSize = 10; + params.set('size', pageSize.toString()); + params.set('page', page.toString()); if (query) { - searchParams.set('q', query); + params.set('q', query); } if (filter.key && filter.value) { - searchParams.set('fk', filter.key); - searchParams.set('fv', filter.value); + params.set('fk', filter.key); + params.set('fv', filter.value); } - const url = API_BASE_URL + '/projects?' + searchParams.toString(); + const url = API_BASE_URL + '/projects?' + params.toString(); const response = useFetch<{ data: ProjectSummary[] }>(url); let data: ProjectSummary[] = []; if (response.data) { diff --git a/app/src/pages/Projects.tsx b/app/src/pages/Projects.tsx index df73bd52..f0195fad 100644 --- a/app/src/pages/Projects.tsx +++ b/app/src/pages/Projects.tsx @@ -6,6 +6,7 @@ import { useProjects } from "../api"; import { compareProjectSummary } from "../summarize"; import { ProjectList } from "../ProjectList"; import { ProjectSearch, FilterKey } from "../ProjectSearch"; +import { Pager } from "react-bootstrap"; const style = { padding: '10px' }; @@ -22,12 +23,13 @@ export function Projects() { value: params.get('fv')! } } + const [page, setPage] = useState(0); const { error, loading, data: projects, setData: setProjects - } = useProjects(q, filter); + } = useProjects(q, filter, page); const [sortkey, setSortKey] = useState('met_id'); const sortOn = (key: string) => { @@ -37,12 +39,31 @@ export function Projects() { setProjects({ data }); setSortKey(key); } + const prevPage = () => { + setPage(page - 1); + }; + const nextPage = () => { + setPage(page + 1); + }; let list = Loading ...; if (error) { list = Error: {error.message} } else if (!loading) { - list = + list = ( + <> + + + + ← Previous + + {' '} + + Next → + + + + ); } function clearFilter() { From 9ca49667e8097dc4811a45294e74d3524b48a247 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 16:58:02 +0200 Subject: [PATCH 09/16] More paging implementations * query and filter can be used at same time * more tests * openapi sort/order added --- api/src/app.test.ts | 7 +- api/src/controller.ts | 3 - api/src/store/search.test.ts | 283 ++++++++++++++++++++++------------- api/src/store/search.ts | 10 +- app/public/openapi.yaml | 16 ++ 5 files changed, 202 insertions(+), 117 deletions(-) diff --git a/api/src/app.test.ts b/api/src/app.test.ts index 6860e98e..d3012ac6 100644 --- a/api/src/app.test.ts +++ b/api/src/app.test.ts @@ -107,10 +107,14 @@ describe('app', () => { ['/api/projects?size=12', { size: 12, from: 0 }], ['/api/projects?page=2', { size: 100, from: 200 }], ['/api/projects?size=12&page=3', { size: 12, from: 36 }], + ['/api/projects?sort=nr_genomes', { size: 100, from: 0, sort: 'nr_genomes' }], + ['/api/projects?order=asc', { size: 100, from: 0, order: 'asc' }], + ['/api/projects?sort=nr_genomes&order=desc', { size: 100, from: 0, sort: 'nr_genomes', order: 'desc' }], ['/api/projects?q=Justin', { query: 'Justin', size: 100, from: 0 }], ['/api/projects?q=Justin&size=12&page=3', { query: 'Justin', size: 12, from: 36 }], ['/api/projects?fk=species&fv=somevalue', { filter: { key: 'species', value: 'somevalue' }, size: 100, from: 0 }], ['/api/projects?fk=species&fv=somevalue&size=12&page=3', { filter: { key: 'species', value: 'somevalue' }, size: 12, from: 36 }], + ['/api/projects?q=Justin&fk=species&fv=somevalue', { query: 'Justin', filter: { key: 'species', value: 'somevalue' }, size: 100, from: 0 }], ])('GET %s', (url, expected) => { it('should call store.searchProjects', async () => { await supertest(app).get(url); @@ -122,13 +126,14 @@ describe('app', () => { ['/api/projects?fk=wrongfield&fv=somevalue', 'Invalid `fk`'], ['/api/projects?fk=species', 'Require both `fk` and `fv` to filter'], ['/api/projects?fv=somevalue', 'Require both `fk` and `fv` to filter'], - ['/api/projects?q=Justin&fk=species&fv=somevalue', 'Either search with `q` or filter with `fk` and `fv`'], ['/api/projects?size=foo', 'Size is not an integer'], ['/api/projects?size=-10', 'Size must be between `1` and `1000`'], ['/api/projects?size=1110', 'Size must be between `1` and `1000`'], ['/api/projects?page=foo', 'Page is not an integer'], ['/api/projects?page=-10', 'Page must be between `0` and `1000`'], ['/api/projects?page=1111', 'Page must be between `0` and `1000`'], + ['/api/projects?sort=somebadfield', 'Invalid `sort`'], + ['/api/projects?order=somebadorder', 'Invalid `order`, must be either `desc` or `asc`'], ])('GET %s', (url, expected) => { let response: any; diff --git a/api/src/controller.ts b/api/src/controller.ts index 8c16eb99..b448575b 100644 --- a/api/src/controller.ts +++ b/api/src/controller.ts @@ -101,9 +101,6 @@ function validateSearchOptions(query: any) { } if (query.fk || query.fv) { if (query.fk && query.fv) { - if (query.q) { - throw 'Either search with `q` or filter with `fk` and `fv`'; - } if (!(query.fk in FilterFields)) { throw 'Invalid `fk`'; } diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 71a347d7..99bb000f 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -122,69 +122,200 @@ describe('new SearchEngine()', () => { }); }); - describe('search({size:1, from:2})', () => { - let hits: any; + describe('search()', () => { + describe('defaults', () => { + let hits: any; - beforeEach(async () => { - hits = await searchEngine.search({ size: 1, from: 2 }); - }); + beforeEach(async () => { + hits = await searchEngine.search({}); + }); - it('should have called client.search', () => { - expect(client.search).toHaveBeenCalledWith({ - index: 'podp', - size: 1, - from: 2, - _source: 'summary', - sort: 'summary.metabolite_id.keyword:desc', - body: { - 'query': { - match_all: {} + it('should have called client.search', () => { + expect(client.search).toHaveBeenCalledWith({ + index: 'podp', + size: 100, + from: 0, + _source: 'summary', + sort: 'summary.metabolite_id.keyword:desc', + body: { + 'query': { + match_all: {} + } } - } + }); + }); + + it('should return hits', async () => { + const expected_project = await genomeProjectSummary(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); }); }); - it('should return hits', async () => { - const expected_project = await genomeProjectSummary(); - const expected = { - data: [expected_project], - total: 1 - }; - expect(hits).toEqual(expected); + describe('paging', () => { + let hits: any; + + beforeEach(async () => { + hits = await searchEngine.search({ size: 1, from: 2 }); + }); + + it('should have called client.search', () => { + expect(client.search).toHaveBeenCalledWith({ + index: 'podp', + size: 1, + from: 2, + _source: 'summary', + sort: 'summary.metabolite_id.keyword:desc', + body: { + 'query': { + match_all: {} + } + } + }); + }); + + it('should return hits', async () => { + const expected_project = await genomeProjectSummary(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); + }); }); - }); - describe('search(\'Justin\')', () => { - const query = 'Justin'; - let hits: any; + describe('query=\'Justin\'', () => { + const query = 'Justin'; + let hits: any; - beforeEach(async () => { - hits = await searchEngine.search({ query }); + beforeEach(async () => { + hits = await searchEngine.search({ query }); + }); + + it('should have called client.search', () => { + expect(client.search).toHaveBeenCalledWith({ + index: 'podp', + from: 0, + size: 100, + _source: 'summary', + body: { + 'query': { + simple_query_string: { + query + } + } + } + }); + }); + + it('should return hits', async () => { + const expected_project = await genomeProjectSummary(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); + }); }); - it('should have called client.search', () => { - expect(client.search).toHaveBeenCalledWith({ - index: 'podp', - from: 0, - size: 100, - _source: 'summary', - body: { - 'query': { - simple_query_string: { - query + describe.each([ + ['principal_investigator', 'Marnix Medema'], + ['submitter', 'Justin van der Hooft'], + ['genome_type', 'genome'], + ['species', 'Streptomyces sp. CNB091'], + ['metagenomic_environment', 'Soil'], + ['instrument_type', 'Time-of-flight (TOF)'], + ['growth_medium', 'A1 medium'], + ['solvent', 'Butanol'], + ['ionization_mode', 'Positive'], + ['ionization_type', 'Electrospray Ionization (ESI)'] + ])('filter={key:\'%s\', value:\'%s\'}', (key: FilterField, value) => { + let hits: any; + beforeEach(async () => { + client.search.mockClear(); + hits = await searchEngine.search({ filter: { key, value } }); + }); + + it('should have called index.search', () => { + expect(client.search).toBeCalled(); + const called = client.search.mock.calls[0][0]; + const expected = { + index: 'podp', + from: 0, + size: 100, + _source: 'summary', + body: { + query: { + match: expect.anything() } } + }; + expect(called).toEqual(expected); + }); + + it('should return hits', async () => { + const expected_project = await genomeProjectSummary(); + const expected = { + data: [expected_project], + total: 1 + }; + expect(hits).toEqual(expected); + }); + }); + + describe('invalid filter field', () => { + it('should throw Error', async () => { + expect.assertions(1); + try { + await searchEngine.search({ + filter: { + key: 'some invalid key' as any, + value: 'somevalue' + } + }); + } catch (error) { + expect(error).toEqual(new Error('Invalid filter field')); } }); }); - it('should return hits', async () => { - const expected_project = await genomeProjectSummary(); - const expected = { - data: [expected_project], - total: 1 - }; - expect(hits).toEqual(expected); + describe('filter & query', () => { + it('should have called index.search', async () => { + const query = 'Justin'; + const filter = { key: 'solvent' as FilterField, value: 'Butanol' }; + + await searchEngine.search({ query, filter }); + + expect(client.search).toBeCalled(); + const called = client.search.mock.calls[0][0]; + const expected = { + index: 'podp', + from: 0, + size: 100, + _source: 'summary', + body: { + query: { + bool: { + must: { + simple_query_string: { + query: 'Justin' + } + }, + filter: { + match: { + 'project.experimental.extraction_methods.solvents.solvent_title.keyword': 'Butanol' + } + } + } + } + } + }; + expect(called).toEqual(expected); + }); + }); }); @@ -199,66 +330,6 @@ describe('new SearchEngine()', () => { }); }); - describe.each([ - ['principal_investigator', 'Marnix Medema'], - ['submitter', 'Justin van der Hooft'], - ['genome_type', 'genome'], - ['species', 'Streptomyces sp. CNB091'], - ['metagenomic_environment', 'Soil'], - ['instrument_type', 'Time-of-flight (TOF)'], - ['growth_medium', 'A1 medium'], - ['solvent', 'Butanol'], - ['ionization_mode', 'Positive'], - ['ionization_type', 'Electrospray Ionization (ESI)'] - ])('search({filter:{key:\'%s\', value:\'%s\'}})', (key: FilterField, value) => { - let hits: any; - beforeEach(async () => { - client.search.mockClear(); - hits = await searchEngine.search({ filter: { key, value } }); - }); - - it('should have called index.search', () => { - expect(client.search).toBeCalled(); - const called = client.search.mock.calls[0][0]; - const expected = { - index: 'podp', - from: 0, - size: 100, - _source: 'summary', - body: { - query: { - match: expect.anything() - } - } - }; - expect(called).toEqual(expected); - }); - - it('should return hits', async () => { - const expected_project = await genomeProjectSummary(); - const expected = { - data: [expected_project], - total: 1 - }; - expect(hits).toEqual(expected); - }); - }); - - describe('search({filter:{key:`invalid field`}})', () => { - it('should throw Error', async () => { - expect.assertions(1); - try { - await searchEngine.search({ - filter: { - key: 'some invalid key' as any, - value: 'somevalue' - } - }); - } catch (error) { - expect(error).toEqual(new Error('Invalid filter field')); - } - }); - }); }); describe('with a single metagenome project', () => { diff --git a/api/src/store/search.ts b/api/src/store/search.ts index da6a2c37..24a91a8f 100644 --- a/api/src/store/search.ts +++ b/api/src/store/search.ts @@ -154,12 +154,8 @@ function buildFilter(key: FilterField, value: string) { function buildQueryFilter(query: any, filter: any) { return { bool: { - must: { - query - }, - filter: { - filter - } + must: query, + filter } }; } @@ -256,7 +252,7 @@ export class SearchEngine { }); } - async search(options: SearchOptions = {}) { + async search(options: SearchOptions) { const defaultSort = (options.query || options.filter) ? 'score' : 'met_id'; const { size = DEFAULT_PAGE_SIZE, diff --git a/app/public/openapi.yaml b/app/public/openapi.yaml index e460d6a0..088d7e71 100644 --- a/app/public/openapi.yaml +++ b/app/public/openapi.yaml @@ -65,6 +65,22 @@ paths: default: 0 minimum: 0 maximum: 1000 + - name: sort + in: query + description: Field to sort on + example: nr_genomes + schema: + type: string + default: 'met_id' + enum: ["nr_genomes" , "score" , "_id" , "met_id" , "PI_name" , "submitters" , "nr_growth_conditions" , "nr_extraction_methods" , "nr_instrumentation_methods" , "nr_genome_metabolomics_links" , "nr_genecluster_mspectra_links"] + - name: order + in: query + description: Order to sort + example: asc + schema: + type: string + default: 'desc' + enum: ['asc', 'desc'] responses: "200": description: Successful Response From 7c37509b247b99ff8fc77143ddc35be2d944aaa6 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 18:09:43 +0200 Subject: [PATCH 10/16] Typo --- api/src/store/search.test.ts | 4 ++-- api/src/summarize.test.ts | 4 ++-- api/src/summarize.ts | 6 +++--- api/src/testhelpers.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 99bb000f..0987207d 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -427,7 +427,7 @@ async function esGenomeProject() { submitters: 'Justin van der Hooft', nr_extraction_methods: 3, nr_genecluster_mspectra_links: 3, - nr_genome_metabolmics_links: 21, + nr_genome_metabolomics_links: 21, nr_genomes: 3, nr_growth_conditions: 3, nr_instrumentation_methods: 1, @@ -465,7 +465,7 @@ async function genomeProjectSummary() { submitters: 'Justin van der Hooft', nr_extraction_methods: 3, nr_genecluster_mspectra_links: 3, - nr_genome_metabolmics_links: 21, + nr_genome_metabolomics_links: 21, nr_genomes: 3, nr_growth_conditions: 3, nr_instrumentation_methods: 1, diff --git a/api/src/summarize.test.ts b/api/src/summarize.test.ts index 28abf89f..67123d56 100644 --- a/api/src/summarize.test.ts +++ b/api/src/summarize.test.ts @@ -19,7 +19,7 @@ describe('summarizeProject()', () => { 'submitters': 'Justin van der Hooft', 'nr_extraction_methods': 3, 'nr_genecluster_mspectra_links': 3, - 'nr_genome_metabolmics_links': 21, + 'nr_genome_metabolomics_links': 21, 'nr_genomes': 3, 'nr_growth_conditions': 3, 'nr_instrumentation_methods': 1 @@ -45,7 +45,7 @@ describe('summarizeProject()', () => { 'submitters': 'Justin van der Hooft & Stefan Verhoeven', 'nr_extraction_methods': 3, 'nr_genecluster_mspectra_links': 3, - 'nr_genome_metabolmics_links': 21, + 'nr_genome_metabolomics_links': 21, 'nr_genomes': 3, 'nr_growth_conditions': 3, 'nr_instrumentation_methods': 1 diff --git a/api/src/summarize.ts b/api/src/summarize.ts index 650d05f4..4fcfe230 100644 --- a/api/src/summarize.ts +++ b/api/src/summarize.ts @@ -10,7 +10,7 @@ export interface ProjectSummary { nr_growth_conditions: number; nr_extraction_methods: number; nr_instrumentation_methods: number; - nr_genome_metabolmics_links: number; + nr_genome_metabolomics_links: number; nr_genecluster_mspectra_links: number; score?: number; } @@ -29,7 +29,7 @@ export const summarizeProject = (d: EnrichedProjectDocument): ProjectSummary => const nr_growth_conditions = project['experimental'] && project['experimental']['sample_preparation'] ? project['experimental']['sample_preparation'].length : 0; const nr_extraction_methods = project['experimental'] && project['experimental']['extraction_methods'] ? project['experimental']['extraction_methods'].length : 0; const nr_instrumentation_methods = project['experimental'] && project['experimental']['instrumentation_methods'] ? project['experimental']['instrumentation_methods'].length : 0; - const nr_genome_metabolmics_links = project['genome_metabolome_links'] ? project['genome_metabolome_links'].length : 0; + const nr_genome_metabolomics_links = project['genome_metabolome_links'] ? project['genome_metabolome_links'].length : 0; const nr_genecluster_mspectra_links = project['BGC_MS2_links'] ? project['BGC_MS2_links']!.length : 0; const summary = { _id: d._id, @@ -40,7 +40,7 @@ export const summarizeProject = (d: EnrichedProjectDocument): ProjectSummary => nr_growth_conditions, nr_extraction_methods, nr_instrumentation_methods, - nr_genome_metabolmics_links, + nr_genome_metabolomics_links, nr_genecluster_mspectra_links, }; if (isMetaboLights(project.metabolomics.project)) { diff --git a/api/src/testhelpers.ts b/api/src/testhelpers.ts index 95f242a1..ac67b774 100644 --- a/api/src/testhelpers.ts +++ b/api/src/testhelpers.ts @@ -19,7 +19,7 @@ export async function mockedElasticSearchClient() { submitters: 'Justin van der Hooft', nr_extraction_methods: 3, nr_genecluster_mspectra_links: 3, - nr_genome_metabolmics_links: 21, + nr_genome_metabolomics_links: 21, nr_genomes: 3, nr_growth_conditions: 3, nr_instrumentation_methods: 1, From bf8c9080e4b6f05a2248cbfc20e6cfc49c2c4e95 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 18:14:38 +0200 Subject: [PATCH 11/16] Push paging/sorting from app to api + added pager + revert sort by clicking header again --- app/src/ProjectList.tsx | 56 +++++++++++++++++++------------ app/src/api.ts | 18 +++++++--- app/src/pages/PendingProjects.tsx | 12 +++---- app/src/pages/Projects.tsx | 55 +++++++++++++++++------------- app/src/summarize.test.ts | 42 ----------------------- app/src/summarize.ts | 38 ++------------------- 6 files changed, 87 insertions(+), 134 deletions(-) delete mode 100644 app/src/summarize.test.ts diff --git a/app/src/ProjectList.tsx b/app/src/ProjectList.tsx index bfb0debe..5f20c859 100644 --- a/app/src/ProjectList.tsx +++ b/app/src/ProjectList.tsx @@ -8,35 +8,47 @@ interface ColumnHeaderProps { skey: string; onClick(key: string): void; title: string; + sortOrder: 'desc' | 'asc' } -const ColumnHeader = ({ active, skey, onClick, title }: ColumnHeaderProps) => ( - active !== skey && onClick(skey)} - style={active !== skey ? { cursor: 'pointer' } : {}} - title="Click to sort on"> - {title} - {active === skey && <> } - -) +const ColumnHeader = ({ active, skey, onClick, title, sortOrder }: ColumnHeaderProps) => { + let glyph = <>; + if (active === skey) { + if (sortOrder === 'asc') { + glyph = ; + } else { + glyph = ; + } + } + return ( + onClick(skey)} + style={{ cursor: 'pointer' }} + title="Click to sort on"> + {title} + <> {glyph} + + ); +} interface Props { projects: ProjectSummary[]; setSortedOn(key: string): void; - sortedOn: string; + sortKey: string; + sortOrder: 'desc' | 'asc'; } -export const ProjectList = ({ projects, setSortedOn, sortedOn }: Props) => { +export const ProjectList = ({ projects, setSortedOn, sortKey, sortOrder }: Props) => { const rows = projects.map(d => ( - {d.GNPSMassIVE_ID ? d.GNPSMassIVE_ID : d.metabolights_study_id} + {d.metabolite_id} {d.PI_name} {d.submitters} {d.nr_genomes} {d.nr_growth_conditions} {d.nr_extraction_methods} {d.nr_instrumentation_methods} - {d.nr_genome_metabolmics_links} + {d.nr_genome_metabolomics_links} {d.nr_genecluster_mspectra_links} )); @@ -47,15 +59,15 @@ export const ProjectList = ({ projects, setSortedOn, sortedOn }: Props) => { - - - - - - - - - + + + + + + + + + diff --git a/app/src/api.ts b/app/src/api.ts index 6c64ca6b..d13fbb1a 100644 --- a/app/src/api.ts +++ b/app/src/api.ts @@ -9,10 +9,11 @@ import { UiSchema } from "react-jsonschema-form"; export const API_BASE_URL = '/api'; -export const useProjects = (query='', filter={ key: '', value: '' }, page=0) => { +export const PAGE_SIZE = 100; + +export const useProjects = (query = '', filter = { key: '', value: '' }, page = 0, sort = '', order = '') => { const params = new URLSearchParams(); - const pageSize = 10; - params.set('size', pageSize.toString()); + params.set('size', PAGE_SIZE.toString()); params.set('page', page.toString()); if (query) { params.set('q', query); @@ -21,15 +22,24 @@ export const useProjects = (query='', filter={ key: '', value: '' }, page=0) => params.set('fk', filter.key); params.set('fv', filter.value); } + if (sort) { + params.set('sort', sort); + } + if (order) { + params.set('order', order); + } const url = API_BASE_URL + '/projects?' + params.toString(); - const response = useFetch<{ data: ProjectSummary[] }>(url); + const response = useFetch<{ data: ProjectSummary[], total: number }>(url); let data: ProjectSummary[] = []; + let total = 0; if (response.data) { data = response.data.data; + total = response.data.total; } return { ...response, data, + total }; }; diff --git a/app/src/pages/PendingProjects.tsx b/app/src/pages/PendingProjects.tsx index 25aab307..4080098f 100644 --- a/app/src/pages/PendingProjects.tsx +++ b/app/src/pages/PendingProjects.tsx @@ -30,32 +30,32 @@ export function PendingProjects() { const onDeny = (project_id: string) => async () => { await denyPendingProject(project_id, token); const pruned_projects = dropProject(project_id, projects.data!.data) - projects.setData({data: pruned_projects}); + projects.setData({ data: pruned_projects }); }; const onApprove = (project_id: string) => async () => { await approvePendingProject(project_id, token); const pruned_projects = dropProject(project_id, projects.data!.data) - projects.setData({data: pruned_projects}); + projects.setData({ data: pruned_projects }); }; const rows = projects.data!.data.map(d => ( - + - + - + )); if (projects.data!.data.length === 0) { - rows.push(); + rows.push(); } return (
diff --git a/app/src/pages/Projects.tsx b/app/src/pages/Projects.tsx index f0195fad..74206428 100644 --- a/app/src/pages/Projects.tsx +++ b/app/src/pages/Projects.tsx @@ -1,12 +1,10 @@ import * as React from "react"; -import { useState } from "react"; import { useLocation, useHistory } from "react-router-dom"; import { useProjects } from "../api"; -import { compareProjectSummary } from "../summarize"; import { ProjectList } from "../ProjectList"; import { ProjectSearch, FilterKey } from "../ProjectSearch"; -import { Pager } from "react-bootstrap"; +import { ProjectPager } from "../ProjectPager"; const style = { padding: '10px' }; @@ -23,27 +21,38 @@ export function Projects() { value: params.get('fv')! } } - const [page, setPage] = useState(0); + const defaultSortKey = q ? 'score' : 'met_id'; + const sortKey = params.has('sort') ? params.get('sort')! : defaultSortKey; + const order = params.has('order') ? params.get('order')! : 'desc'; + const page = params.has('page') ? parseInt(params.get('page')!) : 0; const { error, loading, data: projects, - setData: setProjects - } = useProjects(q, filter, page); + total, + } = useProjects(q, filter, page, sortKey, order); - const [sortkey, setSortKey] = useState('met_id'); const sortOn = (key: string) => { - // TODO reverse sort when sorted column is clicked again - const data = [...projects]; - data.sort(compareProjectSummary(key)); - setProjects({ data }); - setSortKey(key); + if (key === sortKey) { + params.set('order', order === 'asc' ? 'desc' : 'asc'); + } + params.set('sort', key); + history.push("/projects?" + params.toString()); } + const prevPage = () => { - setPage(page - 1); + const newpage = page - 1; + if (newpage === 0) { + params.delete('page'); + } else { + params.set('page', newpage.toString()); + } + history.push("/projects?" + params.toString()); }; const nextPage = () => { - setPage(page + 1); + const newpage = page + 1; + params.set('page', newpage.toString()); + history.push("/projects?" + params.toString()); }; let list = Loading ...; @@ -52,16 +61,14 @@ export function Projects() { } else if (!loading) { list = ( <> - - - - ← Previous - - {' '} - - Next → - - + + ); } diff --git a/app/src/summarize.test.ts b/app/src/summarize.test.ts deleted file mode 100644 index ee309751..00000000 --- a/app/src/summarize.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { compareProjectSummary, ProjectSummary } from "./summarize"; - -describe('compareProjectSummary()', () => { - - describe('a === a', () => { - test.each([ - ['_id'], - ['met_id'], - ['PI_name'], - ['submitters'], - ['nr_genomes'], - ['nr_growth_conditions'], - ['nr_extraction_methods'], - ['nr_instrumentation_methods'], - ['nr_genome_metabolmics_links'], - ['nr_genecluster_mspectra_links'], - ])('%s', (key: string) => { - const comparer = compareProjectSummary(key); - const a: ProjectSummary ={ - _id: 'id1', - GNPSMassIVE_ID: 'somegnpsid', - metabolights_study_id: '', - PI_name: 'somepi', - submitters: 'somesubmitter', - nr_genomes: 3, - nr_growth_conditions: 4, - nr_extraction_methods: 5, - nr_instrumentation_methods: 2, - nr_genome_metabolmics_links: 123, - nr_genecluster_mspectra_links: 42, - }; - const result = comparer(a, a); - expect(result).toEqual(0); - }); - }); - - describe('when key is given which is not part of ProjectSummary', () => { - it('should throw an Error', () => { - expect(() => compareProjectSummary('wrongkey')).toThrowError(/wrongkey/); - }); - }); -}); diff --git a/app/src/summarize.ts b/app/src/summarize.ts index 3954d4a8..11e7f873 100644 --- a/app/src/summarize.ts +++ b/app/src/summarize.ts @@ -2,15 +2,14 @@ import { IOMEGAPairedOmicsDataPlatform as ProjectDocument } from './schema'; export interface ProjectSummary { _id: string; - GNPSMassIVE_ID: string; - metabolights_study_id: string; + metabolite_id: string; PI_name: string; submitters: string; nr_genomes: number; nr_growth_conditions: number; nr_extraction_methods: number; nr_instrumentation_methods: number; - nr_genome_metabolmics_links: number; + nr_genome_metabolomics_links: number; nr_genecluster_mspectra_links: number; } @@ -40,36 +39,3 @@ export interface EnrichedProjectDocument { project: ProjectDocument; enrichments?: ProjectEnrichments; } - -export const compareProjectSummary = function(key: string): (a: ProjectSummary, b: ProjectSummary) => number { - if (key === '_id') { - return (a, b) => { - if (a._id > b._id) { - return -1; - } - if (a._id < b._id) { - return 1; - } - return 0; - } - } else if (key === 'met_id') { - return (a, b) => { - const ia = a.GNPSMassIVE_ID ? a.GNPSMassIVE_ID : a.metabolights_study_id; - const ib = b.GNPSMassIVE_ID ? b.GNPSMassIVE_ID : b.metabolights_study_id; - if (ia < ib) { - return -1; - } - if (ia > ib) { - return 1; - } - return 0; - }; - } else if (key === 'PI_name' || key === 'submitters') { - return (a, b) => a[key].localeCompare(b[key]); - } else if (key === 'nr_genomes' || key === 'nr_growth_conditions' || key === 'nr_extraction_methods' || key === 'nr_instrumentation_methods' || key === 'nr_genome_metabolmics_links' || key === 'nr_genecluster_mspectra_links') { - return (a, b) => { - return b[key] - a[key]; - }; - } - throw new TypeError(`${key} is not a property of ProjectSummary`); -}; \ No newline at end of file From b4360c0dc468b187590111d6486b15ef16f5318e Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 22:21:46 +0200 Subject: [PATCH 12/16] Added app/src/ProjectPager.tsx --- app/src/ProjectPager.tsx | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 app/src/ProjectPager.tsx diff --git a/app/src/ProjectPager.tsx b/app/src/ProjectPager.tsx new file mode 100644 index 00000000..1cff23ee --- /dev/null +++ b/app/src/ProjectPager.tsx @@ -0,0 +1,26 @@ +import * as React from "react"; + +import { Pager } from "react-bootstrap"; +import { PAGE_SIZE } from "./api"; + +interface Props { + prevPage(): void; + nextPage(): void; + page: number; + page_count: number; + total: number; +} + +export const ProjectPager = ({ page, prevPage, nextPage, page_count, total }: Props) => { + return ( + + + ← Previous + +
  • {page * PAGE_SIZE + 1} - {(page * PAGE_SIZE) + page_count} of {total}
  • + = total} onClick={nextPage}> + Next → + +
    + ); +} \ No newline at end of file From 192898f1eec6818e1b87699d9ba8ea686c1eb743 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Wed, 22 Apr 2020 22:36:19 +0200 Subject: [PATCH 13/16] Fixed tests --- api/src/store/search.test.ts | 1 + app/src/pages/PendingProjects.test.tsx | 2 +- app/src/pages/Projects.test.tsx | 17 ++++++++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/api/src/store/search.test.ts b/api/src/store/search.test.ts index 0987207d..fb1cf3ab 100644 --- a/api/src/store/search.test.ts +++ b/api/src/store/search.test.ts @@ -284,6 +284,7 @@ describe('new SearchEngine()', () => { describe('filter & query', () => { it('should have called index.search', async () => { + client.search.mockClear(); const query = 'Justin'; const filter = { key: 'solvent' as FilterField, value: 'Butanol' }; diff --git a/app/src/pages/PendingProjects.test.tsx b/app/src/pages/PendingProjects.test.tsx index d9dd5567..94edf64d 100644 --- a/app/src/pages/PendingProjects.test.tsx +++ b/app/src/pages/PendingProjects.test.tsx @@ -96,7 +96,7 @@ describe('', () => { data: { data: [{ _id: 'id1', - GNPSMassIVE_ID: 'somegnpsid', + metabolite_id: 'somegnpsid', PI_name: 'somepi', submitter: 'somesubmitter', nr_genomes: 3, diff --git a/app/src/pages/Projects.test.tsx b/app/src/pages/Projects.test.tsx index 846cf09f..f81a31bf 100644 --- a/app/src/pages/Projects.test.tsx +++ b/app/src/pages/Projects.test.tsx @@ -138,14 +138,19 @@ describe('', () => { describe('click on PI column header', () => { beforeEach(() => { + (useProjects as jest.Mock).mockClear(); const button = wrapper.getByText('Principal investigator'); fireEvent.click(button); }); it('should have sorted rows on PI', () => { const cells = wrapper.getAllByRole('cell'); - expect(cells[1].textContent).toEqual('otherpi'); - expect(cells[10].textContent).toEqual('somepi'); + expect(cells[10].textContent).toEqual('otherpi'); + expect(cells[1].textContent).toEqual('somepi'); + }); + + it('should call api with sort on PI', () => { + expect(useProjects).toHaveBeenCalledWith(undefined, undefined, 0, 'PI_name', 'desc'); }); }); @@ -167,12 +172,13 @@ describe('', () => { describe('when search is submitted', () => { beforeEach(() => { + (useProjects as jest.Mock).mockClear(); const search = wrapper.getByTitle('Search'); fireEvent.click(search); }); it('should pass search query to api', () => { - expect(useProjects).toHaveBeenCalledWith('foobar', undefined); + expect(useProjects).toHaveBeenCalledWith('foobar', undefined, 0, "score", "desc"); }); it('should include search query in url', () => { @@ -187,7 +193,7 @@ describe('', () => { }); it('should clear search query to api', () => { - expect(useProjects).toHaveBeenCalledWith(undefined, undefined); + expect(useProjects).toHaveBeenCalledWith(undefined, undefined, 0, "met_id", "desc"); }); it('should no longer include search query in url',() => { @@ -203,6 +209,7 @@ describe('', () => { describe('when filter is in url', () => { beforeEach(() => { + (useProjects as jest.Mock).mockClear(); const route = '/projects?fk=submitter&fv=submitter3'; history.push(route); }); @@ -221,7 +228,7 @@ describe('', () => { expect(useProjects).toHaveBeenCalledWith(undefined, { key: 'submitter', value: 'submitter3' - }); + }, 0, "met_id", "desc"); }); describe('when clear filter button is clicked', () => { From 261578efc92a30198b8ee70cb3e0d33d909cb74c Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 23 Apr 2020 08:37:25 +0200 Subject: [PATCH 14/16] Added tests for pager --- app/src/ProjectPager.test.tsx | 167 ++++++++++++++++++++++++++++++++++ app/src/ProjectPager.tsx | 30 ++++-- 2 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 app/src/ProjectPager.test.tsx diff --git a/app/src/ProjectPager.test.tsx b/app/src/ProjectPager.test.tsx new file mode 100644 index 00000000..e493451a --- /dev/null +++ b/app/src/ProjectPager.test.tsx @@ -0,0 +1,167 @@ +import * as React from 'react'; + +import { render, RenderResult, getRoles, fireEvent } from '@testing-library/react'; +import '@testing-library/jest-dom/extend-expect'; +import { ProjectPager } from './ProjectPager'; + +describe('', () => { + describe('when all hits fit on current page', () => { + it('should render nothing', () => { + const nextPage = jest.fn(); + const prevPage = jest.fn(); + + const comp = render(); + + expect(comp.container.innerHTML).toEqual(''); + }); + }); + + describe('when first page of many', () => { + let comp: RenderResult; + let prevPage: jest.Mock; + let nextPage: jest.Mock; + + beforeEach(() => { + nextPage = jest.fn(); + prevPage = jest.fn(); + + comp = render(); + }); + + it('should display page counts', () => { + expect(comp.baseElement).toHaveTextContent(/1 - 100 of 450/); + }); + + it('should have prev page button disabled', () => { + const but = comp.getByTitle('Previous').parentElement; + expect(but!.className).toContain('disabled'); + }); + + it('should have prev page button not disabled', () => { + const but = comp.getByTitle('Next').parentElement; + expect(but!.className).not.toContain('disabled'); + }); + + describe('when next button clicked', () => { + beforeEach(() => { + const but = comp.getByTitle('Next') + fireEvent.click(but); + }); + + it('should call nextPage()', () => { + expect(nextPage).toHaveBeenCalled(); + }); + }); + }); + + describe('when middle page of many', () => { + let comp: RenderResult; + let prevPage: jest.Mock; + let nextPage: jest.Mock; + + beforeEach(() => { + nextPage = jest.fn(); + prevPage = jest.fn(); + + comp = render(); + }); + + it('should display page counts', () => { + expect(comp.baseElement).toHaveTextContent(/101 - 200 of 450/); + }); + + it('should have prev page button not disabled', () => { + const but = comp.getByTitle('Previous').parentElement; + expect(but!.className).not.toContain('disabled'); + }); + + it('should have prev page button not disabled', () => { + const but = comp.getByTitle('Next').parentElement; + expect(but!.className).not.toContain('disabled'); + }); + + describe('when next button clicked', () => { + beforeEach(() => { + const but = comp.getByTitle('Next') + fireEvent.click(but); + }); + + it('should call nextPage()', () => { + expect(nextPage).toHaveBeenCalled(); + }); + }); + + describe('when prev button clicked', () => { + beforeEach(() => { + const but = comp.getByTitle('Previous') + fireEvent.click(but); + }); + + it('should call prevPage()', () => { + expect(prevPage).toHaveBeenCalled(); + }); + }); + }); + + describe('when last page of many', () => { + let comp: RenderResult; + let prevPage: jest.Mock; + let nextPage: jest.Mock; + + beforeEach(() => { + nextPage = jest.fn(); + prevPage = jest.fn(); + + comp = render(); + }); + + it('should display page counts', () => { + expect(comp.baseElement).toHaveTextContent(/401 - 450 of 450/); + }); + + it('should have prev page button not disabled', () => { + const but = comp.getByTitle('Previous').parentElement; + expect(but!.className).not.toContain('disabled'); + }); + + it('should have prev page button not disabled', () => { + const but = comp.getByTitle('Next').parentElement; + expect(but!.className).toContain('disabled'); + }); + + describe('when prev button clicked', () => { + beforeEach(() => { + const but = comp.getByTitle('Previous') + fireEvent.click(but); + }); + + it('should call prevPage()', () => { + expect(prevPage).toHaveBeenCalled(); + }); + }); + }); +}); \ No newline at end of file diff --git a/app/src/ProjectPager.tsx b/app/src/ProjectPager.tsx index 1cff23ee..16bfbe8f 100644 --- a/app/src/ProjectPager.tsx +++ b/app/src/ProjectPager.tsx @@ -12,15 +12,27 @@ interface Props { } export const ProjectPager = ({ page, prevPage, nextPage, page_count, total }: Props) => { - return ( - - - ← Previous + if (page_count === total) { + return <>; + } else { + return ( + + + ← Previous -
  • {page * PAGE_SIZE + 1} - {(page * PAGE_SIZE) + page_count} of {total}
  • - = total} onClick={nextPage}> - Next → +
  • {page * PAGE_SIZE + 1} - {(page * PAGE_SIZE) + page_count} of {total}
  • + = total} + onClick={nextPage} + title="Next" + > + Next → -
    - ); +
    + ); + } } \ No newline at end of file From ab0ed320461c9109aa7873a51673e5c19730f6f5 Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 23 Apr 2020 08:37:54 +0200 Subject: [PATCH 15/16] Updated CHANGELOG --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e7c01c..6bcfaaec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +* Paging projects ([#137](https://github.com/iomega/paired-data-form/issues/137)) + +### Changed + +* Sort projects moved from web application to elastic search ([#138](https://github.com/iomega/paired-data-form/issues/138)) +* Allow search and filter to be combined + ## [0.6.1] 2020-04-16 ### Added * Added ionization modes to stats page ([#132](https://github.com/iomega/paired-data-form/issues/132)) * Search query examples ([#132](https://github.com/iomega/paired-data-form/issues/132)) -* Paging projects ([#137](https://github.com/iomega/paired-data-form/issues/137)) ### Fixed From 82eb5fd555c31835f7e634cfef06d58be3420b2e Mon Sep 17 00:00:00 2001 From: Stefan Verhoeven Date: Thu, 23 Apr 2020 08:46:34 +0200 Subject: [PATCH 16/16] Prep for 0.6.2 --- CHANGELOG.md | 6 ++++-- api/package.json | 2 +- app/package.json | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bcfaaec..e1bf79a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.2] - 2020-04-23 + ### Added * Paging projects ([#137](https://github.com/iomega/paired-data-form/issues/137)) @@ -16,7 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Sort projects moved from web application to elastic search ([#138](https://github.com/iomega/paired-data-form/issues/138)) * Allow search and filter to be combined -## [0.6.1] 2020-04-16 +## [0.6.1] - 2020-04-16 ### Added @@ -27,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Enrichments cause Limit of total fields exceeded error in elastic search ([#131](https://github.com/iomega/paired-data-form/issues/131)) -## [0.6.0] 2020-04-16 +## [0.6.0] - 2020-04-16 Search functionality using elastic search has been added. diff --git a/api/package.json b/api/package.json index dca4a8e0..20b2e3f3 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "podp-api", - "version": "0.6.1", + "version": "0.6.2", "description": "", "license": "Apache-2.0", "scripts": { diff --git a/app/package.json b/app/package.json index 6ad85463..492746f3 100644 --- a/app/package.json +++ b/app/package.json @@ -1,6 +1,6 @@ { "name": "paired-data-platform", - "version": "0.6.1", + "version": "0.6.2", "private": true, "dependencies": { "@root/encoding": "^1.0.1",
    { d.GNPSMassIVE_ID ? d.GNPSMassIVE_ID : d.metabolights_study_id }{d.metabolite_id} {d.PI_name} {d.submitters} {d.nr_genomes} {d.nr_growth_conditions} {d.nr_extraction_methods} {d.nr_instrumentation_methods}{d.nr_genome_metabolmics_links}{d.nr_genome_metabolomics_links} {d.nr_genecluster_mspectra_links}
    No pending projects found.
    No pending projects found.