Skip to content

Commit

Permalink
refactor: add index on batch search nb_queries + on the fly update nb…
Browse files Browse the repository at this point in the history 
…Queries as cached value in db

Co-authored-by: Bruno Thomas <[email protected]>
  • Loading branch information
@caro3801 @bamthomas
caro3801 and bamthomas committed Oct 18, 2023
1 parent 6998006 commit a63453b
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 52 deletions.
94 changes: 47 additions & 47 deletions datashare-db/src/main/java/org/icij/datashare/db/JooqBatchSearchRepository.java
View file
Original file line number Diff line number Diff line change
@@ -1,68 +1,37 @@
package org.icij.datashare.db;

import org.icij.datashare.batch.BatchSearch;
import org.icij.datashare.batch.BatchSearchRecord;
import org.icij.datashare.batch.*;
import org.icij.datashare.batch.BatchSearchRecord.State;
import org.icij.datashare.batch.BatchSearchRepository;
import org.icij.datashare.batch.SearchException;
import org.icij.datashare.batch.SearchResult;
import org.icij.datashare.batch.WebQueryBuilder;
import org.icij.datashare.db.tables.records.BatchSearchProjectRecord;
import org.icij.datashare.db.tables.records.BatchSearchQueryRecord;
import org.icij.datashare.db.tables.records.BatchSearchResultRecord;
import org.icij.datashare.text.Document;
import org.icij.datashare.text.Project;
import org.icij.datashare.user.User;
import org.jooq.DSLContext;
import org.jooq.InsertValuesStep10;
import org.jooq.InsertValuesStep2;
import org.jooq.InsertValuesStep4;
import org.jooq.Record;
import org.jooq.Record1;
import org.jooq.Record12;
import org.jooq.Record17;
import org.jooq.Record19;
import org.jooq.SQLDialect;
import org.jooq.SelectConditionStep;
import org.jooq.SelectJoinStep;
import org.jooq.SelectOnConditionStep;
import org.jooq.SelectSelectStep;
import org.jooq.*;
import org.jooq.impl.DSL;

import javax.sql.DataSource;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.*;
import java.util.stream.IntStream;

import static java.lang.String.join;
import static java.util.Arrays.asList;
import static java.util.Arrays.stream;
import static java.util.Collections.singletonList;
import static java.util.Comparator.comparing;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import static java.util.stream.Collectors.*;
import static org.icij.datashare.batch.BatchSearchRepository.WebQuery.DEFAULT_SORT_FIELD;
import static org.icij.datashare.db.Tables.BATCH_SEARCH_PROJECT;
import static org.icij.datashare.db.tables.BatchSearch.BATCH_SEARCH;
import static org.icij.datashare.db.tables.BatchSearchQuery.BATCH_SEARCH_QUERY;
import static org.icij.datashare.db.tables.BatchSearchResult.BATCH_SEARCH_RESULT;
import static org.icij.datashare.text.Project.project;
import static org.jooq.impl.DSL.count;
import static org.jooq.impl.DSL.countDistinct;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.groupConcat;
import static org.jooq.impl.DSL.select;
import static org.jooq.impl.DSL.*;

public class JooqBatchSearchRepository implements BatchSearchRepository {
private static final String LIST_SEPARATOR = ",";
Expand Down Expand Up @@ -216,8 +185,10 @@ public List<BatchSearchRecord> getRecords(User user, List<String> projectsIds) {

@Override
public List<BatchSearchRecord> getRecords(User user, List<String> projectsIds, WebQuery webQuery) {
try(DSLContext context = DSL.using(dataSource, dialect)){
SelectConditionStep<Record12<String, String, String, String, Timestamp, String, Integer, Integer, String, String, String, Object>> query = createBatchSearchRecordWithQueriesSelectStatement(context)
try(DSLContext context = DSL.using(dataSource, dialect)) {
cacheNbQueries(webQuery, context);

SelectConditionStep<Record12<String, String, String, String, Timestamp, String, Integer, Integer, String, String, String, Integer>> query = createBatchSearchRecordWithQueriesSelectStatement(context)
.where(BATCH_SEARCH.USER_ID.eq(user.id).or(BATCH_SEARCH.PUBLISHED.greaterThan(0)));
List<String> filteredProjects = webQuery.hasFilteredProjects() ? webQuery.project : projectsIds;
addFilterToSelectCondition(webQuery, query);
Expand All @@ -234,7 +205,6 @@ public List<BatchSearchRecord> getRecords(User user, List<String> projectsIds, W
)).orderBy(BATCH_SEARCH.BATCH_DATE.desc()).
fetch().stream().map(this::createBatchSearchRecordFrom).collect(toList());
}

}

@Override
Expand Down Expand Up @@ -436,16 +406,13 @@ private static void addFiltersToBatchSearchResultQuery(WebQuery webQuery, Select
BATCH_SEARCH.ERROR_MESSAGE,
BATCH_SEARCH.ERROR_QUERY,
groupConcat(BATCH_SEARCH_PROJECT.PRJ_ID).as(field("projects")),
field(create.selectCount()
.from(BATCH_SEARCH_QUERY)
.where(BATCH_SEARCH_QUERY.SEARCH_UUID.eq(BATCH_SEARCH.UUID)))
.as(field("nb_queries")))
BATCH_SEARCH.NB_QUERIES)
.from(BATCH_SEARCH
.join(BATCH_SEARCH_PROJECT)
.on(BATCH_SEARCH.UUID.eq(BATCH_SEARCH_PROJECT.SEARCH_UUID)));
}

private SelectOnConditionStep<Record12<String, String, String, String, Timestamp, String, Integer, Integer, String, String, String, Object>>
private SelectOnConditionStep<Record12<String, String, String, String, Timestamp, String, Integer, Integer, String, String, String, Integer>>
createBatchSearchRecordWithQueriesSelectStatement(DSLContext create) {
return create.select(
BATCH_SEARCH.UUID,
Expand All @@ -459,7 +426,7 @@ private static void addFiltersToBatchSearchResultQuery(WebQuery webQuery, Select
BATCH_SEARCH.ERROR_MESSAGE,
BATCH_SEARCH.ERROR_QUERY,
groupConcat(BATCH_SEARCH_PROJECT.PRJ_ID).as(field("projects")),
create.selectCount().from(BATCH_SEARCH_QUERY).where(BATCH_SEARCH_QUERY.SEARCH_UUID.eq(BATCH_SEARCH.UUID)).asField("nbQueries")).
BATCH_SEARCH.NB_QUERIES).
from(BATCH_SEARCH).join(BATCH_SEARCH_PROJECT).on(BATCH_SEARCH.UUID.eq(BATCH_SEARCH_PROJECT.SEARCH_UUID));
}

Expand Down Expand Up @@ -516,7 +483,7 @@ private static List<String> getListFromStringOrNull(String stringList) {
}

private BatchSearchRecord createBatchSearchRecordFrom(final Record record) {
Object nbQueries = record.getValue("nbQueries");
Object nbQueries = record.getValue(BATCH_SEARCH.NB_QUERIES);
String projects = (String) record.get("projects");
org.icij.datashare.db.tables.records.BatchSearchRecord batchSearch = record.into(BATCH_SEARCH);
return new BatchSearchRecord(batchSearch.getUuid(),
Expand Down Expand Up @@ -580,18 +547,51 @@ private static void addFilterToSelectCondition(WebQuery webQuery, SelectConditio
}
}

private SelectConditionStep<Record1<String>> selectBatchSearchWithZeroNbQueries(DSLContext create) {
return create.select(BATCH_SEARCH.UUID).from(BATCH_SEARCH).where(BATCH_SEARCH.NB_QUERIES.eq(0));
}

private void updateBatchSearchNbQueries(DSLContext context, List<String> uuids) {
if (uuids.isEmpty()){
return;
}

context.select(BATCH_SEARCH.UUID, count())
.from(BATCH_SEARCH_QUERY)
.where(BATCH_SEARCH_QUERY.SEARCH_UUID.eq(BATCH_SEARCH.UUID).and(BATCH_SEARCH.UUID.in(uuids)))
.fetch().forEach(record2-> setNbQueries(context,record2.value1(),record2.value2()) );
}

/**
* This method allows retrocompatibility from the times when
* the nbQueries were not cached into the database. Each record
* without the cache is updated on the fly.
*/
private void cacheNbQueries(WebQuery webQuery, DSLContext context) {
SelectConditionStep<Record1<String>> zeroNbQueries = selectBatchSearchWithZeroNbQueries(context);
addFilterToSelectCondition(webQuery, zeroNbQueries);
List<String> uuids = zeroNbQueries.fetch(BATCH_SEARCH.UUID, String.class);
updateBatchSearchNbQueries(context, uuids);
}

int getNbQueries(String batchSearchUUID){
try (DSLContext context = DSL.using(dataSource, dialect)) {
Record1<Integer> record = context.select(BATCH_SEARCH.NB_QUERIES).from(BATCH_SEARCH)
.where(BATCH_SEARCH.UUID.eq(batchSearchUUID)).fetchOne();
return (int) record.getValue(0);
}
}

boolean setNbQueries(DSLContext context, String batchSearchUuid, int nbQueries){
return context.update(BATCH_SEARCH).set(BATCH_SEARCH.NB_QUERIES,nbQueries).where(BATCH_SEARCH.UUID.eq(batchSearchUuid)).execute() > 0;
}

boolean setNbQueries(String batchSearchUuid, int nbQueries){
try (DSLContext context = DSL.using(dataSource, dialect)) {
return context.update(BATCH_SEARCH).set(BATCH_SEARCH.NB_QUERIES,nbQueries).where(BATCH_SEARCH.UUID.eq(batchSearchUuid)).execute() > 0;
return setNbQueries(context, batchSearchUuid, nbQueries);
}
}

@Override
public void close() throws IOException {
if (dataSource instanceof Closeable) {
Expand Down
9 changes: 8 additions & 1 deletion ...rc/main/resources/liquibase/changelog/changes/033-adds-column-batch-search-nb-queries.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,11 @@ databaseChangeLog:
column:
name: nb_queries
type: int
defaultValue: 0
defaultValue: 0
- createIndex:
indexName: batch_search_nb_queries
tableName: batch_search
columns:
- column:
name: nb_queries
type: int
4 changes: 0 additions & 4 deletions datashare-db/src/test/java/org/icij/datashare/db/JooqBatchSearchRepositoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@
import org.icij.datashare.text.Project;
import org.icij.datashare.time.DatashareTime;
import org.icij.datashare.user.User;
import org.jooq.DSLContext;
import org.jooq.Record1;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.DSL;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
Expand All @@ -26,7 +23,6 @@
import static org.fest.assertions.Assertions.assertThat;
import static org.fest.assertions.MapAssert.entry;
import static org.icij.datashare.CollectionUtils.asSet;
import static org.icij.datashare.db.Tables.BATCH_SEARCH;
import static org.icij.datashare.text.DocumentBuilder.createDoc;
import static org.icij.datashare.text.Project.project;

Expand Down

0 comments on commit a63453b

Please sign in to comment.