Skip to content

Commit

Permalink
avniproject/avni-webapp#1317 - removed subject type join to simplify.…
Browse files Browse the repository at this point in the history
… simplified search query has lesser cost.

(cherry picked from commit 394959f)
  • Loading branch information
petmongrels authored and himeshr committed Oct 18, 2024
1 parent 8e3a36c commit 54f1570
Show file tree
Hide file tree
Showing 10 changed files with 121 additions and 56 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ ci-test:

open_test_results:
open avni-server-api/build/reports/tests/test/index.html
open-test-results: open_test_results

build-rpm:
./gradlew clean avni-server-api:buildRpm -x test --info --stacktrace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@

import org.avni.server.dao.search.SearchBuilder;
import org.avni.server.dao.search.SqlQuery;
import org.avni.server.domain.SubjectType;
import org.avni.server.web.request.webapp.search.SubjectSearchRequest;
import org.hibernate.query.internal.NativeQueryImpl;
import org.hibernate.transform.AliasToEntityMapResultTransformer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Repository;
import org.springframework.util.StringUtils;

import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
Expand All @@ -20,18 +22,21 @@
@Repository
public class SubjectSearchRepository extends RoleSwitchableRepository {
private final Logger logger = LoggerFactory.getLogger(SubjectSearchRepository.class);
private final SubjectTypeRepository subjectTypeRepository;

@PersistenceContext
private EntityManager entityManager;

public SubjectSearchRepository(EntityManager entityManager) {
public SubjectSearchRepository(EntityManager entityManager, SubjectTypeRepository subjectTypeRepository) {
super(entityManager);
this.subjectTypeRepository = subjectTypeRepository;
}

@Transactional
public List<Map<String,Object>> search(SubjectSearchRequest searchRequest, SearchBuilder searchBuilder) {
public List<Map<String, Object>> search(SubjectSearchRequest searchRequest, SearchBuilder searchBuilder) {
SubjectType subjectType = StringUtils.isEmpty(searchRequest.getSubjectType()) ? null : subjectTypeRepository.findByUuid(searchRequest.getSubjectType());
SqlQuery query = searchBuilder.getSQLResultQuery(searchRequest, subjectType);
try {
SqlQuery query = searchBuilder.getSQLResultQuery(searchRequest);
setRoleToNone();
logger.debug("Executing query: " + query.getSql());
logger.debug("Parameters: " + query.getParameters());
Expand All @@ -49,8 +54,9 @@ public List<Map<String,Object>> search(SubjectSearchRequest searchRequest, Searc

@Transactional
public BigInteger getTotalCount(SubjectSearchRequest searchRequest, SearchBuilder searchBuilder) {
SubjectType subjectType = StringUtils.isEmpty(searchRequest.getSubjectType()) ? null : subjectTypeRepository.findByUuid(searchRequest.getSubjectType());
SqlQuery query = searchBuilder.getSQLCountQuery(searchRequest, subjectType);
try {
SqlQuery query = searchBuilder.getSQLCountQuery(searchRequest);
setRoleToNone();
Query sql = entityManager.createNativeQuery(query.getSql());
query.getParameters().forEach((name, value) -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public class BaseSubjectSearchQueryBuilder<T> {
private static final String ADDRESS_LEVEL_JOIN = "left outer join address_level al on al.id = i.address_id";
private String orderByClause = "";

private final Set<String> whereClauses = new HashSet<>();
protected final Set<String> whereClauses = new HashSet<>();
private final Set<String> joinClauses = new LinkedHashSet<>();
private final Map<String, Object> parameters = new HashMap<>();
private boolean forCount;
Expand Down Expand Up @@ -193,13 +193,6 @@ public T withGenderFilter(List<String> genders) {
return (T) this;
}

public T withSubjectTypeFilter(String subjectType) {
if (subjectType == null) return (T) this;
addParameter("subjectTypeUuid", subjectType);
whereClauses.add("st.uuid = :subjectTypeUuid");
return (T) this;
}

public T withRegistrationDateFilter(DateRange registrationDateRange) {
if (registrationDateRange == null || registrationDateRange.isEmpty()) return (T) this;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package org.avni.server.dao.search;

import org.avni.server.domain.SubjectType;
import org.avni.server.web.request.webapp.search.SubjectSearchRequest;

public interface SearchBuilder {
SqlQuery getSQLResultQuery(SubjectSearchRequest searchRequest);
SqlQuery getSQLResultQuery(SubjectSearchRequest searchRequest, SubjectType subjectType);

SqlQuery getSQLCountQuery(SubjectSearchRequest searchRequest);
SqlQuery getSQLCountQuery(SubjectSearchRequest searchRequest, SubjectType subjectType);
}
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,12 @@ public SubjectAssignmentSearchQueryBuilder userGroupFilter(String groupUUID, Str
}

@Override
public SqlQuery getSQLResultQuery(SubjectSearchRequest searchRequest) {
public SqlQuery getSQLResultQuery(SubjectSearchRequest searchRequest, SubjectType subjectType) {
return this.withSubjectSearchFilter(searchRequest).build();
}

@Override
public SqlQuery getSQLCountQuery(SubjectSearchRequest searchRequest) {
public SqlQuery getSQLCountQuery(SubjectSearchRequest searchRequest, SubjectType subjectType) {
return this.withSubjectSearchFilter(searchRequest).forCount().build();
}
}
Original file line number Diff line number Diff line change
@@ -1,32 +1,46 @@
package org.avni.server.dao.search;

import org.avni.server.domain.SubjectType;
import org.avni.server.web.request.webapp.search.DateRange;
import org.avni.server.web.request.webapp.search.SubjectSearchRequest;
import org.springframework.stereotype.Component;

@Component
public class SubjectSearchQueryBuilder extends BaseSubjectSearchQueryBuilder<SubjectSearchQueryBuilder> implements SearchBuilder {
public static final String SubjectTypeColumn = " st.name as \"subjectTypeName\",\n";
public static final String HardCodedSubjectTypeColumn = " '$SubjectTypeName' as \"subjectTypeName\",\n";
public static final String SubjectTypeJoin = " left outer join subject_type st on i.subject_type_id = st.id and st.is_voided is false\n";

public SqlQuery build() {
public SqlQuery build(SubjectType subjectType) {
String baseQuery = "select $DISTINCT i.id as \"id\",\n" +
" i.first_name as \"firstName\",\n" +
" i.last_name as \"lastName\",\n" +
" i.profile_picture as \"profilePicture\",\n" +
" cast(concat_ws(' ',i.first_name,i.middle_name,i.last_name)as text) as \"fullName\",\n" +
" i.uuid as \"uuid\",\n" +
" i.address_id as \"addressId\",\n" +
" st.name as \"subjectTypeName\",\n" +
"$SubjectTypeColumn" +
" gender.name as \"gender\",\n" +
" i.date_of_birth as \"dateOfBirth\" $CUSTOM_FIELDS\n" +
"from individual i\n" +
" left outer join gender on i.gender_id = gender.id\n" +
" left outer join subject_type st on i.subject_type_id = st.id and st.is_voided is false\n";
"$SubjectTypeJoin";

if (subjectType == null) {
baseQuery = baseQuery.replace("$SubjectTypeColumn", SubjectTypeColumn);
baseQuery = baseQuery.replace("$SubjectTypeJoin", SubjectTypeJoin);
} else {
baseQuery = baseQuery.replace("$SubjectTypeColumn", HardCodedSubjectTypeColumn.replace("$SubjectTypeName", subjectType.getName()));
baseQuery = baseQuery.replace("$SubjectTypeJoin", "");
}
return super.buildUsingBaseQuery(baseQuery, "");
}

public SubjectSearchQueryBuilder withSubjectSearchFilter(SubjectSearchRequest request) {
public SubjectSearchQueryBuilder withSubjectSearchFilter(SubjectSearchRequest request, SubjectType subjectType) {
return this
.withNameFilter(request.getName())
.withSubjectTypeFilter(subjectType)
.withGenderFilter(request.getGender())
.withSubjectTypeFilter(request.getSubjectType())
.withAgeFilter(request.getAge())
.withRegistrationDateFilter(request.getRegistrationDate())
.withEncounterDateFilter(request.getEncounterDate())
Expand All @@ -40,6 +54,13 @@ public SubjectSearchQueryBuilder withSubjectSearchFilter(SubjectSearchRequest re
.withCustomFields(request.getSubjectType());
}

public SubjectSearchQueryBuilder withSubjectTypeFilter(SubjectType subjectType) {
if (subjectType == null) return this;
whereClauses.add("i.subject_type_id = :subjectTypeId");
addParameter("subjectTypeId", subjectType.getId());
return this;
}

public SubjectSearchQueryBuilder withEncounterDateFilter(DateRange encounterDateRange) {
if (encounterDateRange == null || encounterDateRange.isEmpty()) return this;
return withJoin(ENCOUNTER_JOIN, true)
Expand Down Expand Up @@ -69,12 +90,12 @@ public SubjectSearchQueryBuilder withProgramEnrolmentDateFilter(DateRange dateRa
}

@Override
public SqlQuery getSQLResultQuery(SubjectSearchRequest searchRequest) {
return this.withSubjectSearchFilter(searchRequest).build();
public SqlQuery getSQLResultQuery(SubjectSearchRequest searchRequest, SubjectType subjectType) {
return this.withSubjectSearchFilter(searchRequest, subjectType).build(subjectType);
}

@Override
public SqlQuery getSQLCountQuery(SubjectSearchRequest searchRequest) {
return this.withSubjectSearchFilter(searchRequest).forCount().build();
public SqlQuery getSQLCountQuery(SubjectSearchRequest searchRequest, SubjectType subjectType) {
return this.withSubjectSearchFilter(searchRequest, subjectType).forCount().build(subjectType);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ private LinkedHashMap<String, Object> constructIndividual(List<Map<String, Objec
.map(individualRecord -> ((BigInteger) individualRecord.get("addressId")).longValue())
.collect(Collectors.toList());

List<SearchSubjectEnrolledProgram> searchSubjectEnrolledPrograms = individualIds.size() > 0 ?
List<SearchSubjectEnrolledProgram> searchSubjectEnrolledPrograms = !individualIds.isEmpty() ?
programEnrolmentRepository.findActiveEnrolmentsByIndividualIds(individualIds) :
Collections.emptyList();

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package org.avni.server.dao.search;

import org.avni.server.dao.search.EncounterSearchQueryBuilder;
import org.avni.server.dao.search.SqlQuery;
import org.avni.server.domain.Organisation;
import org.avni.server.domain.UserContext;
import org.avni.server.framework.security.UserContextHolder;
Expand All @@ -13,7 +11,7 @@
import java.util.Date;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.Matchers.is;

public class EncounterSearchQueryBuilderTest {

Expand Down
Original file line number Diff line number Diff line change
@@ -1,109 +1,149 @@
package org.avni.server.dao.search;

import org.avni.server.dao.search.SqlQuery;
import org.avni.server.dao.search.SubjectSearchQueryBuilder;
import org.junit.Test;
import org.avni.server.domain.Organisation;
import org.avni.server.domain.SubjectType;
import org.avni.server.domain.UserContext;
import org.avni.server.domain.metadata.SubjectTypeBuilder;
import org.avni.server.framework.security.UserContextHolder;
import org.avni.server.web.request.webapp.search.Concept;
import org.avni.server.web.request.webapp.search.DateRange;
import org.avni.server.web.request.webapp.search.IntegerRange;
import org.junit.Before;
import org.junit.Test;

import java.util.ArrayList;
import java.util.Arrays;

import static org.assertj.core.api.Assertions.assertThat;

public class SubjectSearchQueryBuilderTest {
private SubjectType subjectType;

@Before
public void setup() {
UserContext userContext = new UserContext();
userContext.setOrganisation(new Organisation());
UserContextHolder.create(userContext);
subjectType = new SubjectTypeBuilder().setId(1l).setName("Individual").build();
}

@Test
public void withoutSubjectType() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withNameFilter("name")
.build(null);
String sql = query.getSql();
assertThat(sql).isNotEmpty();
assertThat(sql).contains(SubjectSearchQueryBuilder.SubjectTypeColumn);
assertThat(sql).contains(SubjectSearchQueryBuilder.SubjectTypeJoin);
}

@Test
public void shouldBuildBaseQueryWhenRunWithoutParameters() {
SqlQuery query = new SubjectSearchQueryBuilder().build();
System.out.println(query.getSql());
assertThat(query.getSql()).isNotEmpty();
SqlQuery query = new SubjectSearchQueryBuilder().build(subjectType);
String sql = query.getSql();
assertThat(sql).isNotEmpty();
}

@Test
public void shouldBeAbleToSearchByName() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withNameFilter("name")
.build();
assertThat(query.getSql()).isNotEmpty();
.withSubjectTypeFilter(subjectType)
.build(subjectType);
String sql = query.getSql();
System.out.println(sql);
assertThat(sql).isNotEmpty();
assertThat(sql).contains("i.subject_type_id = :subjectTypeId");
assertThat(sql).contains("'Individual' as \"subjectTypeName\"");
assertThat(sql).doesNotContain(SubjectSearchQueryBuilder.SubjectTypeColumn);
assertThat(sql).doesNotContain(SubjectSearchQueryBuilder.SubjectTypeJoin);
}

@Test
public void shouldNotAddNameFiltersForNullOrEmptyNames() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withNameFilter(" ")
.build();
.build(subjectType);
assertThat(query.getSql().contains("i.last_name ilike")).isFalse();

query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withNameFilter(null)
.build();
.build(subjectType);
assertThat(query.getSql().contains("i.last_name ilike")).isFalse();
}

@Test
public void shouldBreakNameStringIntoTokensInTheQuery() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withNameFilter("two tokens andAnother")
.build();
assertThat(query.getParameters().values().contains("%two%")).isTrue();
assertThat(query.getParameters().values().contains("%tokens%")).isTrue();
assertThat(query.getParameters().values().contains("%andAnother%")).isTrue();
assertThat(query.getParameters().size()).isEqualTo(5);
.build(subjectType);
assertThat(query.getParameters().containsValue("%two%")).isTrue();
assertThat(query.getParameters().containsValue("%tokens%")).isTrue();
assertThat(query.getParameters().containsValue("%andAnother%")).isTrue();
assertThat(query.getParameters().size()).isEqualTo(6);
}

@Test
public void shouldAddAgeFilter() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withAgeFilter(new IntegerRange(1, null))
.build();
assertThat(query.getParameters().size()).isEqualTo(3);
.build(subjectType);
assertThat(query.getParameters().size()).isEqualTo(4);
}

@Test
public void shouldAddGenderFilter() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withGenderFilter(null)
.build();
assertThat(query.getParameters().size()).isEqualTo(2);
.build(subjectType);
assertThat(query.getParameters().size()).isEqualTo(3);

ArrayList<String> genders = new ArrayList<>();
genders.add("firstGenderUuid");
query = new SubjectSearchQueryBuilder()
.withGenderFilter(genders)
.build();
.build(subjectType);
assertThat(query.getParameters().size()).isEqualTo(3);
}

@Test
public void shouldAddEncounterJoinWhtnAddingEncounterDateFilter() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withEncounterDateFilter(new DateRange("2021-01-01", "2022-01-01"))
.build();
assertThat(query.getParameters().size()).isEqualTo(4);
.build(subjectType);
assertThat(query.getParameters().size()).isEqualTo(5);
}

@Test
public void shouldNotAddTheSameJoinsMultipleTimes() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withProgramEncounterDateFilter(new DateRange("2021-01-01", "2022-01-01"))
.withProgramEnrolmentDateFilter(new DateRange("2021-01-01", "2022-01-01"))
.build();
assertThat(query.getParameters().size()).isEqualTo(6);
.build(subjectType);
assertThat(query.getParameters().size()).isEqualTo(7);
}

@Test
public void shouldAddConditionsForConcepts() {
SqlQuery query = new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.withConceptsFilter(Arrays.asList(
new Concept[]{new Concept("uuid", "registration", "CODED", Arrays.asList(new String[]{"asdf", "qwer"}), null)}))
.build();
.build(subjectType);
}

@Test
public void shouldMakeQueryForCount() {
SqlQuery query = new SubjectSearchQueryBuilder().forCount()
.build();
new SubjectSearchQueryBuilder()
.withSubjectTypeFilter(subjectType)
.forCount().build(subjectType);
}
}
Loading

0 comments on commit 54f1570

Please sign in to comment.