From dfbee6079575ce793b4dc6e9194b0d1bc0c1a2c6 Mon Sep 17 00:00:00 2001 From: DavidCroftDKFZ <46788708+DavidCroftDKFZ@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:21:43 +0200 Subject: [PATCH] Incorporated the directory-sync library into the code Took the code from https://github.com/samply/directory-sync and added it to this repository. This means that there is no dependency with the directory-sync repository any more. The original rational behind the separation was that the library might be used elsewhere, not just in a standalone service. This never actually happened, hence the decision to combine service and library, which makes the code easier to maintain. --- pom.xml | 98 +- .../samply/directory_sync_service/Util.java | 40 + .../samply/directory_sync_service/Utils.java | 21 - ...tionToDirectoryCollectionPutConverter.java | 155 ++++ .../FhirToDirectoryAttributeConverter.java | 110 +++ ...reateFactTablesFromStarModelInputData.java | 149 +++ .../directory/DirectoryApi.java | 870 ++++++++++++++++++ .../directory/DirectoryService.java | 75 ++ ...CollectionGetToDirectoryCollectionPut.java | 60 ++ .../directory/model/BbmriEricId.java | 82 ++ .../directory/model/Biobank.java | 198 ++++ .../directory/model/Collection.java | 6 + .../directory/model/CollectionBundle.java | 13 + .../model/DirectoryCollectionGet.java | 133 +++ .../model/DirectoryCollectionPut.java | 378 ++++++++ .../directory_sync_service/fhir/FhirApi.java | 647 +++++++++++++ .../fhir/FhirReporting.java | 441 +++++++++ .../fhir/PopulateStarModelInputData.java | 227 +++++ .../fhir/model/FhirCollection.java | 101 ++ .../model/StarModelData.java | 317 +++++++ .../{ => service}/Configuration.java | 2 +- .../{ => service}/DirectorySync.java | 26 +- .../{ => service}/DirectorySyncJob.java | 2 +- .../{ => service}/DirectorySyncLauncher.java | 2 +- .../{ => service}/DirectorySyncService.java | 2 +- .../directory_sync_service/sync/Sync.java | 441 +++++++++ .../fhir/CollectionSize.Library.json | 23 + .../fhir/CollectionSize.Measure.json | 57 ++ .../fhir/CollectionSize.cql | 13 + 29 files changed, 4608 insertions(+), 81 deletions(-) create mode 100644 src/main/java/de/samply/directory_sync_service/Util.java delete mode 100644 src/main/java/de/samply/directory_sync_service/Utils.java create mode 100644 src/main/java/de/samply/directory_sync_service/converter/FhirCollectionToDirectoryCollectionPutConverter.java create mode 100644 src/main/java/de/samply/directory_sync_service/converter/FhirToDirectoryAttributeConverter.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/CreateFactTablesFromStarModelInputData.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/DirectoryApi.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/DirectoryService.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/MergeDirectoryCollectionGetToDirectoryCollectionPut.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/model/BbmriEricId.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/model/Biobank.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/model/Collection.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/model/CollectionBundle.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionGet.java create mode 100644 src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionPut.java create mode 100644 src/main/java/de/samply/directory_sync_service/fhir/FhirApi.java create mode 100644 src/main/java/de/samply/directory_sync_service/fhir/FhirReporting.java create mode 100644 src/main/java/de/samply/directory_sync_service/fhir/PopulateStarModelInputData.java create mode 100644 src/main/java/de/samply/directory_sync_service/fhir/model/FhirCollection.java create mode 100644 src/main/java/de/samply/directory_sync_service/model/StarModelData.java rename src/main/java/de/samply/directory_sync_service/{ => service}/Configuration.java (95%) rename src/main/java/de/samply/directory_sync_service/{ => service}/DirectorySync.java (91%) rename src/main/java/de/samply/directory_sync_service/{ => service}/DirectorySyncJob.java (98%) rename src/main/java/de/samply/directory_sync_service/{ => service}/DirectorySyncLauncher.java (98%) rename src/main/java/de/samply/directory_sync_service/{ => service}/DirectorySyncService.java (98%) create mode 100644 src/main/java/de/samply/directory_sync_service/sync/Sync.java create mode 100644 src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Library.json create mode 100644 src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Measure.json create mode 100644 src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.cql diff --git a/pom.xml b/pom.xml index 18a9d1f..4b55f24 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ de.samply directory_sync_service - 1.3.3 + 1.3.5 directory_sync_service Directory sync https://github.com/samply/directory_sync_service @@ -26,62 +26,70 @@ 19 + 6.2.2 + 2.13.3 - - - org.springframework.boot - spring-boot-starter - + + org.springframework.boot + spring-boot-starter + - - org.springframework.boot - spring-boot-starter-test - test - + + org.springframework.boot + spring-boot-starter-test + test + - - org.quartz-scheduler - quartz - 2.3.2 - - - - de.samply - directory-sync - 0.3.3 - - - - de.samply - common-http - 7.4.4 - - - - ca.uhn.hapi.fhir - hapi-fhir-structures-r4 - 6.2.2 - - - ca.uhn.hapi.fhir - hapi-fhir-client - 6.2.2 - - - - commons-validator - commons-validator - 1.7 - + + org.quartz-scheduler + quartz + 2.3.2 + + + + ca.uhn.hapi.fhir + hapi-fhir-structures-r4 + ${hapi.version} + + + + ca.uhn.hapi.fhir + hapi-fhir-client + ${hapi.version} + org.projectlombok lombok provided + + + com.google.code.gson + gson + 2.9.0 + + + + io.vavr + vavr + 0.10.4 + + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + diff --git a/src/main/java/de/samply/directory_sync_service/Util.java b/src/main/java/de/samply/directory_sync_service/Util.java new file mode 100644 index 0000000..36226aa --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/Util.java @@ -0,0 +1,40 @@ +package de.samply.directory_sync_service; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; + +import com.google.gson.Gson; + +public class Util { + + public static Map mapOf() { + return new HashMap<>(); + } + + /** + * Creates a new map with a single entry consisting of the specified key and value. + * + * @param key the key for the entry + * @param value the value associated with the key + * @return a new map containing the specified key-value pair + */ + public static Map mapOf(K key, V value) { + HashMap map = new HashMap<>(); + map.put(key, value); + return map; + } + + /** + * Get a printable stack trace from an Exception object. + * @param e + * @return + */ + public static String traceFromException(Exception e) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + return sw.toString(); + } +} diff --git a/src/main/java/de/samply/directory_sync_service/Utils.java b/src/main/java/de/samply/directory_sync_service/Utils.java deleted file mode 100644 index d0524d6..0000000 --- a/src/main/java/de/samply/directory_sync_service/Utils.java +++ /dev/null @@ -1,21 +0,0 @@ -package de.samply.directory_sync_service; - -import java.io.PrintWriter; -import java.io.StringWriter; - -/** - * Parameters and functions used throughout the code. - */ -public class Utils { - /** - * Get a printable stack trace from an Exception object. - * @param e - * @return - */ - public static String traceFromException(Exception e) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e.printStackTrace(pw); - return sw.toString(); - } -} diff --git a/src/main/java/de/samply/directory_sync_service/converter/FhirCollectionToDirectoryCollectionPutConverter.java b/src/main/java/de/samply/directory_sync_service/converter/FhirCollectionToDirectoryCollectionPutConverter.java new file mode 100644 index 0000000..669fa4f --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/converter/FhirCollectionToDirectoryCollectionPutConverter.java @@ -0,0 +1,155 @@ +package de.samply.directory_sync_service.converter; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionPut; +import de.samply.directory_sync_service.fhir.model.FhirCollection; + +/** + * Takes a list of FhirCollection objects and converts them into a + * DirectoryCollectionPut object. + * + * This is a kind of FHIR to Directory conversion, so there are differences + * in vocabularies, and it is the job of this converter to impedence match them. + */ +public class FhirCollectionToDirectoryCollectionPutConverter { + private static final Logger logger = LoggerFactory.getLogger(FhirCollectionToDirectoryCollectionPutConverter.class); + + public static DirectoryCollectionPut convert(List fhirCollections) { + DirectoryCollectionPut directoryCollectionPut = new DirectoryCollectionPut(); + + for (FhirCollection fhirCollection: fhirCollections) + if (convert(directoryCollectionPut, fhirCollection) == null) { + directoryCollectionPut = null; + break; + } + + return directoryCollectionPut; + } + + private static DirectoryCollectionPut convert(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + try { + convertSize(directoryCollectionPut, fhirCollection); + convertNumberOfDonors(directoryCollectionPut, fhirCollection); + convertSex(directoryCollectionPut, fhirCollection); + convertAgeLow(directoryCollectionPut, fhirCollection); + convertAgeHigh(directoryCollectionPut, fhirCollection); + convertMaterials(directoryCollectionPut, fhirCollection); + convertStorageTemperatures(directoryCollectionPut, fhirCollection); + // convertDiagnosisAvailableEmpty(directoryCollectionPut, fhirCollection); + convertDiagnosisAvailable(directoryCollectionPut, fhirCollection); + } catch(Exception e) { + logger.error("Problem converting FHIR attributes to Directory attributes. " + Util.traceFromException(e)); + return null; + } + + return directoryCollectionPut; + } + + private static void convertSize(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + Integer size = fhirCollection.getSize(); + directoryCollectionPut.setSize(id, size); + // Order of magnitude is mandatory in the Directory and can be derived from size + directoryCollectionPut.setOrderOfMagnitude(id, (int) Math.floor(Math.log10(size))); + } + + public static void convertNumberOfDonors(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + Integer size = fhirCollection.getNumberOfDonors(); + directoryCollectionPut.setNumberOfDonors(id, size); + // Order of magnitude is mandatory in the Directory and can be derived from size + directoryCollectionPut.setOrderOfMagnitudeDonors(id, (int) Math.floor(Math.log10(size))); + } + + public static void convertSex(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + List sex = fhirCollection.getSex(); + + List ucSex = sex.stream() + .map(s -> FhirToDirectoryAttributeConverter.convertSex(s)) + .filter(Objects::nonNull) // Use a method reference to check for non-null values + .distinct() // Remove duplicate elements + .collect(Collectors.toList()); + + directoryCollectionPut.setSex(id, ucSex); + } + + public static void convertAgeLow(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + Integer ageLow = fhirCollection.getAgeLow(); + // No conversion needed + directoryCollectionPut.setAgeLow(id, ageLow); + } + + public static void convertAgeHigh(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + Integer ageHigh = fhirCollection.getAgeHigh(); + // No conversion needed + directoryCollectionPut.setAgeHigh(id, ageHigh); + } + + public static void convertMaterials(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + List materials = fhirCollection.getMaterials(); + + if (materials == null) + materials = new ArrayList(); + + List directoryMaterials = materials.stream() + .map(s -> FhirToDirectoryAttributeConverter.convertMaterial(s)) + .filter(Objects::nonNull) // Use a method reference to check for non-null values + .distinct() // Remove duplicate elements + .collect(Collectors.toList()); + + directoryCollectionPut.setMaterials(id, directoryMaterials); + } + + public static void convertStorageTemperatures(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + List storageTemperatures = fhirCollection.getStorageTemperatures(); + + if (storageTemperatures == null) + storageTemperatures = new ArrayList(); + + List directoryStorageTemperatures = storageTemperatures.stream() + .map(s -> FhirToDirectoryAttributeConverter.convertStorageTemperature(s)) + .filter(Objects::nonNull) // Use a method reference to check for non-null values + .distinct() // Remove duplicate elements + .collect(Collectors.toList()); + + directoryCollectionPut.setStorageTemperatures(id, directoryStorageTemperatures); + } + + public static void convertDiagnosisAvailableEmpty(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + // The Directory is very picky about which ICD10 codes it will accept, and some + // of the codes that are in our test data are not known to the Directory and + // give rise to errors, which lead to the entire PUT to the Directory being + // rejected. So, for the time being, I am turning off the diagnosis conversion. + directoryCollectionPut.setDiagnosisAvailable(id, new ArrayList()); + } + + public static void convertDiagnosisAvailable(DirectoryCollectionPut directoryCollectionPut, FhirCollection fhirCollection) { + String id = fhirCollection.getId(); + List diagnoses = fhirCollection.getDiagnosisAvailable(); + + if (diagnoses == null) + diagnoses = new ArrayList(); + + List miriamDiagnoses = diagnoses.stream() + .map(icd -> FhirToDirectoryAttributeConverter.convertDiagnosis(icd)) + .filter(Objects::nonNull) // Use a method reference to check for non-null values + .distinct() // Remove duplicate diagnoses + .collect(Collectors.toList()); + + directoryCollectionPut.setDiagnosisAvailable(id, miriamDiagnoses); + } +} diff --git a/src/main/java/de/samply/directory_sync_service/converter/FhirToDirectoryAttributeConverter.java b/src/main/java/de/samply/directory_sync_service/converter/FhirToDirectoryAttributeConverter.java new file mode 100644 index 0000000..3e4a261 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/converter/FhirToDirectoryAttributeConverter.java @@ -0,0 +1,110 @@ +package de.samply.directory_sync_service.converter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for converting FHIR attributes to Directory attributes. + * This class provides static methods to perform conversions for specific FHIR attributes + * in order to align them with Directory attribute conventions. + */ +public class FhirToDirectoryAttributeConverter { + private static final Logger logger = LoggerFactory.getLogger(FhirToDirectoryAttributeConverter.class); + + /** + * Converts the given sex attribute to uppercase as per Directory conventions. + * + * @param sex The sex attribute to be converted. + * @return The converted sex attribute in uppercase. + */ + public static String convertSex(String sex) { + // Signifiers for sex largely overlap between FHIR and Directory, but Directory likes + // upper case + return sex.toUpperCase(); + } + + /** + * Converts the given age attribute. No conversion needed. + * + * @param age The age attribute to be converted. + * @return The unchanged age attribute. + */ + public static Integer convertAge(Integer age) { + // No conversion needed + return age; + } + + /** + * Converts the given material attribute to Directory conventions. + * + * @param material The material attribute to be converted. + * @return The converted material attribute according to Directory conventions. + */ + public static String convertMaterial(String material) { + if (material == null) + return null; + + String directoryMaterial = material + // Basic conversion: make everything upper case, replace - with _ + .toUpperCase() + .replaceAll("-", "_") + // Some names are different between FHIR and Directory, so convert those. + .replaceAll("_VITAL", "") + .replaceAll("^TISSUE_FORMALIN$", "TISSUE_PARAFFIN_EMBEDDED") + .replaceAll("^TISSUE$", "TISSUE_FROZEN") + .replaceAll("^CF_DNA$", "CDNA") + .replaceAll("^BLOOD_SERUM$", "SERUM") + .replaceAll("^STOOL_FAECES$", "FECES") + .replaceAll("^BLOOD_PLASMA$", "SERUM") + // Some names are present in FHIR but not in Directory. Use "OTHER" as a placeholder. + .replaceAll("^.*_OTHER$", "OTHER") + .replaceAll("^DERIVATIVE$", "OTHER") + .replaceAll("^CSF_LIQUOR$", "OTHER") + .replaceAll("^LIQUID$", "OTHER") + .replaceAll("^ASCITES$", "OTHER") + .replaceAll("^BONE_MARROW$", "OTHER") + .replaceAll("^TISSUE_PAXGENE_OR_ELSE$", "OTHER") + ; + + return directoryMaterial; + } + + /** + * Converts the given storage temperature attribute to align with Directory conventions. + * + * @param storageTemperature The storage temperature attribute to be converted. + * @return The converted storage temperature attribute according to Directory conventions. + */ + public static String convertStorageTemperature(String storageTemperature) { + if (storageTemperature == null) + return null; + + // The Directory understands most of the FHIR temperature codes, but it doesn't + // know about gaseous nitrogen. + String directoryStorageTemperature = storageTemperature + .replaceAll("temperatureGN", "temperatureOther"); + + return directoryStorageTemperature; + } + + /** + * Converts the given diagnosis attribute to a MIRIAM ICD code if not already in MIRIAM format. + * + * @param diagnosis The diagnosis attribute to be converted. + * @return The converted diagnosis attribute in MIRIAM ICD format or null if the input is null. + */ + public static String convertDiagnosis(String diagnosis) { + if (diagnosis == null) + return null; + + String miriamDiagnosis = null; + if (diagnosis.startsWith("urn:miriam:icd:")) + miriamDiagnosis = diagnosis; + else if (diagnosis.length() == 3 || diagnosis.length() == 5) // E.g. C75 or E23.1 + miriamDiagnosis = "urn:miriam:icd:" + diagnosis; + else + logger.warn("Entities.setDiagnosisAvailable: invalid diagnosis code " + diagnosis); + + return miriamDiagnosis; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/CreateFactTablesFromStarModelInputData.java b/src/main/java/de/samply/directory_sync_service/directory/CreateFactTablesFromStarModelInputData.java new file mode 100644 index 0000000..aa20763 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/CreateFactTablesFromStarModelInputData.java @@ -0,0 +1,149 @@ +package de.samply.directory_sync_service.directory; + +import de.samply.directory_sync_service.model.StarModelData; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for creating fact tables from Star Model input data. + *

+ * The implementation includes methods for creating fact tables from input data, grouping specimens + * by collection, and applying various data transformations. + *

+ */ +public class CreateFactTablesFromStarModelInputData { + private static final Logger logger = LoggerFactory.getLogger(CreateFactTablesFromStarModelInputData.class); + + /** + * Creates fact tables for each collection in the provided Star Model input data. + * Fact tables are generated based on input rows and specified criteria such as minimum donors. + * + * @param starModelInputData The Star Model input data containing information for fact table creation. + * @param maxFacts + * + * @throws NullPointerException if starModelInputData is null. + */ + public static void createFactTables(StarModelData starModelInputData, int maxFacts) { + for (String collectionId: starModelInputData.getInputCollectionIds()) { + List> factTableFinal = createFactTableFinal(collectionId, + starModelInputData.getMinDonors(), + maxFacts, + starModelInputData.getInputRowsAsStringMaps(collectionId)); + starModelInputData.addFactTable(collectionId, factTableFinal); + } + } + + /** + * Creates a final fact table for a specific collection based on input rows, minimum donors, and data transformations. + * + * This code was translated from Petr Holub's R script "CRC-fact-sheet.R". + * + * @param collectionId The identifier for the collection for which to create the fact table. + * @param minDonors The minimum number of donors required for a fact to be included in the table. + * @param maxFacts + * @param patientSamples The list of input rows representing patient samples for the collection. + * @return The final fact table as a list of maps containing key-value pairs. + */ + private static List> createFactTableFinal(String collectionId, int minDonors, int maxFacts, List> patientSamples) { + // Select columns and create a new column "age_range" + List> patientSamplesFacts = patientSamples.stream() + .map(result -> { + Map fact = new HashMap<>(result); + fact.remove("sample_type"); + fact.remove("sample_year_num"); + fact.put("age_range", cutAgeRange(result.get("age_at_primary_diagnosis"))); + return fact; + }) + .collect(Collectors.toList()); + + // Group by certain columns and calculate summary statistics + Map factTable = patientSamplesFacts.stream() + .filter(fact -> !fact.containsValue(null)) + .collect(Collectors.groupingBy( + fact -> String.join("_", + fact.get("sex"), + fact.get("hist_loc"), + fact.get("age_range"), + fact.get("sample_material")), + Collectors.counting())); + + // Filter out rows with fewer than a given number of donors + if (minDonors > 0) + factTable = factTable.entrySet().stream() + .filter(entry -> entry.getValue() >= minDonors) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Perform additional data transformations on the facts table + List> factTableFinal = new ArrayList<>(); + int factTableSize = 0; + for (Map.Entry entry : factTable.entrySet()) { + if (maxFacts >= 0 && factTableSize >= maxFacts) + break; + factTableSize++; + + List keyParts = Arrays.asList(entry.getKey().split("_")); + + String sampleMaterial = keyParts.get(3); + for (int i=4; i mapEntry = new HashMap<>(); + mapEntry.put("sex", keyParts.get(0)); + mapEntry.put("disease", keyParts.get(1)); + mapEntry.put("age_range", keyParts.get(2)); + mapEntry.put("sample_type", sampleMaterial); + mapEntry.put("number_of_donors", Long.toString(entry.getValue())); + mapEntry.put("number_of_samples", Long.toString(entry.getValue())); + mapEntry.put("id", "bbmri-eric:factID:" // All fact IDs must start with this (mandatory). + // Snip "bbmri-eric:ID:" from collection ID and replace : with _ + + collectionId.substring(14, collectionId.length()).replaceAll(":", "_") + + "_" + + Math.abs(entry.getKey().hashCode()) // Add a hash code to make the ID unique + ); + mapEntry.put("last_update", LocalDate.now().toString()); + mapEntry.put("collection", collectionId); + + factTableFinal.add(mapEntry); + } + + return factTableFinal; + } + + /** + * Cuts the age into bins and returns the corresponding age range. + * + * @param age The age to be categorized into bins. + * @return The age range as a string. + */ + private static String cutAgeRange(String age) { + if (age == null || age.isBlank()) + return "Unknown"; + + // Logic to cut age into bins + int ageValue = Integer.parseInt(age); + if (ageValue < 1) { + return "Infant"; + } else if (ageValue < 2) { + return "Infant"; + } else if (ageValue < 13) { + return "Child"; + } else if (ageValue < 18) { + return "Adolescent"; + } else if (ageValue < 45) { + return "Adult"; + } else if (ageValue < 65) { + return "Middle-aged"; + } else if (ageValue < 80) { + return "Aged (65-79 years)"; + } else { + return "Aged (>80 years)"; + } + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/DirectoryApi.java b/src/main/java/de/samply/directory_sync_service/directory/DirectoryApi.java new file mode 100644 index 0000000..ae051b7 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/DirectoryApi.java @@ -0,0 +1,870 @@ +package de.samply.directory_sync_service.directory; + +import de.samply.directory_sync_service.model.StarModelData; +import de.samply.directory_sync_service.Util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity.ERROR; +import static org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity.INFORMATION; +import static org.hl7.fhir.r4.model.OperationOutcome.IssueType.NOTFOUND; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import de.samply.directory_sync_service.directory.model.BbmriEricId; +import de.samply.directory_sync_service.directory.model.Biobank; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionGet; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionPut; +import io.vavr.control.Either; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; +import java.net.URI; + +public class DirectoryApi { + private static final Logger logger = LoggerFactory.getLogger(DirectoryApi.class); + + private final CloseableHttpClient httpClient; + private final String baseUrl; + private final String token; + private final Gson gson = new Gson(); + private String username; + private String password; + + // Setting this variable to true will prevent any contact being made to the Directory. + // All public methods will return feasible fake results. + private boolean mockDirectory = false; + + private DirectoryApi(CloseableHttpClient httpClient, String baseUrl, String token, boolean mockDirectory) { + this.httpClient = Objects.requireNonNull(httpClient); + this.baseUrl = Objects.requireNonNull(baseUrl); + this.mockDirectory = mockDirectory; + if (mockDirectory) + // if we are mocking, then we don't need to check token, because it won't get used anyway. + this.token = token; + else { + if (token == null) + logger.warn("No token provided, directory operations will not be logged in."); + this.token = Objects.requireNonNull(token); + } + } + +/* + public static Either createWithLogin( + CloseableHttpClient httpClient, + String baseUrl, + String username, + String password, + boolean mockDirectory) { + return login(httpClient, baseUrl.replaceFirst("/*$", ""), username, password, mockDirectory) + .map(response -> createWithToken(httpClient, baseUrl, response.token, mockDirectory).setUsernameAndPassword(username, password)); + } +*/ + + public static Either createWithLogin( + CloseableHttpClient httpClient, + String baseUrl, + String username, + String password, + boolean mockDirectory) { + + // Clean up the baseUrl + String cleanedBaseUrl = baseUrl.replaceFirst("/*$", ""); + + logger.info("createWithLogin: cleanedBaseUrl: " + cleanedBaseUrl); + + // Perform the login operation and get the result + Either loginResult = login(httpClient, cleanedBaseUrl, username, password, mockDirectory); + + // If login is successful, map the result to a DirectoryApi, otherwise return the error + Either finalResult = loginResult.map(response -> { + logger.info("createWithLogin: log in to Directory apparently succeeded"); + if (response.token == null) + logger.warn("createWithLogin: response.token is null"); + // Create the DirectoryApi with the token and set the username and password + DirectoryApi api = createWithToken(httpClient, cleanedBaseUrl, response.token, mockDirectory); + return api.setUsernameAndPassword(username, password); + }); + + logger.warn("createWithLogin: log in to Directory failed: " + finalResult); + + return finalResult; + } + + private static Either login(CloseableHttpClient httpClient, + String baseUrl, + String username, String password, boolean mockDirectory) { + if (mockDirectory) + // Don't try logging in if we are mocking + return Either.right(new LoginResponse()); + HttpPost request = loginRequest(baseUrl, username, password); + try (CloseableHttpResponse response = httpClient.execute(request)) { + return Either.right(decodeLoginResponse(response)); + } catch (IOException e) { + return Either.left(error("login", e.getMessage())); + } + } + + /** + * Store username and password internally, to allow relogin where necessary. + * + * @param username + * @param password + * @return + */ + private DirectoryApi setUsernameAndPassword(String username, String password) { + this.username = username; + this.password = password; + + return this; + } + + /** + * Log back in to the Directory. This is typically used in situations where there has + * been a long pause since the last API call to the Directory. It returns a fresh + * DirectoryApi object, which you should use to replace the existing one. + * + * Returns null if something goes wrong. + * + * @return new DirectoryApi object. + */ + public DirectoryApi relogin() { + logger.info("relogin: logging back in"); + HttpPost request = loginRequest(baseUrl, username, password); + + if (mockDirectory) + // In a mocking situation, don't try to log back in. We can safely + // return the old DirectoryApi object because nothing will have changed. + return this; + + String token = null; + try (CloseableHttpResponse response = httpClient.execute(request)) { + LoginResponse loginResponse = decodeLoginResponse(response); + token = loginResponse.token; + if (token == null) { + logger.warn("relogin: got a null token back from the Directory, returning null."); + return null; + } + } catch (IOException e) { + logger.warn("relogin: exception: " + Util.traceFromException(e)); + return null; + } + + return new DirectoryApi(httpClient, baseUrl.replaceFirst("/*$", ""), token, mockDirectory).setUsernameAndPassword(username, password); + } + + private static HttpPost loginRequest(String baseUrl, String username, String password) { + HttpPost request = new HttpPost(baseUrl + "/api/v1/login"); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + request.setEntity(encodeLoginCredentials(username, password)); + return request; + } + + private static StringEntity encodeLoginCredentials(String username, String password) { + return new StringEntity(new Gson().toJson(new LoginCredentials(username, password)), UTF_8); + } + + private static LoginResponse decodeLoginResponse(CloseableHttpResponse tokenResponse) + throws IOException { + String body = EntityUtils.toString(tokenResponse.getEntity(), UTF_8); + return new Gson().fromJson(body, LoginResponse.class); + } + + public static DirectoryApi createWithToken(CloseableHttpClient httpClient, String baseUrl, + String token) { + return createWithToken(httpClient, baseUrl, token, false); + } + + public static DirectoryApi createWithToken(CloseableHttpClient httpClient, String baseUrl, + String token, boolean mockDirectory) { + return new DirectoryApi(httpClient, baseUrl.replaceFirst("/*$", ""), token, mockDirectory); + } + + private static OperationOutcome error(String action, String message) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR).setDiagnostics(errorMsg(action, message)); + return outcome; + } + + private static String errorMsg(String action, String message) { + return String.format("Error in BBMRI Directory response for %s, cause: %s", action, + message); + } + + private static OperationOutcome biobankNotFound(BbmriEricId id) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue() + .setSeverity(INFORMATION) + .setCode(NOTFOUND) + .setDiagnostics(String.format("No Biobank in Directory with id `%s`.", id)); + return outcome; + } + + private static OperationOutcome updateSuccessful(int number) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue() + .setSeverity(INFORMATION) + .setDiagnostics(String.format("Successful update of %d collection size values.", number)); + return outcome; + } + + /** + * Fetches the Biobank with the given {@code id}. + * + * @param id the ID of the Biobank to fetch. + * @return either the Biobank or an error + */ + public Either fetchBiobank(BbmriEricId id) { + try (CloseableHttpResponse response = httpClient.execute(fetchBiobankRequest(id))) { + if (response.getStatusLine().getStatusCode() == 200) { + String payload = EntityUtils.toString(response.getEntity(), UTF_8); + return Either.right(gson.fromJson(payload, Biobank.class)); + } else if (response.getStatusLine().getStatusCode() == 404) { + return Either.left(biobankNotFound(id)); + } else { + String message = EntityUtils.toString(response.getEntity(), UTF_8); + return Either.left(error(id.toString(), message)); + } + } catch (IOException e) { + return Either.left(error(id.toString(), e.getMessage())); + } + } + + private HttpGet fetchBiobankRequest(BbmriEricId id) { + HttpGet request = new HttpGet( + baseUrl + "/api/v2/eu_bbmri_eric_" + id.getCountryCode() + "_biobanks/" + id); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + return request; + } + + /** + * Send the collection sizes to the Directory. + *

+ * Push the counts back to the Directory. You need 'update data' permission on entity type + * 'Collections' at the Directory in order for this to work. + * + * @param countryCode the country code of the endpoint of the national node, e.g. Germany + * @param collectionSizeDtos the individual collection sizes. note that all collection must share + * the given {@code countryCode} + * @return an outcome, either successful or an error + */ + public OperationOutcome updateCollectionSizes(String countryCode, + List collectionSizeDtos) { + + HttpPut request = updateCollectionSizesRequest(countryCode, collectionSizeDtos); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + if (response.getStatusLine().getStatusCode() < 300) { + return updateSuccessful(collectionSizeDtos.size()); + } else { + return error("collection size update status code " + response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), UTF_8)); + } + } catch (IOException e) { + return error("collection size update exception", e.getMessage()); + } + } + + private HttpPut updateCollectionSizesRequest(String countryCode, + List collectionSizeDtos) { + HttpPut request = new HttpPut( + baseUrl + "/api/v2/eu_bbmri_eric_collections/size"); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + request.setEntity(new StringEntity(gson.toJson(new EntitiesDto<>(collectionSizeDtos)), UTF_8)); + return request; + } + + /** + * Make a call to the Directory to get all Collection IDs for the supplied {@code countryCode}. + * + * @param countryCode the country code of the endpoint of the national node, e.g. DE + * @return all the Collections for the national node. E.g. "DE" will return all German collections + */ + public Either> listAllCollectionIds(String countryCode) { + return fetchIdItems(listAllCollectionIdsRequest(countryCode), "list collection ids") + .map(i -> i.items.stream() + .map(e -> e.id) + .map(BbmriEricId::valueOf) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toSet())); + } + + private HttpGet listAllCollectionIdsRequest(String countryCode) { + // If you simply specify "attrs=id", you will only get the first 100 + // IDs. Setting "start" to 0 and "num" its maximum allowed value + // gets them all. Note that in the current Directory implementation + // (12.10.2021), the maximum allowed value of "num" is 10000. + // TODO: to really get all collections, we have to implement paging + HttpGet request = new HttpGet( + baseUrl + "/api/v2/eu_bbmri_eric_collections?attrs=id&start=0&num=10000&q=country==" + + countryCode); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + return request; + } + + /** + * Make API calls to the Directory to fill a DirectoryCollectionGet object containing attributes + * for all of the collections listed in collectionIds. The countryCode is used solely for + * constructing the URL for the API call. + * + * @param countryCode E.g. "DE". + * @param collectionIds IDs of the collections whose data will be harvested. + * @return + */ + public Either fetchCollectionGetOutcomes(String countryCode, List collectionIds) { + DirectoryCollectionGet directoryCollectionGet = new DirectoryCollectionGet(); // for all collections retrieved from Directory + directoryCollectionGet.init(); + for (String collectionId: collectionIds) { + try { + HttpGet request = fetchCollectionsRequest(countryCode, collectionId); + + if (mockDirectory) { + // Dummy return if we're in mock mode + directoryCollectionGet.setMockDirectory(true); + return Either.right(directoryCollectionGet); + } + + CloseableHttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() < 300) { + HttpEntity httpEntity = response.getEntity(); + String json = EntityUtils.toString(httpEntity); + DirectoryCollectionGet singleDirectoryCollectionGet = gson.fromJson(json, DirectoryCollectionGet.class); + Map item = singleDirectoryCollectionGet.getItemZero(); // assume that only one collection matches collectionId + if (item == null) + return Either.left(error("fetchCollectionGetOutcomes: entity get item is null, does the collection exist in the Directory: ", collectionId)); + directoryCollectionGet.getItems().add(item); + } else + return Either.left(error("fetchCollectionGetOutcomes: entity get HTTP error", Integer.toString(response.getStatusLine().getStatusCode()))); + } catch (IOException e) { + return Either.left(error("fetchCollectionGetOutcomes: entity get exception", Util.traceFromException(e))); + } catch (Exception e) { + return Either.left(error("fetchCollectionGetOutcomes: unknown exception", Util.traceFromException(e))); + } + } + + return Either.right(directoryCollectionGet); + } + + private HttpGet fetchCollectionsRequest(String countryCode, String collectionId) { + String url = buildCollectionApiUrl(countryCode) + "?q=id==%22" + collectionId + "%22"; + + logger.info("DirectoryApi.fetchCollectionsRequest: url=" + url); + + HttpGet request = new HttpGet(url); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + + logger.info("DirectoryApi.fetchCollectionsRequest: request successfully built"); + + return request; + } + + /** + * Send aggregated collection information to the Directory. + * + * @param directoryCollectionPut Summary information about one or more collections + * @return an outcome, either successful or an error + */ + public OperationOutcome updateEntities(DirectoryCollectionPut directoryCollectionPut) { + logger.info("DirectoryApi.updateEntities: entered"); + + HttpPut request = updateEntitiesRequest(directoryCollectionPut); + + logger.info("DirectoryApi.updateEntities: url=" + request.getURI()); + + if (mockDirectory) + // Dummy return if we're in mock mode + return updateSuccessful(directoryCollectionPut.size()); + + logger.info("DirectoryApi.updateEntities: try things"); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + logger.info("DirectoryApi.updateEntities: well, now Im in a try statement!"); + if (response.getStatusLine().getStatusCode() < 300) { + logger.info("DirectoryApi.updateEntities: status code: " + response.getStatusLine().getStatusCode()); + return updateSuccessful(directoryCollectionPut.size()); + } else { + logger.info("DirectoryApi.updateEntities: returning an error"); + return error("entity update status code " + response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), UTF_8)); + } + } catch (IOException e) { + logger.info("DirectoryApi.updateEntities: returning an exception: " + Util.traceFromException(e)); + return error("entity update exception", e.getMessage()); + } + } + + private HttpPut updateEntitiesRequest(DirectoryCollectionPut directoryCollectionPut) { + HttpPut request = new HttpPut(buildCollectionApiUrl(directoryCollectionPut.getCountryCode())); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + logger.info("updateEntitiesRequest: directoryCollectionPut: " + gson.toJson(directoryCollectionPut)); + request.setEntity(new StringEntity(gson.toJson(directoryCollectionPut), UTF_8)); + return request; + } + + /** + * Updates the Star Model data in the Directory service based on the provided StarModelInputData. + * + * Before sending any star model data to the Directory, the original + * star model data for all known collections will be deleted from the + * Directory. + * + * @param starModelInputData The input data for updating the Star Model. + * @return An OperationOutcome indicating the success or failure of the update. + */ + public OperationOutcome updateStarModel(StarModelData starModelInputData) { + // Get rid of previous star models first. This is necessary, because: + // 1. A new star model may be decomposed into different hypercubes. + // 2. The new fact IDs may be different from the old ones. + // 3. We will be using a POST and it will return an error if we try + // to overwrite an existing fact. + OperationOutcome deleteOutcome = deleteStarModel(starModelInputData); + if (deleteOutcome.getIssue().size() > 0) { + logger.warn("updateStarModel: Problem deleting star models"); + return deleteOutcome; + } + + String countryCode = starModelInputData.getCountryCode(); + List> factTables = starModelInputData.getFactTables(); + int blockSize = 1000; + + // Break the fact table into blocks of 1000 before sending to the Directory. + // This is the maximum number of facts allowed per Directory API call. + for (int i = 0; i < factTables.size(); i += blockSize) { + List> factTablesBlock = factTables.subList(i, Math.min(i + blockSize, factTables.size())); + + // Now push the new data + HttpPost request = updateStarModelRequestBlock(countryCode, factTablesBlock); + + if (mockDirectory) + // Dummy return if we're in mock mode + return updateSuccessful(starModelInputData.getFactCount()); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + if (response.getStatusLine().getStatusCode() >= 300) + return error("entity update status code " + response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), UTF_8)); + } catch (IOException e) { + return error("entity update exception", e.getMessage()); + } + } + + return updateSuccessful(starModelInputData.getFactCount()); + } + + /** + * Constructs an HTTP POST request for updating Star Model data based on the provided StarModelInputData. + * + * @param countryCode + * @param factTablesBlock + * @return An HttpPost request object. + */ + private HttpPost updateStarModelRequestBlock(String countryCode, List> factTablesBlock) { + HttpPost request = new HttpPost(buildApiUrl(countryCode, "facts")); + // Directory likes to have its data wrapped in a map with key "entities". + Map body = new HashMap(); + body.put("entities", factTablesBlock); + String jsonBody = gson.toJson(body); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + request.setEntity(new StringEntity(jsonBody, UTF_8)); + return request; + } + + /** + * Deletes existing star models from the Directory service for each of the collection IDs in the supplied StarModelInputData object. + * + * @param starModelInputData The input data for deleting existing star models. + * @return An OperationOutcome indicating the success or failure of the deletion. + */ + private OperationOutcome deleteStarModel(StarModelData starModelInputData) { + String apiUrl = buildApiUrl(starModelInputData.getCountryCode(), "facts"); + + if (mockDirectory) + // Dummy return if we're in mock mode + return new OperationOutcome(); + + try { + for (String collectionId: starModelInputData.getInputCollectionIds()) { + List factIds; + // Loop until no more facts are left in the Directory. + // We need to do things this way, because the Directory implements paging + // and a single pass may not get all facts. + do { + // First get a list of fact IDs for this collection + Map factWrapper = fetchFactWrapperByCollection(apiUrl, collectionId); + if (factWrapper == null) + return error("deleteStarModel: Problem getting facts for collection, factWrapper == null, collectionId=", collectionId); + if (!factWrapper.containsKey("items")) + return error("deleteStarModel: Problem getting facts for collection, no item key present: ", collectionId); + List> facts = (List>) factWrapper.get("items"); + if (facts.size() == 0) + break; + factIds = facts.stream() + .map(map -> map.get("id")) + .collect(Collectors.toList()); + + // Take the list of fact IDs and delete all of the corresponding facts + // at the Directory. + OperationOutcome deleteOutcome = deleteFactsByIds(apiUrl, factIds); + if (deleteOutcome.getIssue().size() > 0) + return deleteOutcome; + } while (true); + } + } catch(Exception e) { + return error("deleteStarModel: Exception during delete", Util.traceFromException(e)); + } + + return new OperationOutcome(); + } + + /** + * Fetches the fact wrapper object by collection from the Directory service. + * + * @param apiUrl The base URL for the Directory API. + * @param collectionId The ID of the collection for which to fetch the fact wrapper. + * @return A Map representing the fact wrapper retrieved from the Directory service. + */ + public Map fetchFactWrapperByCollection(String apiUrl, String collectionId) { + Map body = null; + try { + HttpGet request = fetchFactWrapperByCollectionRequest(apiUrl, collectionId); + + CloseableHttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() < 300) { + HttpEntity httpEntity = response.getEntity(); + String json = EntityUtils.toString(httpEntity); + body = gson.fromJson(json, Map.class); + } else + logger.warn("fetchFactWrapperByCollection: entity get HTTP error: " + Integer.toString(response.getStatusLine().getStatusCode()) + ", apiUrl=" + apiUrl + ", collectionId=" + collectionId); + } catch (IOException e) { + logger.warn("fetchFactWrapperByCollection: entity get exception: " + Util.traceFromException(e)); + } catch (Exception e) { + logger.warn("fetchFactWrapperByCollection: unknown exception: " + Util.traceFromException(e)); + } + + return body; + } + + /** + * Constructs an HTTP GET request for fetching the fact wrapper object by collection from the Directory service. + * + * @param apiUrl The base URL for the Directory API. + * @param collectionId The ID of the collection for which to fetch the fact wrapper. + * @return An HttpGet request object. + */ + private HttpGet fetchFactWrapperByCollectionRequest(String apiUrl, String collectionId) { + String url = apiUrl + "?q=collection==%22" + collectionId + "%22"; + logger.info("fetchFactWrapperByCollectionRequest: url=" + url); + HttpGet request = new HttpGet(url); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + return request; + } + + public void runTestQuery() { + try { + String url = "https://bbmritestnn.gcc.rug.nl/api/v2/eu_bbmri_eric_DE_collections?q=id==%22bbmri-eric:ID:DE_DKFZ_TEST:collection:Test1%22"; + logger.info("runTestQuery: url=" + url); + HttpGet request = new HttpGet(url); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + + CloseableHttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() < 300) { + HttpEntity httpEntity = response.getEntity(); + String json = EntityUtils.toString(httpEntity); + logger.info("runTestQuery: SUCCESS, json=" + json); + } else + logger.warn("runTestQuery: FAILURE, entity get HTTP error: " + Integer.toString(response.getStatusLine().getStatusCode())); + } catch (IOException e) { + logger.warn("runTestQuery: FAILURE, entity get exception: " + Util.traceFromException(e)); + } catch (Exception e) { + logger.warn("runTestQuery: FAILURE, unknown exception: " + Util.traceFromException(e)); + } + } + + /** + * Deletes facts from the Directory service based on a list of fact IDs. + * + * @param apiUrl The base URL for the Directory API. + * @param factIds The list of fact IDs to be deleted. + * @return An OperationOutcome indicating the success or failure of the deletion. + */ + public OperationOutcome deleteFactsByIds(String apiUrl, List factIds) { + if (factIds.size() == 0) + // Nothing to delete + return new OperationOutcome(); + + HttpDeleteWithBody request = deleteFactsByIdsRequest(apiUrl, factIds); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + if (response.getStatusLine().getStatusCode() < 300) { + return new OperationOutcome(); + } else { + return error("entity delete status code " + response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), UTF_8)); + } + } catch (IOException e) { + return error("entity delete exception", e.getMessage()); + } + } + + /** + * Constructs an HTTP DELETE request with a request body for deleting facts by IDs from the Directory service. + * + * @param apiUrl The base URL for the Directory API. + * @param factIds The list of fact IDs to be deleted. + * @return An HttpDeleteWithBody request object. + */ + private HttpDeleteWithBody deleteFactsByIdsRequest(String apiUrl, List factIds) { + HttpDeleteWithBody request = new HttpDeleteWithBody(apiUrl); + // Directory likes to have its delete data wrapped in a map with key "entityIds". + Map body = new HashMap(); + body.put("entityIds", factIds); + String jsonBody = gson.toJson(body); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + request.setEntity(new StringEntity(jsonBody, UTF_8)); + return request; + } + + /** + * Custom HTTP DELETE request with a request body support. + * Used for sending delete requests with a request body to the Directory service. + */ + class HttpDeleteWithBody extends HttpEntityEnclosingRequestBase { + public static final String METHOD_NAME = "DELETE"; + + public HttpDeleteWithBody() { + super(); + } + + public HttpDeleteWithBody(final URI uri) { + super(); + setURI(uri); + } + + public HttpDeleteWithBody(final String uri) { + super(); + setURI(URI.create(uri)); + } + + @Override + public String getMethod() { + return METHOD_NAME; + } + } + + /** + * Collects diagnosis corrections from the Directory. + * + * It checks with the Directory if the diagnosis codes are valid ICD values and corrects them if necessary. + * + * Two levels of correction are possible: + * + * 1. If the full code is not correct, remove the number after the period and try again. If the new truncated code is OK, use it to replace the existing diagnosis. + * 2. If that doesn't work, replace the existing diagnosis with null. + * + * @param diagnoses A string map containing diagnoses to be corrected. + */ + public void collectDiagnosisCorrections(Map diagnoses) { + int diagnosisCounter = 0; // for diagnostics only + int invalidIcdValueCounter = 0; + int correctedIcdValueCounter = 0; + for (String diagnosis: diagnoses.keySet()) { + if (diagnosisCounter%500 == 0) + logger.info("__________ collectDiagnosisCorrections: diagnosisCounter: " + diagnosisCounter + ", total diagnoses: " + diagnoses.size()); + if (!isValidIcdValue(diagnosis)) { + invalidIcdValueCounter++; + String diagnosisCategory = diagnosis.split("\\.")[0]; + if (isValidIcdValue(diagnosisCategory)) { + correctedIcdValueCounter++; + diagnoses.put(diagnosis, diagnosisCategory); + } else + diagnoses.put(diagnosis, null); + } + diagnosisCounter++; + } + + logger.info("__________ collectDiagnosisCorrections: invalidIcdValueCounter: " + invalidIcdValueCounter + ", correctedIcdValueCounter: " + correctedIcdValueCounter); + } + + /** + * Checks if a given diagnosis code is a valid ICD value by querying the Directory service. + * + * @param diagnosis The diagnosis code to be validated. + * @return true if the diagnosis code is a valid ICD value, false if not, or if an error condition was encountered. + */ + private boolean isValidIcdValue(String diagnosis) { + String url = baseUrl + "/api/v2/eu_bbmri_eric_disease_types?q=id=='" + diagnosis + "'"; + try { + HttpGet request = isValidIcdValueRequest(url); + CloseableHttpResponse response = httpClient.execute(request); + if (response.getStatusLine().getStatusCode() < 300) { + HttpEntity httpEntity = response.getEntity(); + String json = EntityUtils.toString(httpEntity); + Map body = gson.fromJson(json, Map.class); + if (body.containsKey("total")) { + Object total = body.get("total"); + if (total instanceof Double) { + Integer intTotal = ((Double) total).intValue(); + if (intTotal > 0) + return true; + } + } + } else + logger.warn("ICD validation get HTTP error; " + Integer.toString(response.getStatusLine().getStatusCode())); + } catch (IOException e) { + logger.warn("ICD validation get exception: " + Util.traceFromException(e)); + } catch (Exception e) { + logger.warn("ICD validation, unknown exception: " + Util.traceFromException(e)); + } + + return false; + } + + /** + * Constructs an HTTP GET request for validating an ICD value against the Directory service. + * + * @param url The URL for validating the ICD value. + * @return An HttpGet request object. + */ + private HttpGet isValidIcdValueRequest(String url) { + HttpGet request = new HttpGet(url); + request.setHeader("x-molgenis-token", token); + request.setHeader("Accept", "application/json"); + request.setHeader("Content-type", "application/json"); + return request; + } + + private String buildCollectionApiUrl(String countryCode) { + return buildApiUrl(countryCode, "collections"); + } + + /** + * Create a URL for a specific Directory API endpoint. + * + * @param countryCode a code such as "DE" specifying the country the URL should address. May be null. + * @param function specifies the type of the endpoint, e.g. "collections". + * @return + */ + private String buildApiUrl(String countryCode, String function) { + String countryCodeInsert = ""; + if (countryCode != null && !countryCode.isEmpty()) + countryCodeInsert = countryCode + "_"; + String collectionApiUrl = baseUrl + "/api/v2/eu_bbmri_eric_" + countryCodeInsert + function; + + return collectionApiUrl; + } + + private Either> fetchIdItems(HttpGet request, String action) { + try (CloseableHttpResponse response = httpClient.execute(request)) { + if (response.getStatusLine().getStatusCode() == 200) { + return Either.right(decodeIdItems(response)); + } else { + return Either.left(error(action, EntityUtils.toString(response.getEntity(), UTF_8))); + } + } catch (IOException e) { + return Either.left(error(action, e.getMessage())); + } + } + + private ItemsDto decodeIdItems(CloseableHttpResponse response) throws IOException { + String payload = EntityUtils.toString(response.getEntity(), UTF_8); + return gson.fromJson(payload, new TypeToken>() { + }.getType()); + } + + static class LoginCredentials { + + String username, password; + + LoginCredentials(String username, String password) { + this.username = username; + this.password = password; + } + } + + static class LoginResponse { + + String username, token; + + LoginResponse() { + } + } + + private static class EntitiesDto { + + public EntitiesDto(List entities) { + this.entities = entities; + } + + List entities; + } + + static class CollectionSizeDto { + + private final String id; + private final int size; + + public CollectionSizeDto(BbmriEricId id, int size) { + this.id = id.toString(); + this.size = size; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CollectionSizeDto that = (CollectionSizeDto) o; + return size == that.size && id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id, size); + } + } + + private static class ItemsDto { + + List items; + } + + private static class IdDto { + + String id; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/DirectoryService.java b/src/main/java/de/samply/directory_sync_service/directory/DirectoryService.java new file mode 100644 index 0000000..b20ca06 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/DirectoryService.java @@ -0,0 +1,75 @@ +package de.samply.directory_sync_service.directory; + +import de.samply.directory_sync_service.directory.DirectoryApi.CollectionSizeDto; +import de.samply.directory_sync_service.directory.model.BbmriEricId; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionGet; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionPut; +import de.samply.directory_sync_service.model.StarModelData; + +import io.vavr.control.Either; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.hl7.fhir.r4.model.OperationOutcome; + +public class DirectoryService { + + private DirectoryApi api; + + public DirectoryService(DirectoryApi api) { + this.api = Objects.requireNonNull(api); + } + + public void setApi(DirectoryApi api) { + this.api = api; + } + + public List updateCollectionSizes(Map collectionSizes) { + return groupCollectionSizesByCountryCode(collectionSizes) + .entrySet().stream() + .map(entry -> updateCollectionSizes(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } + + private Map>> groupCollectionSizesByCountryCode( + Map collectionSizes) { + return collectionSizes.entrySet().stream() + .collect(Collectors.groupingBy(e -> e.getKey().getCountryCode())); + } + + public OperationOutcome updateCollectionSizes(String countryCode, + List> collectionSizes) { + + Either> result = api.listAllCollectionIds(countryCode); + if (result.isLeft()) { + return result.getLeft(); + } + + Set existingCollectionIds = result.get(); + + List collectionSizeDtos = collectionSizes.stream() + .filter(e -> existingCollectionIds.contains(e.getKey())) + .map(e -> new CollectionSizeDto(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + + return api.updateCollectionSizes(countryCode, collectionSizeDtos); + } + + public List updateEntities(DirectoryCollectionPut directoryCollectionPut) { + OperationOutcome operationOutcome = api.updateEntities(directoryCollectionPut); + return Collections.singletonList(operationOutcome); + } + + public Either fetchDirectoryCollectionGetOutcomes(String countryCode, List collectionIds) { + return(api.fetchCollectionGetOutcomes(countryCode, collectionIds)); + } + + public List updateStarModel(StarModelData starModelInputData) { + OperationOutcome operationOutcome = api.updateStarModel(starModelInputData); + return Collections.singletonList(operationOutcome); + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/MergeDirectoryCollectionGetToDirectoryCollectionPut.java b/src/main/java/de/samply/directory_sync_service/directory/MergeDirectoryCollectionGetToDirectoryCollectionPut.java new file mode 100644 index 0000000..1497481 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/MergeDirectoryCollectionGetToDirectoryCollectionPut.java @@ -0,0 +1,60 @@ +package de.samply.directory_sync_service.directory; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionGet; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionPut; + +/** + * Takes information from a DirectoryCollectionGet object and inserts it into + * a preexisting DirectoryCollectionPut object. + * + * Both objects should contain lists of collections with identical IDs. + */ +public class MergeDirectoryCollectionGetToDirectoryCollectionPut { + private static final Logger logger = LoggerFactory.getLogger(MergeDirectoryCollectionGetToDirectoryCollectionPut.class); + + /** + * Merges collection information from a DirectoryCollectionGet object into a + * DirectoryCollectionPut object. + * + * Returns false if there is a problem, e.g. if there are discrepancies between + * the collection IDs in the two objects. + * + * @param directoryCollectionGet + * @param directoryCollectionPut + * @return + */ + public static boolean merge(DirectoryCollectionGet directoryCollectionGet, DirectoryCollectionPut directoryCollectionPut) { + List collectionIds = directoryCollectionPut.getCollectionIds(); + // Only do a merge if we are not mocking + if (!directoryCollectionGet.isMockDirectory()) + for (String collectionId: collectionIds) + if (merge(collectionId, directoryCollectionGet, directoryCollectionPut) == null) + return false; + + return true; + } + + private static DirectoryCollectionPut merge(String collectionId, DirectoryCollectionGet directoryCollectionGet, DirectoryCollectionPut directoryCollectionPut) { + try { + directoryCollectionPut.setName(collectionId, directoryCollectionGet.getName(collectionId)); + directoryCollectionPut.setDescription(collectionId, directoryCollectionGet.getDescription(collectionId)); + directoryCollectionPut.setContact(collectionId, directoryCollectionGet.getContactId(collectionId)); + directoryCollectionPut.setCountry(collectionId, directoryCollectionGet.getCountryId(collectionId)); + directoryCollectionPut.setBiobank(collectionId, directoryCollectionGet.getBiobankId(collectionId)); + directoryCollectionPut.setType(collectionId, directoryCollectionGet.getTypeIds(collectionId)); + directoryCollectionPut.setDataCategories(collectionId, directoryCollectionGet.getDataCategoryIds(collectionId)); + directoryCollectionPut.setNetworks(collectionId, directoryCollectionGet.getNetworkIds(collectionId)); + } catch(Exception e) { + logger.error("Problem merging DirectoryCollectionGet into DirectoryCollectionPut. " + Util.traceFromException(e)); + return null; + } + + return directoryCollectionPut; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/model/BbmriEricId.java b/src/main/java/de/samply/directory_sync_service/directory/model/BbmriEricId.java new file mode 100644 index 0000000..793790f --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/model/BbmriEricId.java @@ -0,0 +1,82 @@ +package de.samply.directory_sync_service.directory.model; + +import de.samply.directory_sync_service.service.DirectorySync; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class represents a BBMRI-ERIC identifier which has the following form: + *

+ * {@literal bbmri-eric:ID:_} + */ +public class BbmriEricId { + private static Logger logger = LogManager.getLogger(BbmriEricId.class); + + private static final Pattern PATTERN = Pattern.compile("bbmri-eric:ID:([a-zA-Z]{2})(_.+)"); + + private final String countryCode; + private final String suffix; + + private BbmriEricId(String countryCode, String suffix) { + if (countryCode != null) + countryCode = countryCode.toUpperCase(); + this.countryCode = Objects.requireNonNull(countryCode); + this.suffix = Objects.requireNonNull(suffix); + } + + /** + * Returns the two-letter upper-case country code of this identifier. + * + * @return the two-letter upper-case country code of this identifier. + */ + public String getCountryCode() { + return countryCode; + } + + /** + * Tries to create a BBMRI-ERIC identifier from string. + * + * @param s the string to parse. + * @return a BBMRI-ERIC identifier or {@link Optional#empty empty} if {@code s} doesn't represent + * a valid BBMRI-ERIC identifier + */ + public static Optional valueOf(String s) { + if (s == null) { + logger.info("valueOf: input is null, cannot determine an ID"); + return Optional.empty(); + } + Matcher matcher = PATTERN.matcher(s); + if (!matcher.matches()) { + logger.info("valueOf: input doesnt match BBMRI ID pattern, cannot determine an ID"); + return Optional.empty(); + } + return Optional.of(new BbmriEricId(matcher.group(1), matcher.group(2))); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BbmriEricId that = (BbmriEricId) o; + return countryCode.equals(that.countryCode) && suffix.equals(that.suffix); + } + + @Override + public int hashCode() { + return Objects.hash(countryCode, suffix); + } + + @Override + public String toString() { + return "bbmri-eric:ID:" + countryCode + suffix; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/model/Biobank.java b/src/main/java/de/samply/directory_sync_service/directory/model/Biobank.java new file mode 100644 index 0000000..9a95d1f --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/model/Biobank.java @@ -0,0 +1,198 @@ +package de.samply.directory_sync_service.directory.model; + +//TODO Add other relevant attributes +public class Biobank { + + String id; + + String name; + + String acronym; + + String description; + + String url; + + String juridical_person; + + boolean it_support_available; + + int it_staff_site; + + boolean is_available; + + boolean partner_charter_signed; + + String head_firstname; + + String head_lastname; + + String head_role; + + String latitude; + + String longitude; + + String[] also_known; + + boolean collaboration_commercial; + + boolean collaboration_non_for_profit; + + public Biobank() { + + } + + @SuppressWarnings("OptionalGetWithoutIsPresent") + public BbmriEricId getId() { + return BbmriEricId.valueOf(id).get(); + } + + public void setId(BbmriEricId id) { + this.id = id.toString(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAcronym() { + return acronym; + } + + public void setAcronym(String acronym) { + this.acronym = acronym; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getJuridical_person() { + return juridical_person; + } + + public void setJuridical_person(String juridical_person) { + this.juridical_person = juridical_person; + } + + public boolean isIt_support_available() { + return it_support_available; + } + + public void setIt_support_available(boolean it_support_available) { + this.it_support_available = it_support_available; + } + + public int getIt_staff_site() { + return it_staff_site; + } + + public void setIt_staff_site(int it_staff_site) { + this.it_staff_site = it_staff_site; + } + + public boolean isIs_available() { + return is_available; + } + + public void setIs_available(boolean is_available) { + this.is_available = is_available; + } + + public boolean isPartner_charter_signed() { + return partner_charter_signed; + } + + public void setPartner_charter_signed(boolean partner_charter_signed) { + this.partner_charter_signed = partner_charter_signed; + } + + public String getHead_firstname() { + return head_firstname; + } + + public void setHead_firstname(String head_firstname) { + this.head_firstname = head_firstname; + } + + public String getHead_lastname() { + return head_lastname; + } + + public void setHead_lastname(String head_lastname) { + this.head_lastname = head_lastname; + } + + public String getHead_role() { + return head_role; + } + + public void setHead_role(String head_role) { + this.head_role = head_role; + } + + public String getLatitude() { + return latitude; + } + + public void setLatitude(String latitude) { + this.latitude = latitude; + } + + public String getLongitude() { + return longitude; + } + + public void setLongitude(String longitude) { + this.longitude = longitude; + } + + public String[] getAlso_known() { + return also_known; + } + + public void setAlso_known(String[] also_known) { + this.also_known = also_known; + } + + public boolean isCollaboration_commercial() { + return collaboration_commercial; + } + + public void setCollaboration_commercial(boolean collaboration_commercial) { + this.collaboration_commercial = collaboration_commercial; + } + + public boolean isCollaboration_non_for_profit() { + return collaboration_non_for_profit; + } + + public void setCollaboration_non_for_profit(boolean collaboration_non_for_profit) { + this.collaboration_non_for_profit = collaboration_non_for_profit; + } + + @Override + public String toString() { + return "Biobank{" + + "id='" + id + '\'' + + ", name='" + name + '\'' + + '}'; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/model/Collection.java b/src/main/java/de/samply/directory_sync_service/directory/model/Collection.java new file mode 100644 index 0000000..1674535 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/model/Collection.java @@ -0,0 +1,6 @@ +package de.samply.directory_sync_service.directory.model; + +public class Collection { + + +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/model/CollectionBundle.java b/src/main/java/de/samply/directory_sync_service/directory/model/CollectionBundle.java new file mode 100644 index 0000000..aa5c622 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/model/CollectionBundle.java @@ -0,0 +1,13 @@ +package de.samply.directory_sync_service.directory.model; + +import java.util.List; + +public class CollectionBundle { + + int total; + + List items; + + public CollectionBundle() { + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionGet.java b/src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionGet.java new file mode 100644 index 0000000..7f31735 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionGet.java @@ -0,0 +1,133 @@ +package de.samply.directory_sync_service.directory.model; + +import java.util.Map; +import java.util.HashMap; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a data transfer object that maps onto the JSON returned by a GET request + * to the Directory API when you want to obtain information about collections. + * + * It simply extends a Map and adds a single key, "items". This contains a list + * of collections. Each collection is also a Map, with keys corresponding to the + * various attributes needed when updating, such as collection name or ID. + * + * The getter methods allow you to get attributes in collections identified by + * collection ID. If you use an ID that is not known, you will get a null pointer + * exception. + */ +public class DirectoryCollectionGet extends HashMap { + private static final Logger logger = LoggerFactory.getLogger(DirectoryCollectionGet.class); + private boolean mockDirectory = false; + + public void setMockDirectory(boolean mockDirectory) { + this.mockDirectory = mockDirectory; + } + + public boolean isMockDirectory() { + return mockDirectory; + } + + public void init() { + put("items", new ArrayList()); + } + + public String getCountryId(String id) { + return (String) ((Map) getItem(id).get("country")).get("id"); + } + + public String getContactId(String id) { + return (String) ((Map) getItem(id).get("contact")).get("id"); + } + + public String getBiobankId(String id) { + return (String) ((Map) getItem(id).get("biobank")).get("id"); + } + + public List getTypeIds(String id) { + Map item = getItem(id); + List> types = (List>) item.get("type"); + List typeLabels = new ArrayList(); + for (Map type: types) + typeLabels.add((String) type.get("id")); + + return typeLabels; + } + + public List getDataCategoryIds(String id) { + Map item = getItem(id); + List> dataCategories = (List>) item.get("data_categories"); + List dataCategoryLabels = new ArrayList(); + for (Map type: dataCategories) + dataCategoryLabels.add((String) type.get("id")); + + return dataCategoryLabels; + } + + public List getNetworkIds(String id) { + Map item = getItem(id); + List> networks = (List>) item.get("network"); + List networkLabels = new ArrayList(); + for (Map type: networks) + networkLabels.add((String) type.get("id")); + + return networkLabels; + } + + public String getName(String id) { + return (String) getItem(id).get("name"); + } + + public String getDescription(String id) { + return (String) getItem(id).get("description"); + } + + public List getCollectionIds() { + return getItems().stream() + .map(entity -> (String) entity.get("id")) + .collect(Collectors.toList()); + } + + public List getItems() { + if (!this.containsKey("items")) { + logger.warn("DirectoryCollectionGet.getItems: no items key, aborting"); + return null; + } + return (List) get("items"); + } + + public Map getItemZero() { + if (!containsKey("items")) + return null; + List itemList = (List) get("items"); + if (itemList == null || itemList.size() == 0) + return null; + return itemList.get(0); + } + + private Map getItem(String id) { + Map item = null; + + List items = getItems(); + if (items == null) + return null; + + for (Map e: items) { + if (e == null) { + logger.warn("DirectoryCollectionGet.getItem: problem with getItems()"); + continue; + } + if (e.get("id").equals(id)) { + item = e; + break; + } + } + + return item; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionPut.java b/src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionPut.java new file mode 100644 index 0000000..766bef7 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/directory/model/DirectoryCollectionPut.java @@ -0,0 +1,378 @@ +package de.samply.directory_sync_service.directory.model; + +import java.util.HashMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +import com.google.gson.Gson; +import de.samply.directory_sync_service.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a data transfer object that maps onto the JSON needed for a PUT request + * to the Directory API when you want to update one or more collections. + * + * It simply extends a Map and adds a single key, "entities". This contains a list + * of collections. Each collection is also a Map, with keys corresponding to the + * various attributes needed when updating, such as collection name or ID. + * + * The setter methods allow you to set attributes in collections identified by + * collection ID. If you use an ID that is not yet known, a new collection with this + * ID will first be created. + */ +public class DirectoryCollectionPut extends HashMap { + private static final Logger logger = LoggerFactory.getLogger(DirectoryCollectionPut.class); + + public DirectoryCollectionPut() { + // Initializes the list of entities. + this.put("entities", new ArrayList()); + } + + private final Gson gson = new Gson(); + + public void setCountry(String collectionId, String country) { + getEntity(collectionId).setCountry(country); + } + + public void setName(String collectionId, String name) { + getEntity(collectionId).setName(name); + } + + public void setDescription(String collectionId, String description) { + getEntity(collectionId).setDescription(description); + } + + public void setContact(String collectionId, String contact) { + getEntity(collectionId).setContact(contact); + } + + public void setBiobank(String collectionId, String biobank) { + getEntity(collectionId).setBiobank(biobank); + } + + public void setSize(String collectionId, Integer size) { + getEntity(collectionId).setSize(size); + } + + public void setOrderOfMagnitude(String collectionId, Integer size) { + getEntity(collectionId).setOrderOfMagnitude(size); + } + + public void setNumberOfDonors(String collectionId, Integer size) { + getEntity(collectionId).setNumberOfDonors(size); + } + + public void setOrderOfMagnitudeDonors(String collectionId, Integer size) { + getEntity(collectionId).setOrderOfMagnitudeDonors(size); + } + + public void setType(String collectionId, List type) { + getEntity(collectionId).setType(type); + } + + public void setDataCategories(String collectionId, List dataCategories) { + getEntity(collectionId).setDataCategories(dataCategories); + } + + public void setNetworks(String collectionId, List networks) { + getEntity(collectionId).setNetworks(networks); + } + + public void setSex(String collectionId, List sex) { + getEntity(collectionId).setSex(sex); + } + + public void setAgeLow(String collectionId, Integer value) { + getEntity(collectionId).setAgeLow(value); + } + + public void setAgeHigh(String collectionId, Integer value) { + getEntity(collectionId).setAgeHigh(value); + } + + public void setMaterials(String collectionId, List value) { + getEntity(collectionId).setMaterials(value); + } + + public void setStorageTemperatures(String collectionId, List value) { + getEntity(collectionId).setStorageTemperatures(value); + } + + public void setDiagnosisAvailable(String collectionId, List value) { + getEntity(collectionId).setDiagnosisAvailable(value); + } + + public List getCollectionIds() { + return getEntities().stream() + .map(entity -> (String) entity.get("id")) + .collect(Collectors.toList()); + } + + /** + * Gets the country code for the collections, e.g. "DE". + * + * Assumes that all collections will have the same code and simply returns + * the code of the first collection. + * + * If there are no collections, returns null. + * + * May throw a null pointer exception. + * + * @return Country code + */ + public String getCountryCode() { + logger.info("getCountryCode: ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ entered"); + String countryCode = null; + try { + List entities = getEntities(); + if (entities == null || entities.size() == 0) + return null; + logger.info("getCountryCode: entities.size: " + entities.size()); + Entity entity = entities.get(0); + logger.info("getCountryCode: entity: " + gson.toJson(entity)); + countryCode = entity.getCountry(); + if (countryCode == null || countryCode.isEmpty()) { + logger.info("getCountryCode: countryCode from first entity is null or empty"); + String entityId = entity.getId(); + logger.info("getCountryCode: entityId: " + entityId); + Optional bbmriEricId = BbmriEricId.valueOf(entityId); + logger.info("getCountryCode: bbmriEricId: " + bbmriEricId); + countryCode = bbmriEricId.orElse(null).getCountryCode(); + } + } catch (Exception e) { + logger.info("getCountryCode: exception: " + Util.traceFromException(e)); + return null; + } + + logger.info("getCountryCode: countryCode: " + countryCode); + return countryCode; + } + + private List getEntities() { + return (List) get("entities"); + } + + /** + * Retrieves or creates an Entity with the specified collection ID. + * + * This method searches through the existing entities to find one with a matching + * ID. If found, the existing entity is returned; otherwise, a new Entity is created + * with the given collection ID and added to the list of entities. + * + * @param collectionId The unique identifier for the Entity. + * @return The Entity with the specified collection ID. If not found, a new Entity + * is created and returned. + */ + private Entity getEntity(String collectionId) { + Entity entity = null; + + for (Entity e: getEntities()) + if (e.get("id").equals(collectionId)) { + entity = e; + break; + } + + if (entity == null) { + entity = new Entity(collectionId); + this.getEntities().add(entity); + } + + return entity; + } + + /** + * Represents an entity with attributes related to a collection. + * This class extends HashMap to store key-value pairs. + */ + public class Entity extends HashMap { + /** + * Constructs an Entity with the specified collection ID. + * + * @param collectionId The unique identifier for the Entity. + */ + public Entity(String collectionId) { + setId(collectionId); + } + + /** + * Sets the collection ID for the Entity. + * + * @param collectionId The unique identifier for the Entity. + */ + public void setId(String collectionId) { + put("id", collectionId); + setTimestamp(); + } + + public void setTimestamp() { + LocalDateTime dateTime = LocalDateTime.now(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + String formattedDateTime = dateTime.format(formatter); + + put("timestamp", formattedDateTime); + } + + /** + * Retrieves the collection ID of the Entity. + * + * @return The collection ID. + */ + public String getId() { + return (String) get("id"); + } + + public void setCountry(String country) { + if (country == null || country.isEmpty()) + return; + + put("country", country); + } + + public String getCountry() { + return (String) get("country"); + } + + public void setName(String name) { + if (name == null || name.isEmpty()) + return; + + put("name", name); + } + + public void setDescription(String description) { + if (description == null || description.isEmpty()) + return; + + put("description", description); + } + + public void setContact(String contact) { + if (contact == null || contact.isEmpty()) + return; + + put("contact", contact); + } + + public void setBiobank(String biobank) { + if (biobank == null || biobank.isEmpty()) + return; + + put("biobank", biobank); + } + + public void setSize(Integer size) { + if (size == null) + return; + + put("size", size); + } + + public void setOrderOfMagnitude(Integer orderOfMagnitude) { + if (orderOfMagnitude == null) + return; + + put("order_of_magnitude", orderOfMagnitude); + } + + public void setNumberOfDonors(Integer size) { + if (size == null) + return; + + put("number_of_donors", size); + } + + public void setOrderOfMagnitudeDonors(Integer orderOfMagnitude) { + if (orderOfMagnitude == null) + return; + + put("order_of_magnitude_donors", orderOfMagnitude); + } + + public void setType(List type) { + if (type == null) + type = new ArrayList(); + + put("type", type); + } + + public void setDataCategories(List dataCategories) { + if (dataCategories == null) + dataCategories = new ArrayList(); + + put("data_categories", dataCategories); + } + + public void setNetworks(List networks) { + if (networks == null) + networks = new ArrayList(); + + put("network", networks); + } + + public void setSex(List sex) { + if (sex == null) + sex = new ArrayList(); + + put("sex", sex); + } + + public void setAgeLow(Integer value) { + put("age_low", value); + } + + public void setAgeHigh(Integer value) { + put("age_high", value); + } + + public void setMaterials(List materials) { + if (materials == null) + materials = new ArrayList(); + + put("materials", materials); + } + + public void setStorageTemperatures(List storageTemperatures) { + if (storageTemperatures == null) + storageTemperatures = new ArrayList(); + + put("storage_temperatures", storageTemperatures); + } + + public void setDiagnosisAvailable(List diagnoses) { + if (diagnoses == null) + diagnoses = new ArrayList(); + + put("diagnosis_available", diagnoses); + } + + public List getDiagnosisAvailable() { + return (List) get("diagnosis_available"); + } + } + + /** + * Applies corrections to the available diagnoses of each Entity based on a provided map. + * The method iterates through the list of entities and updates the available diagnoses + * using the provided map of corrections. + * + * @param correctedDiagnoses A map containing diagnosis corrections, where the keys + * represent the original diagnoses and the values represent + * the corrected diagnoses. + */ + public void applyDiagnosisCorrections(Map correctedDiagnoses) { + for (Entity entity: getEntities()) { + List directoryDiagnoses = entity.getDiagnosisAvailable().stream() + .filter(diagnosis -> diagnosis != null && correctedDiagnoses.containsKey(diagnosis) && correctedDiagnoses.get(diagnosis) != null) + .map(correctedDiagnoses::get) + .distinct() + .collect(Collectors.toList()); + entity.setDiagnosisAvailable(directoryDiagnoses); + } + } +} diff --git a/src/main/java/de/samply/directory_sync_service/fhir/FhirApi.java b/src/main/java/de/samply/directory_sync_service/fhir/FhirApi.java new file mode 100644 index 0000000..99c2931 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/fhir/FhirApi.java @@ -0,0 +1,647 @@ +package de.samply.directory_sync_service.fhir; + +import static ca.uhn.fhir.rest.api.PreferReturnEnum.OPERATION_OUTCOME; +import static ca.uhn.fhir.rest.api.SummaryEnum.COUNT; +import static java.util.Collections.emptyList; +import static org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity.ERROR; + +import ca.uhn.fhir.rest.api.MethodOutcome; +import ca.uhn.fhir.rest.api.SummaryEnum; +import ca.uhn.fhir.rest.client.api.IGenericClient; +import ca.uhn.fhir.rest.client.exceptions.FhirClientConnectionException; +import ca.uhn.fhir.rest.gclient.ICreateTyped; +import ca.uhn.fhir.rest.gclient.IQuery; +import ca.uhn.fhir.rest.gclient.IUpdateExecutable; +import ca.uhn.fhir.rest.gclient.UriClientParam; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.directory.model.BbmriEricId; +import io.vavr.control.Either; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.Map; +import java.util.HashMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.function.Predicate; +import java.util.function.Function; +import java.util.HashSet; + +import org.hl7.fhir.instance.model.api.IBaseBundle; +import org.hl7.fhir.instance.model.api.IBaseOperationOutcome; +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.DateType; +import org.hl7.fhir.r4.model.Extension; +import org.hl7.fhir.r4.model.Identifier; +import org.hl7.fhir.r4.model.Measure; +import org.hl7.fhir.r4.model.MeasureReport; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Organization; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.ResourceType; +import org.hl7.fhir.r4.model.Specimen; +import org.hl7.fhir.r4.model.StringType; +import org.hl7.fhir.r4.model.Reference; +import org.hl7.fhir.r4.model.Resource; +import org.hl7.fhir.r4.model.Condition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides convenience methods for selected FHIR operations. + */ +public class FhirApi { + + private static final String BIOBANK_PROFILE_URI = "https://fhir.bbmri.de/StructureDefinition/Biobank"; + private static final String COLLECTION_PROFILE_URI = "https://fhir.bbmri.de/StructureDefinition/Collection"; + private static final String SAMPLE_DIAGNOSIS_URI = "https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis"; + private static final String DEFAULT_COLLECTION_ID = "DEFAULT"; + + private static final Logger logger = LoggerFactory.getLogger(FhirApi.class); + + Map> specimensByCollection = null; + Map> patientsByCollection = null; + + /** + * Returns the BBMRI-ERIC identifier of {@code collection} if some valid one could be found. + * + * @param collection the Organization resource, possibly containing a BBMRI-ERIC identifier + * @return the found BBMRI-ERIC identifier or {@link Optional#empty empty} + */ + public static Optional bbmriEricId(Organization collection) { + return collection.getIdentifier().stream() + .filter(i -> "http://www.bbmri-eric.eu/".equals(i.getSystem())) + .findFirst().map(Identifier::getValue).flatMap(BbmriEricId::valueOf); + } + + private final IGenericClient fhirClient; + + public FhirApi(IGenericClient fhirClient) { + this.fhirClient = Objects.requireNonNull(fhirClient); + } + + public OperationOutcome updateResource(IBaseResource theResource) { + try { + logger.info("updateResource: run resourceUpdate"); + IUpdateExecutable resourceUpdater = resourceUpdate(theResource); + logger.info("updateResource: run getOperationOutcome"); + IBaseOperationOutcome outcome = resourceUpdater.execute().getOperationOutcome(); + return (OperationOutcome) outcome; +// return (OperationOutcome) resourceUpdate(theResource).execute().getOperationOutcome(); + } catch (Exception e) { + logger.info("updateResource: exception: " + Util.traceFromException(e)); + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR).setDiagnostics(e.getMessage()); + return outcome; + } + } + + private IUpdateExecutable resourceUpdate(IBaseResource theResource) { + return fhirClient.update().resource(theResource).prefer(OPERATION_OUTCOME); + } + + public Either createResource(IBaseResource resource) { + logger.info("createResource: entered"); + try { + MethodOutcome outcome = resourceCreate(resource).execute(); + if (outcome.getCreated()) { + return Either.right(null); + } else { + return Either.left("error while creating a resource"); + } + } catch (Exception e) { + return Either.left(e.getMessage()); + } + } + + private ICreateTyped resourceCreate(IBaseResource resource) { + logger.info("resourceCreate: entered"); + return fhirClient.create().resource(resource).prefer(OPERATION_OUTCOME); + } + + /** + * Lists all Organization resources with the biobank profile. + * + * @return either a list of {@link Organization} resources or an {@link OperationOutcome} on * + * errors + */ + public Either> listAllBiobanks() { + return listAllOrganizations(BIOBANK_PROFILE_URI) + .map(bundle -> extractOrganizations(bundle, BIOBANK_PROFILE_URI)); + } + + private Either listAllOrganizations(String profileUri) { + try { + return Either.right((Bundle) fhirClient.search().forResource(Organization.class) + .withProfile(profileUri).execute()); + } catch (Exception e) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR) + .setDiagnostics(e.getMessage()); + return Either.left(outcome); + } + } + + private static List extractOrganizations(Bundle bundle, String profileUrl) { + return bundle.getEntry().stream() + .map(BundleEntryComponent::getResource) + .filter(r -> r.getResourceType() == ResourceType.Organization) + .filter(r -> r.getMeta().hasProfile(profileUrl)) + .map(r -> (Organization) r) + .collect(Collectors.toList()); + } + + /** + * Lists all Organization resources with the collection profile. + * + * @return either a list of {@link Organization} resources or an {@link OperationOutcome} on + * errors + */ + public Either> listAllCollections() { + return listAllOrganizations(COLLECTION_PROFILE_URI) + .map(bundle -> extractOrganizations(bundle, COLLECTION_PROFILE_URI)); + } + + /** + * Checks whether a resource of {@code type} and canonical {@code uri} exists. + * + * @param type the resource type + * @param uri the canonical URI + * @return a Right with {@code true} if the resource exists or a Left in case of an error + */ + public Either resourceExists(Class type, String uri) { + logger.info("Check whether " + type + " with canonical URI " + uri + " exists."); + try { + return Either.right(resourceQuery(type, uri).execute().getTotal() == 1); + } catch (Exception e) { + logger.info("Problem running check"); + return Either.left(e.getMessage()); + } + } + + private IQuery resourceQuery(Class type, String uri) { + logger.info("resourceQuery: uri: " + uri); + return fhirClient.search().forResource(type) + .where(new UriClientParam("url").matches().value(uri)) + .summaryMode(COUNT) + .returnBundle(Bundle.class); + } + + /** + * Executes the Measure with the given canonical URL. + * + * @param url canonical URL of the Measure to be executed + * @return MeasureReport or OperationOutcome in case of error. + */ + Either evaluateMeasure(String url) { + // Create the input parameters to pass to the server + Parameters inParams = new Parameters(); + inParams.addParameter().setName("periodStart").setValue(new DateType("1900")); + inParams.addParameter().setName("periodEnd").setValue(new DateType("2100")); + inParams.addParameter().setName("measure").setValue(new StringType(url)); + + try { + Parameters outParams = fhirClient + .operation() + .onType(Measure.class) + .named("$evaluate-measure") + .withParameters(inParams) + .useHttpGet() + .execute(); + + return Either.right((MeasureReport) outParams.getParameter().get(0).getResource()); + } catch (Exception e) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR) + .setDiagnostics(e.getMessage()); + return Either.left(outcome); + } + } + + /** + * Loads the Organization resource for each of the FHIR ids given. + * + * @param ids logical ids of the Organization resources to load + * @return List of Organization Resources or OperationOutcome in case of failure. + */ + Either> fetchCollections(Set ids) { + if (ids.isEmpty()) { + return Either.right(emptyList()); + } + try { + Bundle response = (Bundle) fhirClient.search().forResource(Organization.class) + .where(Organization.RES_ID.exactly().codes(ids)).execute(); + + return Either.right(response.getEntry().stream() + .filter(e -> ResourceType.Organization == e.getResource().getResourceType()) + .map(e -> (Organization) e.getResource()) + .collect(Collectors.toList())); + } catch (Exception e) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR) + .setDiagnostics(e.getMessage()); + return Either.left(outcome); + } + } + + /** + * Fetches specimens from the FHIR server and groups them by their collection id. + * If no default collection id is provided, tries to find one from the available collections. + * If the default collection id is invalid or not found, removes the specimens without a collection id from the result. + * + * @param defaultBbmriEricCollectionId the default collection id supplied by the site, to be used for specimens without a collection id. May be null + * @return an Either object containing either a map of collection id to list of specimens, or an OperationOutcome object in case of an error + */ + public Either>> fetchSpecimensByCollection(BbmriEricId defaultBbmriEricCollectionId) { + logger.info("__________ fetchSpecimensByCollection: entered"); + + // This method is slow, so use cached value if available. + if (specimensByCollection != null) + return Either.right(specimensByCollection); + + logger.info("__________ fetchSpecimensByCollection: get specimens from FHIR store"); + + try { + specimensByCollection = getAllSpecimensAsMap(); + + logger.info("__________ fetchSpecimensByCollection: specimensByCollection size: " + specimensByCollection.size()); + + defaultBbmriEricCollectionId = determineDefaultCollectionId(defaultBbmriEricCollectionId, specimensByCollection); + + logger.info("__________ fetchSpecimensByCollection: defaultBbmriEricCollectionId: " + defaultBbmriEricCollectionId); + + // Remove specimens without a collection from specimensByCollection, but keep + // the relevant specimen list, just in case we have a valid default ID to + // associate with them. + List defaultCollection = specimensByCollection.remove(DEFAULT_COLLECTION_ID); + + if (defaultCollection == null) + logger.info("__________ fetchSpecimensByCollection: defaultCollection is null"); + else + logger.info("__________ fetchSpecimensByCollection: defaultCollection size: " + defaultCollection.size()); + + // Replace the DEFAULT_COLLECTION_ID key in specimensByCollection by a sensible collection ID, + // assuming, of course, that there were any specemins caregorized by DEFAULT_COLLECTION_ID. + if (defaultCollection != null && defaultCollection.size() != 0 && defaultBbmriEricCollectionId != null) { + logger.info("__________ fetchSpecimensByCollection: Replace the DEFAULT_COLLECTION_ID key"); + + specimensByCollection.put(defaultBbmriEricCollectionId.toString(), defaultCollection); + } + + logger.info("__________ fetchSpecimensByCollection: specimensByCollection size: " + specimensByCollection.size()); + + return Either.right(specimensByCollection); + } catch (Exception e) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(OperationOutcome.IssueSeverity.ERROR).setDiagnostics(Util.traceFromException(e)); + return Either.left(outcome); + } + } + + /** + * Retrieves all Specimens from the FHIR server and organizes them into a Map based on their Collection ID. + * + * @return A Map where keys are Collection IDs and values are Lists of Specimens associated with each Collection ID. + * @throws FhirClientConnectionException If there is an issue connecting to the FHIR server. + */ + private Map> getAllSpecimensAsMap() { + logger.info("__________ getAllSpecimensAsMap: entered"); + + Map> result = new HashMap>(); + + // Use ITransactionTyped instead of returnBundle(Bundle.class) + IQuery bundleTransaction = fhirClient.search().forResource(Specimen.class); + Bundle bundle = (Bundle) bundleTransaction.execute(); + + logger.info("__________ getAllSpecimensAsMap: gather specimens"); + + // Keep looping until the store has no more specimens. + // This gets around the page size limit of 50 that is imposed by the current implementation of Blaze. + do { + // Add entries to the result map + for (Bundle.BundleEntryComponent entry : bundle.getEntry()) { + Specimen specimen = (Specimen) entry.getResource(); + String collectionId = extractCollectionIdFromSpecimen(specimen); + if (!result.containsKey(collectionId)) + result.put(collectionId, new ArrayList<>()); + result.get(collectionId).add(specimen); + } + + logger.info("__________ getAllSpecimensAsMap: Added " + bundle.getEntry().size() + " entries to result, result size: " + result.size()); + + // Check if there are more pages + if (bundle.getLink(Bundle.LINK_NEXT) != null) + // Use ITransactionTyped to load the next page + bundle = fhirClient.loadPage().next(bundle).execute(); + else + bundle = null; + } while (bundle != null); + + logger.info("__________ getAllSpecimensAsMap: done"); + + return result; + } + + /** + * Fetches Patient resources from the FHIR server and groups them by their collection ID. + * Starts with the available specimens and uses Patient references to find the patients. + * Note that this approach means that Patients with no specimens will not be included. + * + * @param defaultCollectionId + * @return + */ + Either>> fetchPatientsByCollection(Map> specimensByCollection) { + // This method is slow, so use cached value if available. + if (patientsByCollection != null) + return Either.right(patientsByCollection); + + patientsByCollection = specimensByCollection.entrySet().stream() + .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey(), extractPatientListFromSpecimenList(entry.getValue()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) ; + + return Either.right(patientsByCollection); + } + + /** + * Distingushing function used to ensure that Patient objects do not get duplicated. + * Takes a function as argument and uses the return value of this function when + * making comparsons. + * + * @param keyExtractor + * @return + * @param + */ + public static Predicate distinctBy(Function keyExtractor) { + Set seen = new HashSet<>(); + return t -> seen.add(keyExtractor.apply(t)); + } + + /** + * Given a list of Specimen resources, returns a list of Patient resources derived from + * the subject references in the specimens. + * + * @param specimens + * @return + */ + private List extractPatientListFromSpecimenList(List specimens) { + List patients = specimens.stream() + // filter out specimens without a patient reference + .filter(specimen -> specimen.hasSubject()) + // Find a Patient object corresponding to the specimen's subject + .map(specimen -> extractPatientFromSpecimen(specimen)) + // Avoid duplicating the same patient + .filter(distinctBy(Patient::getId)) + // collect the patients into a new list + .collect(Collectors.toList()); + + return patients; + } + + /** + * Extracts a Patient resource from a Specimen resource. + * + * @param specimen a Specimen resource that contains a reference to a Patient resource + * @return a Patient resource that matches the reference in the Specimen resource, or null if not found + * @throws ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException if the FHIR server cannot find the Patient resource + */ + public Patient extractPatientFromSpecimen(Specimen specimen) { + return fhirClient + .read() + .resource(Patient.class) + .withId(specimen.getSubject() + .getReference() + .replaceFirst("Patient/", "")) + .execute(); + } + + Boolean conditionsPresentInFhirStore = null; + + /** + * Extracts a list of condition codes from a Patient resource using a FHIR client. + * The condition codes are based on the system "http://hl7.org/fhir/sid/icd-10". + * @param patient a Patient resource that has an ID element + * @return a list of strings that represent the condition codes of the patient, or an empty list if none are found + */ + public List extractConditionCodesFromPatient(Patient patient) { + List conditionCodes = new ArrayList(); + try { + // If there are no conditions in the FHIR store, then we don't + // need to bother checking the patient for conditions. + if (conditionsPresentInFhirStore == null) { + int conditionCount = fhirClient + .search() + .forResource(Condition.class) + .returnBundle(Bundle.class) + .summaryMode(SummaryEnum.COUNT) + .execute() + .getTotal(); + conditionsPresentInFhirStore = conditionCount > 0; + } + if (!conditionsPresentInFhirStore) + return conditionCodes; + + // Search for Condition resources by patient ID + Bundle bundle = fhirClient + .search() + .forResource(Condition.class) + .where(Condition.SUBJECT.hasId(patient.getIdElement())) + .returnBundle(Bundle.class) + .execute(); + if (!bundle.hasEntry()) + return conditionCodes; + + // Create a stream of Condition resources from the Bundle + Stream conditionStream = bundle.getEntry().stream() + // Map the bundle entries to Condition resources + .map(entry -> (Condition) entry.getResource()); + + // Loop over the Condition resources + conditionStream.forEach(condition -> { + // Get the code element of the Condition resource + CodeableConcept code = condition.getCode(); + // Get the list of coding elements from the code element + List codings = code.getCoding(); + // Loop over the coding elements + for (Coding coding : codings) { + // Check if the coding element has the system "http://hl7.org/fhir/sid/icd-10" + if (coding.getSystem().equals("http://hl7.org/fhir/sid/icd-10")) { + // Get the code value and the display value from the coding element + String codeValue = coding.getCode(); + //String displayValue = coding.getDisplay(); + conditionCodes.add(codeValue); + } + } + }); + } catch (ResourceNotFoundException e) { + logger.error("extractConditionCodesFromPatient: could not find Condition, stack trace:\n" + Util.traceFromException(e)); + } + + return conditionCodes; + } + + + /** + * Determines a plausible collection id for specimens that do not have a collection id. + * If no default collection id is provided, tries to find one from the available collections. + * If no valid collection id can be found, returns null. + * + * @param defaultBbmriEricCollectionId the default collection id supplied by the site + * @param specimensByCollection a map of collection id to list of specimens + * @return the default collection id, or null if none is found + */ + private BbmriEricId determineDefaultCollectionId(BbmriEricId defaultBbmriEricCollectionId, Map> specimensByCollection) { + logger.info("determineDefaultCollectionId: entered"); + logger.info("determineDefaultCollectionId: initial defaultBbmriEricCollectionId: " + defaultBbmriEricCollectionId); + + // If no default collection ID has been provided by the site, see if we can find a plausible value. + // If there are no specimens with a collection ID, but there is a single collection, + // then we can reasonably assume that the collection can be used as a default. + if (defaultBbmriEricCollectionId == null && specimensByCollection.size() == 1 && specimensByCollection.containsKey(DEFAULT_COLLECTION_ID)) { + logger.info("determineDefaultCollectionId: first conditional succeeded"); + + Either> collectionsOutcome = listAllCollections(); + if (collectionsOutcome.isRight()) { + logger.info("determineDefaultCollectionId: second conditional succeeded"); + + List collections = collectionsOutcome.get(); + if (collections.size() == 1) { + logger.info("determineDefaultCollectionId: third conditional succeeded"); + + String defaultCollectionId = extractValidDirectoryIdentifierFromCollection(collections.get(0)); + + logger.info("determineDefaultCollectionId: defaultCollectionId: " + defaultCollectionId); + + defaultBbmriEricCollectionId = BbmriEricId + .valueOf(defaultCollectionId) + .orElse(null); + } + } + } + + logger.info("determineDefaultCollectionId: final defaultBbmriEricCollectionId: " + defaultBbmriEricCollectionId); + + return defaultBbmriEricCollectionId; + } + + /** + * Extracts the collection id from a Specimen object that has a Custodian extension. + * The collection id is either a valid Directory collection id or the default value DEFAULT_COLLECTION_ID. + * If the Specimen object does not have a Custodian extension, the default value is returned. + * If the Specimen object has a Custodian extension, the collection id is obtained from the Organization reference in the extension. + * The collection id is then validated against the list of all collections returned by the listAllCollections() method. + * If the collection id is not found or invalid, the default value is returned. + * + * @param specimen the Specimen object to extract the collection id from + * @return the collection id as a String + */ + private String extractCollectionIdFromSpecimen(Specimen specimen) { + // We expect the specimen to have an extension for a collection, where we would find a collection + // ID. If we can't find that, then return the default collection ID. + if (!specimen.hasExtension()) + return DEFAULT_COLLECTION_ID; + Extension extension = specimen.getExtensionByUrl("https://fhir.bbmri.de/StructureDefinition/Custodian"); + if (extension == null) + return DEFAULT_COLLECTION_ID; + + // Pull the locally-used collection ID from the specimen extension. + String reference = ((Reference) extension.getValue()).getReference(); + String localCollectionId = reference.replaceFirst("Organization/", ""); + + String collectionId = extractValidDirectoryIdentifierFromCollection( + fhirClient + .read() + .resource(Organization.class) + .withId(localCollectionId) + .execute()); + + return collectionId; + } + + /** + * Gets the Directory collection ID from the identifier of the supplied collection. + * Returns DEFAULT_COLLECTION_ID if there is no identifier or if the identifier's value is not a valid + * Directory ID. + * + * @param collection + * @return + */ + private String extractValidDirectoryIdentifierFromCollection(Organization collection) { + String collectionId = DEFAULT_COLLECTION_ID; + List collectionIdentifiers = collection.getIdentifier(); + for (Identifier collectionIdentifier : collectionIdentifiers) { + String collectionIdentifierString = collectionIdentifier.getValue(); + if (isValidDirectoryCollectionIdentifier(collectionIdentifierString)) { + collectionId = collectionIdentifierString; + break; + } + } + + return collectionId; + } + + public List extractDiagnosesFromSpecimen(Specimen specimen) { + return extractExtensionElementValuesFromSpecimen(specimen, SAMPLE_DIAGNOSIS_URI); + } + + /** + * Extracts the code value of each extension element with a given URL from a Specimen resource. + * The extension element must have a value of type CodeableConcept. + * @param specimen a Specimen resource that may have extension elements + * @param url the URL of the extension element to extract + * @return a list of strings that contains the code value of each extension element with the given URL, or an empty list if none is found + */ + public List extractExtensionElementValuesFromSpecimen(Specimen specimen, String url) { + List extensions = specimen.getExtensionsByUrl(url); + List elementValues = new ArrayList(); + + // Check if the list is not empty + for (Extension extension: extensions) { + // Get the value of the extension as a Quantity object + CodeableConcept codeableConcept = (CodeableConcept) extension.getValue(); + + elementValues.add(codeableConcept.getCoding().get(0).getCode()); + } + + return elementValues; + } + + /** + * Extracts the code values of the extension elements with a given URL from a list of Specimen resources. + * The extension elements must have a value of type CodeableConcept. + * @param specimens a list of Specimen resources that may have extension elements + * @param url the URL of the extension elements to extract + * @return a list of strings that contains the distinct code values of the extension elements with the given URL, or an empty list if none are found + */ + public List extractExtensionElementValuesFromSpecimens(List specimens, String url) { + return specimens.stream() + // Flatten each specimen's extension elements into a single stream + .flatMap(s -> extractExtensionElementValuesFromSpecimen(s, url).stream()) + // Collect the results into a non-duplicating list + .distinct() + .collect(Collectors.toList()); + } + + private boolean isValidDirectoryCollectionIdentifier(String collectionIdentifier) { + if (collectionIdentifier == null) + return false; + String[] parts = collectionIdentifier.split(":"); + if (parts.length != 5) + return false; + if ( ! parts[1].equals("ID")) + return false; + if ( ! parts[3].equals("collection")) + return false; + return true; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/fhir/FhirReporting.java b/src/main/java/de/samply/directory_sync_service/fhir/FhirReporting.java new file mode 100644 index 0000000..9e9d476 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/fhir/FhirReporting.java @@ -0,0 +1,441 @@ +package de.samply.directory_sync_service.fhir; + +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Optional.empty; +import static org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity.ERROR; + +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.parser.IParser; +import com.google.common.io.ByteStreams; + +import de.samply.directory_sync_service.model.StarModelData; +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.directory.model.BbmriEricId; +import de.samply.directory_sync_service.fhir.model.FhirCollection; +import io.vavr.Tuple; +import io.vavr.control.Either; +import java.io.IOException; +import java.io.InputStream; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.hl7.fhir.instance.model.api.IBaseResource; +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Library; +import org.hl7.fhir.r4.model.Measure; +import org.hl7.fhir.r4.model.MeasureReport; +import org.hl7.fhir.r4.model.MeasureReport.StratifierGroupComponent; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Organization; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Specimen; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides functionality related to FHIR MeasureReports. + */ +public class FhirReporting { + + private static final Logger logger = LoggerFactory.getLogger(FhirReporting.class); + + private static final String LIBRARY_URI = "https://fhir.bbmri.de/Library/collection-size"; + private static final String MEASURE_URI = "https://fhir.bbmri.de/Measure/collection-size"; + private static final String STORAGE_TEMPERATURE_URI = "https://fhir.bbmri.de/StructureDefinition/StorageTemperature"; + private static final String SAMPLE_DIAGNOSIS_URI = "https://fhir.bbmri.de/StructureDefinition/SampleDiagnosis"; + + private final FhirContext fhirContext; + private final FhirApi fhirApi; + + public FhirReporting(FhirContext fhirContext, FhirApi fhirApi) { + this.fhirContext = Objects.requireNonNull(fhirContext); + this.fhirApi = Objects.requireNonNull(fhirApi); + } + + /** + * The returned map key is an optional FHIR logical ID. The empty case encompasses all Specimen + * which are not assigned to a Collection. + */ + private static Map, Integer> extractStratifierCounts(MeasureReport report) { + return report.getGroupFirstRep().getStratifierFirstRep().getStratum().stream() + .collect(Collectors.toMap(FhirReporting::extractFhirId, + stratum -> stratum.getPopulationFirstRep().getCount(), + Integer::sum)); + } + + private static Optional extractFhirId(StratifierGroupComponent stratum) { + String[] parts = stratum.getValue().getText().split("/"); + return parts.length == 2 ? Optional.of(parts[1]) : empty(); + } + + /** + * Maps the logical FHIR ID keys of {@code counts} to BBMRI-ERIC ID keys using + * {@code collections}. + * + * @param counts map from FHIR logical ID to counts + * @param collections list of Organization resources to use for resolving the BBMRI-ERIC ID's + * @return a map of BBMRI_ERIC ID to counts + */ + private static Map resolveBbmriEricIds(Map counts, + List collections) { + return collections.stream() + .map(c -> Tuple.of(FhirApi.bbmriEricId(c), counts.get(c.getIdElement().getIdPart()))) + .filter(t -> t._1.isPresent()) + .filter(t -> t._2 != null) + .collect(Collectors.toMap(t -> t._1.get(), t -> t._2, Integer::sum)); + } + + /** + * Tries to create Library and Measure resources if not present on the FHIR server. + * + * @return either an error or nothing + */ + public Either initLibrary() { + logger.info("initLibrary: entered"); + return fhirApi.resourceExists(Library.class, LIBRARY_URI) + .flatMap(exists -> exists + ? Either.right(null) + : slurp("CollectionSize.Library.json") + .flatMap(s -> parseResource(Library.class, s)) + .flatMap(this::appendCql) + .flatMap(fhirApi::createResource)); + } + + public Either initMeasure() { + logger.info("initMeasure: entered"); + return fhirApi.resourceExists(Measure.class, MEASURE_URI) + .flatMap(exists -> exists + ? Either.right(null) + : slurp("CollectionSize.Measure.json") + .flatMap(s -> parseResource(Measure.class, s)) + .flatMap(fhirApi::createResource)); + } + + private static Either slurp(String name) { + logger.info("slurp: file name: " + name); + try (InputStream in = FhirApi.class.getResourceAsStream(name)) { + if (in == null) { + logger.error("file `{}` not found in classpath", name); + return Either.left(format("file `%s` not found in classpath", name)); + } else { + logger.info("read file `{}` from classpath", name); + return Either.right(new String(ByteStreams.toByteArray(in), UTF_8)); + } + } catch (IOException e) { + logger.error("error while reading the file `{}` from classpath", name, e); + return Either.left(format("error while reading the file `%s` from classpath", name)); + } + } + + private Either parseResource(Class type, String s) { + logger.info("parseResource: s: " + s); + IParser parser = fhirContext.newJsonParser(); + logger.info("parseResource: try parsing it"); + try { + return Either.right(type.cast(parser.parseResource(s))); + } catch (Exception e) { + return Either.left(e.getMessage()); + } + } + + private Either appendCql(Library library) { + return slurp("CollectionSize.cql").map(cql -> { + library.getContentFirstRep().setContentType("text/cql"); + library.getContentFirstRep().setData(cql.getBytes(UTF_8)); + return library; + }); + } + + /** + * Returns collection sample counts indexed by BBMRI-ERIC identifier. + *

+ * Executes the collection-size + * measure. + *

+ * In case all samples are unassigned, meaning the stratum code has text {@literal null} and only + * one collection exists, all that samples are assigned to this single collection. + * + * @return collection sample counts indexed by BBMRI-ERIC identifier or OperationOutcome + * indicating an error + */ + public Either> fetchCollectionSizes() { + return fhirApi.evaluateMeasure(MEASURE_URI) + .map(FhirReporting::extractStratifierCounts) + .flatMap(counts -> { + if (counts.size() == 1 && counts.containsKey(Optional.empty())) { + return fhirApi.listAllCollections() + .map(collections -> { + if (collections.size() == 1) { + return FhirApi.bbmriEricId(collections.get(0)) + .map(ericId -> Util.mapOf(ericId, counts.get(Optional.empty()))) + .orElseGet(Util::mapOf); + } else { + return Util.mapOf(); + } + }); + } else { + return fhirApi.fetchCollections(filterPresents(counts.keySet())) + .map(collections -> resolveBbmriEricIds(filterPresents(counts), collections)); + } + }); + } + + /** + * Pulls information relevant to collections from the FHIR store. + * + * Returns a list of FhirCollection objects, one per collection. + * + * @param defaultBbmriEricCollectionId + * @return + */ + public Either> fetchFhirCollections(BbmriEricId defaultBbmriEricCollectionId) { + Map fhirCollectionMap = new HashMap(); + + // Group specimens according to collection, extract aggregated information + // from each group, and put this information into FhirCollection objects. + Either>> specimensByCollectionOutcome = fhirApi.fetchSpecimensByCollection(defaultBbmriEricCollectionId); + if (specimensByCollectionOutcome.isLeft()) + return Either.left(createOutcomeWithError("fetchFhirCollections: Problem finding specimens")); + updateFhirCollectionsWithSpecimenData(fhirCollectionMap, specimensByCollectionOutcome.get()); + + // Group patients according to collection, extract aggregated information + // from each group, and put this information into FhirCollection objects. + Either>> patientsByCollectionOutcome = fhirApi.fetchPatientsByCollection(specimensByCollectionOutcome.get()); + if (patientsByCollectionOutcome.isLeft()) + return Either.left(createOutcomeWithError("Problem finding patients")); + updateFhirCollectionsWithPatientData(fhirCollectionMap, patientsByCollectionOutcome.get()); + + return Either.right(new ArrayList(fhirCollectionMap.values())); + } + + private void updateFhirCollectionsWithSpecimenData(Map entities, Map> specimensByCollection) { + for (String key: specimensByCollection.keySet()) { + List specimenList = specimensByCollection.get(key); + FhirCollection fhirCollection = entities.getOrDefault(key, new FhirCollection()); + fhirCollection.setId(key); + fhirCollection.setSize(specimenList.size()); + fhirCollection.setMaterials(extractMaterialsFromSpecimenList(specimenList)); + fhirCollection.setStorageTemperatures(extractStorageTemperaturesFromSpecimenList(specimenList)); + fhirCollection.setDiagnosisAvailable(extractDiagnosesFromSpecimenList(specimenList)); + entities.put(key, fhirCollection); + } + } + + private void updateFhirCollectionsWithPatientData(Map entities, Map> patientsByCollection) { + for (String key: patientsByCollection.keySet()) { + List patientList = patientsByCollection.get(key); + FhirCollection fhirCollection = entities.getOrDefault(key, new FhirCollection()); + fhirCollection.setNumberOfDonors(patientList.size()); + fhirCollection.setSex(extractSexFromPatientList(patientList)); + fhirCollection.setAgeLow(extractAgeLowFromPatientList(patientList)); + fhirCollection.setAgeHigh(extractAgeHighFromPatientList(patientList)); + entities.put(key, fhirCollection); + } + } + + public Either fetchStarModelInputData(BbmriEricId defaultBbmriEricCollectionId) { + PopulateStarModelInputData populateStarModelInputData = new PopulateStarModelInputData(fhirApi); + StarModelData starModelInputData = populateStarModelInputData.populate(defaultBbmriEricCollectionId); + + return Either.right(starModelInputData); + } + + /** + * Fetches diagnoses from Specimens and Patients to which collections can be assigned. + * + * This method retrieves specimens grouped by collection. + * + * It then extracts diagnoses from Specimen extensions and Patient condition codes, eliminating duplicates, + * and combines the results into a list of unique diagnoses. + * + * @param defaultBbmriEricCollectionId The BBMRI ERIC collection ID to fetch specimens and diagnoses. + * @return Either an OperationOutcome indicating an error or a List of unique diagnoses. + * If an error occurs during the fetching process, an OperationOutcome with an error message is returned. + * Otherwise, a List of unique diagnoses is returned. + */ + public Either> fetchDiagnoses(BbmriEricId defaultBbmriEricCollectionId) { + logger.info("fetchDiagnoses: defaultBbmriEricCollectionId: " + defaultBbmriEricCollectionId); + // Group specimens according to collection. + Either>> specimensByCollectionOutcome = fhirApi.fetchSpecimensByCollection(defaultBbmriEricCollectionId); + if (specimensByCollectionOutcome.isLeft()) + return Either.left(createOutcomeWithError("fetchDiagnoses: Problem finding specimens")); + Map> specimensByCollection = specimensByCollectionOutcome.get(); + + // Get diagnoses from Specimen extensions + List diagnoses = specimensByCollection.values().stream() + .flatMap(List::stream) + .map(s -> fhirApi.extractDiagnosesFromSpecimen(s)) + .flatMap(List::stream) + .distinct() + .collect(Collectors.toList()); + + // Get diagnoses from Patients + Either>> patientsByCollectionOutcome = fhirApi.fetchPatientsByCollection(specimensByCollection); + Map> patientsByCollection = patientsByCollectionOutcome.get(); + List patientDiagnoses = patientsByCollection.values().stream() + .flatMap(List::stream) + .map(s -> fhirApi.extractConditionCodesFromPatient(s)) + .flatMap(List::stream) + .distinct() + .collect(Collectors.toList()); + + // Combine diagnoses from specimens and patients, ensuring that there + // are no duplicates. + diagnoses = Stream.concat(diagnoses.stream(), patientDiagnoses.stream()) + .distinct() + .collect(Collectors.toList()); + + return Either.right(diagnoses); + } + + private OperationOutcome createOutcomeWithError(String message) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR).setDiagnostics(message); + return outcome; + } + +/* + private List extractMaterialsFromSpecimenList(List specimens) { + return specimens.stream() + // Map each specimen to its type + .map(Specimen::getType) + // Map each CodeableConcept to its display name + .map(c -> c.getCoding().get(0).getCode()) + // Collect the results into a non-duplicating list + .collect(Collectors.toSet()).stream().collect(Collectors.toList()); + } +*/ + + /** + * Extracts unique material codes from a list of specimens. + * + * @param specimens A list of {@code Specimen} objects from which to extract material codes. + * @return A list of unique material codes (as strings) extracted from the specimens. + */ + private List extractMaterialsFromSpecimenList(List specimens) { + // Print a log info + logger.info("extractMaterialsFromSpecimenList: entered"); + logger.info("extractMaterialsFromSpecimenList: specimens: " + specimens); + logger.info("extractMaterialsFromSpecimenList: Number of specimens: " + specimens.size()); + + // Step 1: Stream the list of specimens + // Convert the list of specimens to a stream to process each element individually + Stream specimenStream = specimens.stream(); + + logger.info("extractMaterialsFromSpecimenList: step 2"); + + // Step 2: Map each specimen to its type (returns a CodeableConcept object) + Stream typeStream = specimenStream.map(Specimen::getType); + + logger.info("extractMaterialsFromSpecimenList: typeStream: " + typeStream); + logger.info("extractMaterialsFromSpecimenList: step 3"); + + // Step 3: Map each CodeableConcept to its first coding's code + Stream codeStream = typeStream + // Filter out any CodeableConcept objects where getCoding returns an empty list + .filter(c -> c.getCoding() != null && !c.getCoding().isEmpty()) + // Map each remaining CodeableConcept to its first coding's code + .map(c -> c.getCoding().get(0).getCode()); + + logger.info("extractMaterialsFromSpecimenList: codeStream: " + codeStream); + logger.info("extractMaterialsFromSpecimenList: step 4"); + + // Step 4: Collect the results into a Set to remove duplicates + Set uniqueCodes = codeStream.collect(Collectors.toSet()); + + logger.info("extractMaterialsFromSpecimenList: step 5"); + + // Step 5: Convert the Set back into a List and return + List uniqueCodeList = uniqueCodes.stream().collect(Collectors.toList()); + + logger.info("extractMaterialsFromSpecimenList: returning"); + + return uniqueCodeList; + } + + private List extractStorageTemperaturesFromSpecimenList(List specimens) { + return fhirApi.extractExtensionElementValuesFromSpecimens(specimens, STORAGE_TEMPERATURE_URI); + } + + private List extractDiagnosesFromSpecimenList(List specimens) { + return fhirApi.extractExtensionElementValuesFromSpecimens(specimens, SAMPLE_DIAGNOSIS_URI); + } + + private List extractSexFromPatientList(List patients) { + return patients.stream() + .filter(patient -> Objects.nonNull(patient.getGenderElement())) // Filter out patients with null gender + .map(patient -> patient.getGenderElement().getValueAsString()) // Map each patient to their gender + .collect(Collectors.toSet()).stream().collect(Collectors.toList()); // Collect the results into a new list + } + + private Integer extractAgeLowFromPatientList(List patients) { + return patients.stream() + // Filter out patients with null age + .filter(p -> Objects.nonNull(determinePatientAge(p))) + // Map each patient to their age + .mapToInt(p -> determinePatientAge(p)) + // Find the minimum age + .min() + // Get the result as an int or a default value + .orElse(-1); + } + + private Integer extractAgeHighFromPatientList(List patients) { + return patients.stream() + // Filter out patients with null age + .filter(p -> Objects.nonNull(determinePatientAge(p))) + // Map each patient to their age + .mapToInt(p -> determinePatientAge(p)) + // Find the maximum age + .max() + // Get the result as an int or a default value + .orElse(-1); + } + + private Integer determinePatientAge(Patient patient) { + if (!patient.hasBirthDate()) + return null; + + // Get the patient's date of birth as a Date object + Date birthDate = patient.getBirthDate(); + + // Convert the Date object to a LocalDate object + LocalDate birthDateLocal = birthDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + + // Get the current date as a LocalDate object + LocalDate currentDate = LocalDate.now(); + + // Calculate the difference between the two dates in years + int age = currentDate.getYear() - birthDateLocal.getYear(); + + // Adjust the age if the current date is before the patient's birthday + if (currentDate.getDayOfYear() < birthDateLocal.getDayOfYear()) + age--; + + return age; + } + + private static Set filterPresents(Set> optionals) { + return optionals.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toSet()); + } + + private static Map filterPresents(Map, V> optionals) { + return filterPresents(optionals.keySet()).stream() + .collect(Collectors.toMap(Function.identity(), k -> optionals.get(Optional.of(k)))); + } +} diff --git a/src/main/java/de/samply/directory_sync_service/fhir/PopulateStarModelInputData.java b/src/main/java/de/samply/directory_sync_service/fhir/PopulateStarModelInputData.java new file mode 100644 index 0000000..2fbc464 --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/fhir/PopulateStarModelInputData.java @@ -0,0 +1,227 @@ +package de.samply.directory_sync_service.fhir; + +import java.time.LocalDate; +import java.time.Period; +import java.time.ZoneId; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.hl7.fhir.r4.model.CodeableConcept; +import org.hl7.fhir.r4.model.Coding; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.Specimen; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.samply.directory_sync_service.model.StarModelData; +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.directory.model.BbmriEricId; +import io.vavr.control.Either; + +/** + * Pull data about Patients, Specimens and Dieseases from the FHIR store and + * use the information they contain to fill a StarModelInputData object. + */ +public class PopulateStarModelInputData { + private static final Logger logger = LoggerFactory.getLogger(PopulateStarModelInputData.class); + private FhirApi fhirApi; + + public PopulateStarModelInputData(FhirApi fhirApi) { + this.fhirApi = fhirApi; + } + + /** + * Populates a Star Model input data object based on specimens fetched from the FHIR server, + * grouped according to the specified default BBMRI-ERIC collection ID. + * + * @param defaultBbmriEricCollectionId The default BBMRI-ERIC collection ID to group specimens. May be null. + * @return A StarModelData object populated with data extracted from the fetched specimens. + */ + public StarModelData populate(BbmriEricId defaultBbmriEricCollectionId) { + // Group specimens according to collection. + Either>> specimensByCollectionOutcome = fhirApi.fetchSpecimensByCollection(defaultBbmriEricCollectionId); + if (specimensByCollectionOutcome.isLeft()) { + logger.error("Problem finding specimens"); + return null; + } + Map> specimensByCollection = specimensByCollectionOutcome.get(); + + StarModelData starModelInputData = new StarModelData(); + for (String collectionId: specimensByCollection.keySet()) + populateCollection(starModelInputData, collectionId, specimensByCollection.get(collectionId)); + + return starModelInputData; + } + + /** + * Populates the Star Model input data with information extracted from a list of specimens + * associated with a specific collection. + * + * @param starModelInputData The Star Model input data to be populated. + * @param collectionId The identifier for the collection to which the specimens belong. + * @param specimens The list of specimens from which to extract data and populate the input data. + * + * @throws NullPointerException if starModelInputData, collectionId, or specimens is null. + */ + private void populateCollection(StarModelData starModelInputData, String collectionId, List specimens) { + for (Specimen specimen: specimens) + populateSpecimen(starModelInputData, collectionId, specimen); + } + + /** + * Populates the Star Model input data with information extracted from a single specimen. + * + * @param starModelInputData The Star Model input data to be populated. + * @param collectionId The identifier for the collection to which the specimen belongs. + * @param specimen The specimen from which to extract data and populate the input data. + * + * @throws NullPointerException if starModelInputData, collectionId, or specimen is null. + */ + private void populateSpecimen(StarModelData starModelInputData, String collectionId, Specimen specimen) { + // Get the Patient who donated the sample + Patient patient = fhirApi.extractPatientFromSpecimen(specimen); + + String material = extractMaterialFromSpecimen(specimen); + String patientId = patient.getIdElement().getIdPart(); + String sex = patient.getGender().getDisplay(); + String age = determinePatientAgeAtCollection(patient, specimen); + + // Create a new Row object to hold data extracted from patient and specimen + StarModelData.InputRow row = starModelInputData.newInputRow(collectionId, material, patientId, sex, age); + + List diagnoses = extractDiagnosesFromPatientAndSpecimen(patient, specimen); + + // Add all of the collected information to the input data table. + for (String diagnosis: diagnoses) + starModelInputData.addInputRow(collectionId, starModelInputData.newInputRow(row, diagnosis)); + } + + /** + * Determines the patient's age at the time of specimen collection. + * + * @param patient The FHIR Patient object from which to retrieve the birth date. + * @param specimen The FHIR Specimen object from which to extract the collection date. + * @return The patient's age at the time of specimen collection in years, or null if the age calculation fails. + * + * @throws NullPointerException if either patient or specimen is null. + * @throws RuntimeException if an unexpected error occurs during the age calculation. + */ + private String determinePatientAgeAtCollection(Patient patient, Specimen specimen) { + String age = null; + + try { + Date birthDate = patient.getBirthDate(); + if (birthDate == null) { + logger.warn("determinePatientAgeAtCollection: patient.getBirthDate() is null, returning null."); + return null; + } + // Get the patient's birth date as a LocalDate object + LocalDate localBirthDate = birthDate.toInstant() + .atZone(java.time.ZoneId.systemDefault()) + .toLocalDate(); + + LocalDate collectionDate = extractCollectionLocalDateFromSpecimen(specimen); + + // Calculate the patient's age in years using the Period class + int ageInYears = Period.between(localBirthDate, collectionDate).getYears(); + + if (ageInYears < 0) { + logger.warn("determinePatientAgeAtCollection: age at collection is negative, substituting null"); + age = null; + } else + age = Integer.toString(ageInYears); + } catch (Exception e) { + logger.warn("determinePatientAgeAtCollection: problem determining patient age, following exception caught: " + Util.traceFromException(e)); + } + + if (age == null) + logger.warn("determinePatientAgeAtCollection: returning null."); + + return age; + } + + /** + * Extracts the collection date as a LocalDate from the given FHIR Specimen. + * If the Specimen is null or does not have a collection date, it returns null. + * + * @param specimen The FHIR Specimen object from which to extract the collection date. + * @return The collection date as a LocalDate, or null if the specimen is null or lacks a collection date. + * + * @throws NullPointerException if specimen is null. + */ + private LocalDate extractCollectionLocalDateFromSpecimen(Specimen specimen) { + // Check if the specimen is null or has no collection date + if (specimen == null) { + logger.warn("extractCollectionLocalDateFromSpecimen: specimen is null, returning null"); + return null; + } + if (!specimen.hasCollection()) { + logger.warn("extractCollectionLocalDateFromSpecimen: specimen has no collection date, returning null"); + return null; + } + + Specimen.SpecimenCollectionComponent collection = specimen.getCollection(); + if (collection.hasCollectedDateTimeType()) { + DateTimeType collected = collection.getCollectedDateTimeType(); + Date date = collected.getValue(); // Get the java.util.Date object + LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + + return localDate; + } else { + logger.warn("extractCollectionLocalDateFromSpecimen: no date/time for specimen collection, returning null"); + return null; + } + } + + /** + * Extracts unique diagnoses associated with a given Patient and Specimen. + * This method combines diagnoses obtained from the Patient's conditions and Specimen's diagnoses. + * + * @param patient The FHIR Patient object from which to extract diagnoses. + * @param specimen The FHIR Specimen object from which to extract diagnoses. + * @return A List of unique diagnoses associated with the given Patient and Specimen. + * + * @throws NullPointerException if either patient or specimen is null. + */ + private List extractDiagnosesFromPatientAndSpecimen(Patient patient, Specimen specimen) { + // Find any diagnoses associated with this patient + List patientConditionCodes = fhirApi.extractConditionCodesFromPatient(patient); + + // Find any diagnoses associated with this specimen + List diagnosesFromSpecimen = fhirApi.extractDiagnosesFromSpecimen(specimen); + + // Combine diagnosis lists + return Stream.concat(patientConditionCodes.stream(), diagnosesFromSpecimen.stream()) + .distinct() + .collect(Collectors.toList()); + } + + /** + * Extracts the material from a Specimen object. + *

+ * This method returns the text or the code of the type element of the Specimen object, + * or null if the type element is missing or empty. + *

+ * @param specimen the Specimen object to extract the material from + * @return the material as a String, or null if not available + */ + private String extractMaterialFromSpecimen(Specimen specimen) { + String material = null; + + CodeableConcept type = specimen.getType(); + if (type.hasText()) + material = type.getText(); + else { + List coding = type.getCoding(); + if (coding.size() > 0) + material = coding.get(0).getCode(); + } + + return material; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/fhir/model/FhirCollection.java b/src/main/java/de/samply/directory_sync_service/fhir/model/FhirCollection.java new file mode 100644 index 0000000..7cae33b --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/fhir/model/FhirCollection.java @@ -0,0 +1,101 @@ +package de.samply.directory_sync_service.fhir.model; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A DTO for carrying the data collected from a FHIR store relating to a single + * collection of samples. + */ +public class FhirCollection { + private static final Logger logger = LoggerFactory.getLogger(FhirCollection.class); + + private String id; + private String country; + private Integer size; + private Integer numberOfDonors; + private List sex; + private Integer ageLow; + private Integer ageHigh; + private List materials; + private List storageTemperatures; + private List diagnosisAvailable; + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setCountry(String country) { + this.country = country; + } + + public void setSize(Integer size) { + this.size = size; + } + + public Integer getSize() { + return size; + } + + public void setNumberOfDonors(Integer numberOfDonors) { + this.numberOfDonors = numberOfDonors; + } + + public Integer getNumberOfDonors() { + return numberOfDonors; + } + + public void setSex(List sex) { + this.sex = sex; + } + + public List getSex() { + return sex; + } + + public void setAgeLow(Integer ageLow) { + this.ageLow = ageLow; + } + + public Integer getAgeLow() { + return ageLow; + } + + public void setAgeHigh(Integer ageHigh) { + this.ageHigh = ageHigh; + } + + public Integer getAgeHigh() { + return ageHigh; + } + + public void setMaterials(List materials) { + this.materials = materials; + } + + public List getMaterials() { + return materials; + } + + public void setStorageTemperatures(List storageTemperatures) { + this.storageTemperatures = storageTemperatures; + } + + public List getStorageTemperatures() { + return storageTemperatures; + } + + public void setDiagnosisAvailable(List diagnosisAvailable) { + this.diagnosisAvailable = diagnosisAvailable; + } + + public List getDiagnosisAvailable() { + return diagnosisAvailable; + } +} diff --git a/src/main/java/de/samply/directory_sync_service/model/StarModelData.java b/src/main/java/de/samply/directory_sync_service/model/StarModelData.java new file mode 100644 index 0000000..e9a0d6c --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/model/StarModelData.java @@ -0,0 +1,317 @@ +package de.samply.directory_sync_service.model; + +import java.util.HashMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import de.samply.directory_sync_service.converter.FhirToDirectoryAttributeConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.samply.directory_sync_service.directory.model.BbmriEricId; + +/** + * Represents data for the STAR model, organized by collection. + * + * Input data represents data read in from the FHIR store. + * + * Output data is in a format that is ready to be exported to the Directory. + */ +public class StarModelData { + private static final Logger logger = LoggerFactory.getLogger(StarModelData.class); + + // *** Miscellaneous data + + // Minimum number of donors per fact + private int minDonors = 10; // default value + + /** + * Gets the current minimum number of donors required per fact. + * + * @return The minimum number of donors per fact. + */ + public int getMinDonors() { + return minDonors; + } + + /** + * Sets the minimum number of donors required per fact. + * + * @param minDonors The new minimum number of donors per fact to be set. + */ + public void setMinDonors(int minDonors) { + this.minDonors = minDonors; + } + + public List> getInputRowsAsStringMaps(String collectionId) { + List> rowsAsStringMaps = new ArrayList>(); + for (InputRow row: inputData.get(collectionId)) { + rowsAsStringMaps.add(row); + } + + return rowsAsStringMaps; + } + + // *** Input data for the star model. + + /** + * Represents an input row of the inputData table, with attributes commonly associated with medical data. + * Extends the HashMap class to provide a key-value mapping for attributes. + * + * The attributes include collection, sample material, patient ID, sex, histological location, + * and age at primary diagnosis. + */ + public class InputRow extends HashMap { + + /** + * Constructs a new InputRow with the specified attributes. + * + * @param collection The identifier for the collection associated with the input row. + * @param sampleMaterial The sample material associated with the input row. + * @param patientId The identifier of the patient associated with the input row. + * @param sex The gender information of the patient. + * @param age The age at primary diagnosis of the patient. + */ + public InputRow(String collection, String sampleMaterial, String patientId, String sex, String age) { + setCollection(collection); + setSampleMaterial(sampleMaterial); + setId(patientId); + setSex(sex); + setAgeAtPrimaryDiagnosis(age); + } + + /** + * Constructs a new InputRow based on an existing row and updates it with a new diagnosis. + * + * @param row The existing input row to base the new row on. + * @param histLoc The new histological location to be associated with the input row. + */ + public InputRow(InputRow row, String histLoc) { + for (String key : row.keySet()) + put(key, row.get(key)); + setHistLoc(histLoc); + } + + /** + * Sets the collection attribute for the input row. + * + * @param collection The identifier for the collection to be associated with the input row. + */ + public void setCollection(String collection) { + if (collection == null) + return; + put("collection", collection); + } + + /** + * Sets the sample material attribute for the input row. + * Converts the provided sample material using the FhirToDirectoryAttributeConverter. + * + * @param sampleMaterial The sample material to be associated with the input row. + * + * @see FhirToDirectoryAttributeConverter#convertMaterial(String) + */ + public void setSampleMaterial(String sampleMaterial) { + if (sampleMaterial == null) + return; + put("sample_material", FhirToDirectoryAttributeConverter.convertMaterial(sampleMaterial)); + } + + /** + * Sets the patient ID attribute for the input row. + * + * @param id The identifier of the patient to be associated with the input row. + */ + public void setId(String id) { + if (id == null) + return; + put("id", id); + } + + /** + * Sets the sex attribute for the input row. + * Converts the provided sex information using the FhirToDirectoryAttributeConverter. + * + * @param sex The gender information to be associated with the input row. + * + * @see FhirToDirectoryAttributeConverter#convertSex(String) + */ + public void setSex(String sex) { + if (sex == null) + return; + put("sex", FhirToDirectoryAttributeConverter.convertSex(sex)); + } + + /** + * Sets the histological location attribute for the input row. + * Converts the provided histological location using the FhirToDirectoryAttributeConverter. + * + * @param histLoc The histological location to be associated with the input row. + * + * @see FhirToDirectoryAttributeConverter#convertDiagnosis(String) + */ + public void setHistLoc(String histLoc) { + if (histLoc == null) + return; + put("hist_loc", FhirToDirectoryAttributeConverter.convertDiagnosis(histLoc)); + } + + /** + * Sets the age at primary diagnosis attribute for the input row. + * + * @param age The age at primary diagnosis to be associated with the input row. + */ + public void setAgeAtPrimaryDiagnosis(String age) { + if (age == null) { + logger.warn("setAgeAtPrimaryDiagnosis: age is null, ignoring."); + return; + } + put("age_at_primary_diagnosis", age); + } + } + + // Data relevant for Directory sync that has been read in from the FHIR store. + // This comprises of one row per Patient/Specimen/Diagnosis combination. + // Each row is a map containing attributes relevant to the star model. + // A Map of a List of Maps: collectionID_1 -> [row0, row1, ...] + private Map> inputData = new HashMap>(); + + /** + * Adds an input row to the specified collection in the inputData map. + * If the collectionId does not exist in the map, a new entry is created. + * + * @param collectionId The identifier for the collection where the input row will be added. + * @param row The input row to be added to the collection. + * + * @throws NullPointerException if collectionId or row is null. + */ + public void addInputRow(String collectionId, InputRow row) { + if (!inputData.containsKey(collectionId)) + inputData.put(collectionId, new ArrayList()); + List rows = inputData.get(collectionId); + rows.add(row); + } + + /** + * Creates and returns a new InputRow with the specified attributes. + * + * @param collection The identifier for the collection of the new input row. + * @param sampleMaterial The sample material associated with the input row. + * @param patientId The identifier of the patient associated with the input row. + * @param sex The gender information of the patient. + * @param age The age information of the patient. + * + * @return A new InputRow with the provided attributes. + * + * @throws NullPointerException if any of the parameters is null. + * + * @see InputRow + */ + public InputRow newInputRow(String collection, String sampleMaterial, String patientId, String sex, String age) { + return new InputRow(collection, sampleMaterial, patientId, sex, age); + } + + /** + * Creates a new InputRow based on an existing row and adds a diagnosis. + * + * @param row The existing input row to base the new row on. + * @param histLoc The diagnosis. + * + * @return A new InputRow with the diagnosis added. + * + * @throws NullPointerException if row or histLoc is null. + */ + public InputRow newInputRow(InputRow row, String histLoc) { + return new InputRow(row, histLoc); + } + + public Set getInputCollectionIds() { + return inputData.keySet(); + } + + // *** Output data. + + // One big fact table for everything. Every fact contains a mandatory collection ID. + // A single "fact" is a simple String map, with medically relevant keys. + private List> factTables = new ArrayList>(); + + /** + * Adds a collection of facts to the existing fact table. + * + * @param collectionId The mandatory collection ID associated with the facts. + * @param factTable The list of facts represented as String maps to be added to the fact table. + */ + public void addFactTable(String collectionId, List> factTable) { + factTables.addAll(factTable); + } + + /** + * Retrieves the entire fact table containing medically relevant information. + * + * @return The list of facts represented as String maps with mandatory collection ID. + */ + public List> getFactTables() { + return factTables; + } + + /** + * Gets the count of facts in the fact table. + * + * @return The number of facts present in the fact table. + */ + public int getFactCount() { + return factTables.size(); + } + + /** + * Implements diagnosis corrections for the facts in the factTables. + * For each fact in the factTables, checks if it contains a "disease" key. + * If the "disease" key is present, it attempts to replace its value with + * the corrected diagnosis from the diagnoses map. If the corrected diagnosis + * is found, it updates the fact's "disease" value; otherwise, it removes + * the "disease" key from the fact. + * + * Note: This method directly modifies the factTables in-place. + * + * @param diagnoses Maps FHIR diagnoses onto Directory diagnoses. + * @throws NullPointerException if diagnoses or any fact in factTables is null. + */ + public void applyDiagnosisCorrections(Map diagnoses) { + for (Map fact: factTables) { + if (!fact.containsKey("disease")) + continue; + String disease = fact.get("disease"); + if (disease != null && diagnoses.containsKey(disease)) + fact.put("disease", diagnoses.get(disease)); + if (fact.get("disease") == null) + fact.remove("disease"); + } + } + + /** + * Gets the country code for the collections, e.g. "DE". + * + * Assumes that all collections will have the same code and simply returns + * the code of the first collection. + * + * If there are no collections, returns null. + * + * May throw a null pointer exception. + * + * @return Country code + */ + public String getCountryCode() { + if (factTables == null || factTables.size() == 0) + return null; + + String countryCode = BbmriEricId + .valueOf((String) factTables.get(0).get("collection")) + .orElse(null) + .getCountryCode(); + + return countryCode; + } + +} diff --git a/src/main/java/de/samply/directory_sync_service/Configuration.java b/src/main/java/de/samply/directory_sync_service/service/Configuration.java similarity index 95% rename from src/main/java/de/samply/directory_sync_service/Configuration.java rename to src/main/java/de/samply/directory_sync_service/service/Configuration.java index 3cd8020..bb006f0 100644 --- a/src/main/java/de/samply/directory_sync_service/Configuration.java +++ b/src/main/java/de/samply/directory_sync_service/service/Configuration.java @@ -1,4 +1,4 @@ -package de.samply.directory_sync_service; +package de.samply.directory_sync_service.service; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/de/samply/directory_sync_service/DirectorySync.java b/src/main/java/de/samply/directory_sync_service/service/DirectorySync.java similarity index 91% rename from src/main/java/de/samply/directory_sync_service/DirectorySync.java rename to src/main/java/de/samply/directory_sync_service/service/DirectorySync.java index 87b42ad..605d288 100644 --- a/src/main/java/de/samply/directory_sync_service/DirectorySync.java +++ b/src/main/java/de/samply/directory_sync_service/service/DirectorySync.java @@ -1,13 +1,14 @@ -package de.samply.directory_sync_service; +package de.samply.directory_sync_service.service; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; -import de.samply.directory_sync.Sync; -import de.samply.directory_sync.directory.DirectoryApi; -import de.samply.directory_sync.directory.DirectoryService; -import de.samply.directory_sync.fhir.FhirApi; -import de.samply.directory_sync.fhir.FhirReporting; +import de.samply.directory_sync_service.sync.Sync; +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.directory.DirectoryApi; +import de.samply.directory_sync_service.directory.DirectoryService; +import de.samply.directory_sync_service.fhir.FhirApi; +import de.samply.directory_sync_service.fhir.FhirReporting; import io.vavr.control.Either; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; @@ -75,10 +76,10 @@ private boolean syncWithDirectory(Configuration configuration) { } directoryApi = directoryApiContainer.get(); } catch (IOException e) { - logger.error("__________ syncWithDirectory: createDirectoryApi failed: " + Utils.traceFromException(e)); + logger.error("__________ syncWithDirectory: createDirectoryApi failed: " + Util.traceFromException(e)); return false; } catch (NullPointerException e) { - logger.error("__________ syncWithDirectory: createDirectoryApi failed: " + Utils.traceFromException(e)); + logger.error("__________ syncWithDirectory: createDirectoryApi failed: " + Util.traceFromException(e)); return false; } DirectoryService directoryService = new DirectoryService(directoryApi); @@ -111,23 +112,26 @@ private boolean syncWithDirectory(Configuration configuration) { } } operationOutcomes = sync.sendUpdatesToDirectory(directoryDefaultCollectionId); + boolean failed = false; for (OperationOutcome operationOutcome : operationOutcomes) { String errorMessage = getErrorMessageFromOperationOutcome(operationOutcome); if (errorMessage.length() > 0) { logger.error("__________ syncWithDirectory: there was a problem during sync to Directory: " + errorMessage); - return false; + failed = true; } } + if (failed) + return false; operationOutcomes = sync.updateAllBiobanksOnFhirServerIfNecessary(); for (OperationOutcome operationOutcome : operationOutcomes) { String errorMessage = getErrorMessageFromOperationOutcome(operationOutcome); if (errorMessage.length() > 0) { logger.error("__________ syncWithDirectory: there was a problem during sync from Directory: " + errorMessage); - return false; +// return false; } } - logger.info("__________ syncWithDirectory: completed"); + logger.info("__________ syncWithDirectory: all synchronization tasks completed"); return true; } diff --git a/src/main/java/de/samply/directory_sync_service/DirectorySyncJob.java b/src/main/java/de/samply/directory_sync_service/service/DirectorySyncJob.java similarity index 98% rename from src/main/java/de/samply/directory_sync_service/DirectorySyncJob.java rename to src/main/java/de/samply/directory_sync_service/service/DirectorySyncJob.java index 16c3815..4314589 100644 --- a/src/main/java/de/samply/directory_sync_service/DirectorySyncJob.java +++ b/src/main/java/de/samply/directory_sync_service/service/DirectorySyncJob.java @@ -1,4 +1,4 @@ -package de.samply.directory_sync_service; +package de.samply.directory_sync_service.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/de/samply/directory_sync_service/DirectorySyncLauncher.java b/src/main/java/de/samply/directory_sync_service/service/DirectorySyncLauncher.java similarity index 98% rename from src/main/java/de/samply/directory_sync_service/DirectorySyncLauncher.java rename to src/main/java/de/samply/directory_sync_service/service/DirectorySyncLauncher.java index 3a7e430..9f64c23 100644 --- a/src/main/java/de/samply/directory_sync_service/DirectorySyncLauncher.java +++ b/src/main/java/de/samply/directory_sync_service/service/DirectorySyncLauncher.java @@ -1,4 +1,4 @@ -package de.samply.directory_sync_service; +package de.samply.directory_sync_service.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/de/samply/directory_sync_service/DirectorySyncService.java b/src/main/java/de/samply/directory_sync_service/service/DirectorySyncService.java similarity index 98% rename from src/main/java/de/samply/directory_sync_service/DirectorySyncService.java rename to src/main/java/de/samply/directory_sync_service/service/DirectorySyncService.java index fd5a944..56438f6 100644 --- a/src/main/java/de/samply/directory_sync_service/DirectorySyncService.java +++ b/src/main/java/de/samply/directory_sync_service/service/DirectorySyncService.java @@ -1,4 +1,4 @@ -package de.samply.directory_sync_service; +package de.samply.directory_sync_service.service; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; diff --git a/src/main/java/de/samply/directory_sync_service/sync/Sync.java b/src/main/java/de/samply/directory_sync_service/sync/Sync.java new file mode 100644 index 0000000..2a2692a --- /dev/null +++ b/src/main/java/de/samply/directory_sync_service/sync/Sync.java @@ -0,0 +1,441 @@ +package de.samply.directory_sync_service.sync; + +import ca.uhn.fhir.context.FhirContext; +import de.samply.directory_sync_service.Util; +import de.samply.directory_sync_service.converter.FhirCollectionToDirectoryCollectionPutConverter; +import de.samply.directory_sync_service.directory.CreateFactTablesFromStarModelInputData; +import de.samply.directory_sync_service.directory.DirectoryApi; +import de.samply.directory_sync_service.directory.DirectoryService; +import de.samply.directory_sync_service.directory.MergeDirectoryCollectionGetToDirectoryCollectionPut; +import de.samply.directory_sync_service.directory.model.BbmriEricId; +import de.samply.directory_sync_service.directory.model.Biobank; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionGet; +import de.samply.directory_sync_service.directory.model.DirectoryCollectionPut; +import de.samply.directory_sync_service.fhir.FhirApi; +import de.samply.directory_sync_service.fhir.FhirReporting; +import de.samply.directory_sync_service.fhir.model.FhirCollection; +import de.samply.directory_sync_service.converter.FhirToDirectoryAttributeConverter; +import de.samply.directory_sync_service.model.StarModelData; +import io.vavr.control.Either; +import io.vavr.control.Option; +import java.util.Map; +import java.util.Objects; + +import org.hl7.fhir.r4.model.OperationOutcome; +import org.hl7.fhir.r4.model.Organization; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity.ERROR; +import static org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity.INFORMATION; + +/** + * Provides functionality to synchronize between a BBMRI Directory instance and a FHIR store in both directions. + * This class provides methods to update biobanks, synchronize collection sizes, generate diagnosis corrections, + * send star model updates, and perform aggregated updates to the Directory service based on information from + * the FHIR store. + * + * Usage: + * + * You will need to first do the initial set up of the Sync class, which includes + * connecting to the FHIR store and to the Directory. Your code might look something like + * this: + * + * DirectoryService directoryService = new DirectoryService(DirectoryApi.createWithLogin(HttpClients.createDefault(), directoryUrl, directoryUserName, directoryPassCode)); + * FhirReporting fhirReporting = new FhirReporting(ctx, fhirApi); + * Sync sync = new Sync(fhirApi, fhirReporting, directoryApi, directoryService); + * sync.initResources() + * + * Next, if your FHIR store does not use WHO ICD 10 codes for diagnosis, you should + * first generate a map, mapping your local ICD 10 codes onto WHO, which are used by + * the Directory: + * + * sync.generateDiagnosisCorrections(directoryDefaultCollectionId); // directoryDefaultCollectionId may be null + * + * Now you can start to do some synchronization, e.g.: + * + * Only send the collection sizes to the Directory (deprecated): + * sync.syncCollectionSizesToDirectory + * + * Send all standard attributes to Directory: + * sync.sendUpdatesToDirectory(directoryDefaultCollectionId); + * + * Send star model to Directory: + * sync.sendStarModelUpdatesToDirectory(directoryDefaultCollectionId, directoryMinDonors); // e.g. directoryMinDonors=10 + * + * Get biobank information from Directory and put into local FHIR store: + * sync.updateAllBiobanksOnFhirServerIfNecessary(); + */ +public class Sync { + private static final Logger logger = LoggerFactory.getLogger(Sync.class); + + private static final Function UPDATE_BIOBANK_NAME = t -> { + t.fhirBiobank.setName(t.dirBiobank.getName()); + return t; + }; + + private final FhirApi fhirApi; + private final FhirReporting fhirReporting; + private DirectoryApi directoryApi; + private final DirectoryService directoryService; + + public Sync(FhirApi fhirApi, FhirReporting fhirReporting, DirectoryApi directoryApi, + DirectoryService directoryService) { + this.fhirApi = fhirApi; + this.fhirReporting = fhirReporting; + this.directoryApi = directoryApi; + this.directoryService = directoryService; + } + + public static void main(String[] args) { + FhirContext fhirContext = FhirContext.forR4(); + FhirApi fhirApi = new FhirApi(fhirContext.newRestfulGenericClient(args[0])); + FhirReporting fhirReporting = new FhirReporting(fhirContext, fhirApi); + Sync sync = new Sync(fhirApi, fhirReporting, null, null); + Either result = sync.initResources(); + System.out.println("result = " + result); + Either> collectionSizes = fhirReporting.fetchCollectionSizes(); + System.out.println("collectionSizes = " + collectionSizes); + } + + /** + * Initializes necessary resources for the synchronization process. + * + * @return An {@link Either} containing either a success message or an error message. + */ + public Either initResources() { + logger.info("initResources: Initializes necessary resources for the synchronization process"); + return fhirReporting.initLibrary().flatMap(_void -> fhirReporting.initMeasure()); + } + + private static OperationOutcome missingIdentifierOperationOutcome() { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR).setDiagnostics("No BBMRI Identifier for Organization"); + return outcome; + } + + private static OperationOutcome noUpdateNecessaryOperationOutcome() { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(INFORMATION).setDiagnostics("No Update " + + "necessary"); + return outcome; + } + + /** + * Updates all biobanks from the FHIR server with information from the Directory. + * + * @return the individual {@link OperationOutcome}s from each update + */ + public List updateAllBiobanksOnFhirServerIfNecessary() { + return fhirApi.listAllBiobanks() + .map(orgs -> orgs.stream().map(this::updateBiobankOnFhirServerIfNecessary).collect(Collectors.toList())) + .fold(Collections::singletonList, Function.identity()); + } + +// /** +// * Takes a biobank from FHIR and updates it with current information from the Directory. +// * +// * @param fhirBiobank the biobank to update. +// * @return the {@link OperationOutcome} from the FHIR server update +// */ +// OperationOutcome updateBiobankOnFhirServerIfNecessary(Organization fhirBiobank) { +// return Option.ofOptional(FhirApi.bbmriEricId(fhirBiobank)) +// .toEither(missingIdentifierOperationOutcome()) +// .flatMap(directoryApi::fetchBiobank) +// .map(dirBiobank -> new BiobankTuple(fhirBiobank, dirBiobank)) +// .map(UPDATE_BIOBANK_NAME) +// .filterOrElse(BiobankTuple::hasChanged, tuple -> noUpdateNecessaryOperationOutcome()) +// .map(tuple -> fhirApi.updateResource(tuple.fhirBiobank)) +// .fold(Function.identity(), Function.identity()); +// } + /** + * Takes a biobank from FHIR and updates it with current information from the Directory. + * + * @param fhirBiobank the biobank to update. + * @return the {@link OperationOutcome} from the FHIR server update + */ + OperationOutcome updateBiobankOnFhirServerIfNecessary(Organization fhirBiobank) { + logger.info("updateBiobankOnFhirServerIfNecessary: Step 1: Retrieve the biobank's BBMRI-ERIC identifier from the FHIR organization"); + + // Step 1: Retrieve the biobank's BBMRI-ERIC identifier from the FHIR organization + Optional bbmriEricIdOpt = FhirApi.bbmriEricId(fhirBiobank); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 2: Convert the Optional to an Either type"); + + // Step 2: Convert the Optional to an Either type + Either bbmriEricIdEither = Option.ofOptional(bbmriEricIdOpt) + .toEither(missingIdentifierOperationOutcome()); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 3: Fetch the corresponding biobank from the Directory API"); + + // Step 3: Fetch the corresponding biobank from the Directory API + Either dirBiobankEither = bbmriEricIdEither + .flatMap(directoryApi::fetchBiobank); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 4: Create a BiobankTuple containing the FHIR biobank and the Directory biobank"); + + // Step 4: Create a BiobankTuple containing the FHIR biobank and the Directory biobank + Either biobankTupleEither = dirBiobankEither + .map(dirBiobank -> new BiobankTuple(fhirBiobank, dirBiobank)); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 5: Update the biobank name if necessary"); + + // Step 5: Update the biobank name if necessary + Either updatedBiobankTupleEither = biobankTupleEither + .map(UPDATE_BIOBANK_NAME); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 6: Check if any changes have been made; if not, return a no-update necessary outcome"); + + // Step 6: Check if any changes have been made; if not, return a no-update necessary outcome + Either finalBiobankTupleEither = updatedBiobankTupleEither + .filterOrElse(BiobankTuple::hasChanged, tuple -> noUpdateNecessaryOperationOutcome()); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 7: Update the biobank resource on the FHIR server if changes were made"); + + // Step 7: Update the biobank resource on the FHIR server if changes were made + Either updateOutcomeEither = finalBiobankTupleEither + .map(tuple -> fhirApi.updateResource(tuple.fhirBiobank)); + + logger.info("updateBiobankOnFhirServerIfNecessary: Step 8: Return the result of the update, folding Either to an OperationOutcome"); + + // Step 8: Return the result of the update, folding Either to an OperationOutcome + return updateOutcomeEither.fold(Function.identity(), Function.identity()); + } + + /** + * Updates collection sample count information for all collections that exist with the same + * BBMRI-ERIC identifier on the FHIR server and in the directory. + * + * @return the outcome of the directory update operation. + */ + public List syncCollectionSizesToDirectory() { + return fhirReporting.fetchCollectionSizes() + .map(directoryService::updateCollectionSizes) + .fold(Collections::singletonList, Function.identity()); + } + + private Map correctedDiagnoses = null; + + /** + * Generates corrections to the diagnoses obtained from the FHIR store, to make them + * compatible with the Directory. You should supply this method with an empty map + * via the correctedDiagnoses variable. This map will be filled by the method and + * you can subsequently use it elsewhere. + * + * This method performs the following steps: + * + * * Retrieves diagnoses from the FHIR store for specimens with identifiable collections and their associated patients. + * * Converts raw ICD-10 codes into MIRIAM-compatible codes. + * * Collects corrected diagnosis codes from the Directory API based on the MIRIAM-compatible codes. + * + * @param defaultCollectionId Default collection ID. May be null. + * @return A list containing a single OperationOutcome indicating the success of the diagnosis corrections process. + * If any errors occur during the process, an OperationOutcome with error details is returned. + */ + public List generateDiagnosisCorrections(String defaultCollectionId) { + correctedDiagnoses = new HashMap(); + try { + // Convert string version of collection ID into a BBMRI ERIC ID. + BbmriEricId defaultBbmriEricCollectionId = BbmriEricId + .valueOf(defaultCollectionId) + .orElse(null); + + // Get all diagnoses from the FHIR store for specemins with identifiable + // collections and their associated patients. + Either> fhirDiagnosesOutcome = fhirReporting.fetchDiagnoses(defaultBbmriEricCollectionId); + if (fhirDiagnosesOutcome.isLeft()) + return createErrorOutcome("Problem getting diagnosis information from FHIR store, " + errorMessageFromOperationOutcome(fhirDiagnosesOutcome.getLeft())); + List fhirDiagnoses = fhirDiagnosesOutcome.get(); + logger.info("__________ generateDiagnosisCorrections: fhirDiagnoses.size(): " + fhirDiagnoses.size()); + + // Convert the raw ICD 10 codes into MIRIAM-compatible codes and put the + // codes into a map with identical keys and values. + fhirDiagnoses.forEach(diagnosis -> { + String miriamDiagnosis = FhirToDirectoryAttributeConverter.convertDiagnosis(diagnosis); + correctedDiagnoses.put(miriamDiagnosis, miriamDiagnosis); + }); + logger.info("__________ generateDiagnosisCorrections: 1 correctedDiagnoses.size(): " + correctedDiagnoses.size()); + + // Get corrected diagnosis codes from the Directory + directoryApi.collectDiagnosisCorrections(correctedDiagnoses); + logger.info("__________ generateDiagnosisCorrections: 2 correctedDiagnoses.size(): " + correctedDiagnoses.size()); + + // Return a successful outcome. + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(INFORMATION).setDiagnostics("Diagnosis corrections generated successfully"); + return Collections.singletonList(outcome); + } catch (Exception e) { + return createErrorOutcome("generateDiagnosisCorrections - unexpected error: " + Util.traceFromException(e)); + } + } + + /** + * Sends updates for Star Model data to the Directory service, based on FHIR store information. + * This method fetches Star Model input data from the FHIR store, generates star model fact tables, + * performs diagnosis corrections, and then updates the Directory service with the prepared data. + *

+ * The method handles errors by returning a list of OperationOutcome objects describing the issues. + *

+ * + * @param defaultCollectionId The default BBMRI-ERIC collection ID for fetching data from the FHIR store. + * @param minDonors The minimum number of donors required for a fact to be included in the star model output. + * @param maxFacts The maximum number of facts to be included in the star model output. Negative number means no limit. + * @return A list of OperationOutcome objects indicating the outcome of the star model updates. + * + * @throws IllegalArgumentException if the defaultCollectionId is not a valid BbmriEricId. + */ + public List sendStarModelUpdatesToDirectory(String defaultCollectionId, int minDonors, int maxFacts) { + logger.info("__________ sendStarModelUpdatesToDirectory: minDonors: " + minDonors); + try { + BbmriEricId defaultBbmriEricCollectionId = BbmriEricId + .valueOf(defaultCollectionId) + .orElse(null); + + // Pull data from the FHIR store and save it in a format suitable for generating + // star model hypercubes. + Either starModelInputDataOutcome = fhirReporting.fetchStarModelInputData(defaultBbmriEricCollectionId); + if (starModelInputDataOutcome.isLeft()) + return createErrorOutcome("Problem getting star model information from FHIR store, " + errorMessageFromOperationOutcome(starModelInputDataOutcome.getLeft())); + StarModelData starModelInputData = starModelInputDataOutcome.get(); + logger.info("__________ sendStarModelUpdatesToDirectory: number of collection IDs: " + starModelInputData.getInputCollectionIds().size()); + + relogin(); + + // Hypercubes containing less than the minimum number of donors will not be + // included in the star model output. + starModelInputData.setMinDonors(minDonors); + + // Take the patient list and the specimen list from starModelInputData and + // use them to generate the star model fact tables. + CreateFactTablesFromStarModelInputData.createFactTables(starModelInputData, maxFacts); + logger.info("__________ sendStarModelUpdatesToDirectory: 1 starModelInputData.getFactCount(): " + starModelInputData.getFactCount()); + + // Apply corrections to ICD 10 diagnoses, to make them compatible with + // the Directory. + if (correctedDiagnoses != null) + starModelInputData.applyDiagnosisCorrections(correctedDiagnoses); + logger.info("__________ sendStarModelUpdatesToDirectory: 2 starModelInputData.getFactCount(): " + starModelInputData.getFactCount()); + + // Send fact tables to Direcory. + relogin(); + List starModelUpdateOutcome = directoryService.updateStarModel(starModelInputData); + logger.info("__________ sendStarModelUpdatesToDirectory: star model has been updated"); + // Return some kind of results count or whatever + return starModelUpdateOutcome; + } catch (Exception e) { + return createErrorOutcome("sendStarModelUpdatesToDirectory - unexpected error: " + Util.traceFromException(e)); + } + } + + /** + * Take information from the FHIR store and send aggregated updates to the Directory. + * + * This is a multi step process: + * 1. Fetch a list of collections objects from the FHIR store. These contain aggregated + * information over all specimens in the collections. + * 2. Convert the FHIR collection objects into Directory collection PUT DTOs. Copy + * over avaialble information from FHIR, converting where necessary. + * 3. Using the collection IDs found in the FHIR store, send queries to the Directory + * and fetch back the relevant GET collections. If any of the collection IDs cannot be + * found, this ie a breaking error. + * 4. Transfer data from the Directory GET collections to the corresponding Directory PUT + * collections. + * 5. Push the new information back to the Directory. + * + * @param defaultCollectionId The default collection ID to use for fetching collections from the FHIR store. + * @return A list of OperationOutcome objects indicating the outcome of the update operation. + */ + public List sendUpdatesToDirectory(String defaultCollectionId) { + try { + BbmriEricId defaultBbmriEricCollectionId = BbmriEricId + .valueOf(defaultCollectionId) + .orElse(null); + + Either> fhirCollectionOutcomes = fhirReporting.fetchFhirCollections(defaultBbmriEricCollectionId); + if (fhirCollectionOutcomes.isLeft()) + return createErrorOutcome("Problem getting collections from FHIR store, " + errorMessageFromOperationOutcome(fhirCollectionOutcomes.getLeft())); + logger.info("__________ sendUpdatesToDirectory: FHIR collection count): " + fhirCollectionOutcomes.get().size()); + + DirectoryCollectionPut directoryCollectionPut = FhirCollectionToDirectoryCollectionPutConverter.convert(fhirCollectionOutcomes.get()); + if (directoryCollectionPut == null) + return createErrorOutcome("Problem converting FHIR attributes to Directory attributes"); + logger.info("__________ sendUpdatesToDirectory: 1 directoryCollectionPut.getCollectionIds().size()): " + directoryCollectionPut.getCollectionIds().size()); + + List collectionIds = directoryCollectionPut.getCollectionIds(); + String countryCode = directoryCollectionPut.getCountryCode(); + relogin(); + Either directoryCollectionGetOutcomes = directoryService.fetchDirectoryCollectionGetOutcomes(countryCode, collectionIds); + if (directoryCollectionGetOutcomes.isLeft()) + return createErrorOutcome("Problem getting collections from Directory, " + errorMessageFromOperationOutcome(directoryCollectionGetOutcomes.getLeft())); + DirectoryCollectionGet directoryCollectionGet = directoryCollectionGetOutcomes.get(); + logger.info("__________ sendUpdatesToDirectory: 1 directoryCollectionGet.getItems().size()): " + directoryCollectionGet.getItems().size()); + + if (!MergeDirectoryCollectionGetToDirectoryCollectionPut.merge(directoryCollectionGet, directoryCollectionPut)) + return createErrorOutcome("Problem merging Directory GET attributes to Directory PUT attributes"); + logger.info("__________ sendUpdatesToDirectory: 2 directoryCollectionGet.getItems().size()): " + directoryCollectionGet.getItems().size()); + + // Apply corrections to ICD 10 diagnoses, to make them compatible with + // the Directory. + if (correctedDiagnoses != null) + directoryCollectionPut.applyDiagnosisCorrections(correctedDiagnoses); + logger.info("__________ sendUpdatesToDirectory: 2 directoryCollectionPut.getCollectionIds().size()): " + directoryCollectionPut.getCollectionIds().size()); + + relogin(); + List outcomes = directoryService.updateEntities(directoryCollectionPut); + logger.info("__________ sendUpdatesToDirectory: 2 outcomes: " + outcomes); + return outcomes; + } catch (Exception e) { + return createErrorOutcome("sendUpdatesToDirectory - unexpected error: " + Util.traceFromException(e)); + } + } + + /** + * Renew the Directory login. + * + * This generates a new DirectoryApi object, which needs to be distributed to the places + * where it will be used. + * + * Consequence: this method has significant side effects. + */ + private void relogin() { + directoryApi = directoryApi.relogin(); + directoryService.setApi(directoryApi); + } + + private String errorMessageFromOperationOutcome(OperationOutcome operationOutcome) { + return operationOutcome.getIssue().stream() + .filter(issue -> issue.getSeverity() == OperationOutcome.IssueSeverity.ERROR || issue.getSeverity() == OperationOutcome.IssueSeverity.FATAL) + .map(OperationOutcome.OperationOutcomeIssueComponent::getDiagnostics) + .collect(Collectors.joining("\n")); + } + + private List createErrorOutcome(String diagnostics) { + OperationOutcome outcome = new OperationOutcome(); + outcome.addIssue().setSeverity(ERROR).setDiagnostics(diagnostics); + return Collections.singletonList(outcome); + } + + private static class BiobankTuple { + + private final Organization fhirBiobank; + private final Organization fhirBiobankCopy; + private final Biobank dirBiobank; + + private BiobankTuple(Organization fhirBiobank, Biobank dirBiobank) { + this.fhirBiobank = Objects.requireNonNull(fhirBiobank); + this.fhirBiobankCopy = fhirBiobank.copy(); + this.dirBiobank = Objects.requireNonNull(dirBiobank); + } + + private boolean hasChanged() { + return !fhirBiobank.equalsDeep(fhirBiobankCopy); + } + } +} diff --git a/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Library.json b/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Library.json new file mode 100644 index 0000000..caaa83f --- /dev/null +++ b/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Library.json @@ -0,0 +1,23 @@ +{ + "resourceType": "Library", + "id": "CollectionSize", + "name": "CollectionSize", + "url": "https://fhir.bbmri.de/Library/collection-size", + "status": "active", + "type": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/library-type", + "code": "logic-library" + } + ] + }, + "subjectCodeableConcept": { + "coding": [ + { + "system": "http://hl7.org/fhir/resource-types", + "code": "Specimen" + } + ] + } +} diff --git a/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Measure.json b/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Measure.json new file mode 100644 index 0000000..6e3fc48 --- /dev/null +++ b/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.Measure.json @@ -0,0 +1,57 @@ +{ + "resourceType": "Measure", + "id": "CollectionSize", + "name": "CollectionSize", + "url": "https://fhir.bbmri.de/Measure/collection-size", + "status": "active", + "subjectCodeableConcept": { + "coding": [ + { + "system": "http://hl7.org/fhir/resource-types", + "code": "Specimen" + } + ] + }, + "library": [ + "https://fhir.bbmri.de/Library/collection-size" + ], + "scoring": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/measure-scoring", + "code": "cohort" + } + ] + }, + "group": [ + { + "population": [ + { + "code": { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/measure-population", + "code": "initial-population" + } + ] + }, + "criteria": { + "language": "text/cql", + "expression": "InInitialPopulation" + } + } + ], + "stratifier": [ + { + "code": { + "text": "Collection" + }, + "criteria": { + "language": "text/cql", + "expression": "Collection" + } + } + ] + } + ] +} diff --git a/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.cql b/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.cql new file mode 100644 index 0000000..0fe928e --- /dev/null +++ b/src/main/resources/de/samply/directory_sync_service/fhir/CollectionSize.cql @@ -0,0 +1,13 @@ +library Retrieve +using FHIR version '4.0.0' +include FHIRHelpers version '4.0.0' + +context Specimen + +define InInitialPopulation: + true + +define Collection: + First(from Specimen.extension E + where E.url = 'https://fhir.bbmri.de/StructureDefinition/Custodian' + return (E.value as Reference).reference)