From d4941e08adeac2a76eb050468eab492ba3e9b1a9 Mon Sep 17 00:00:00 2001 From: Kai Kewley Date: Fri, 16 Sep 2022 11:04:42 +0100 Subject: [PATCH] ISTO-61 More lenient FHIR package import. - Use the package index file as the source of truth for the url and version of CodeSystems imported. This avoids some duplicates in the HL7 package. - Automatically skip importing a duplicate CodeSystem version if it has "content:not-present". This case is logged at INFO level. --- CHANGELOG.md | 12 +++++++++++ .../fhir/services/FHIRCodeSystemService.java | 2 +- .../fhir/services/FHIRConceptService.java | 9 +++++---- .../fhir/services/FHIRLoadPackageService.java | 20 +++++++++++++------ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b27354ea0..c7a18f04d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Beta Fix Release 8.1.1 - FHIR Multiple Code Systems and Packages (fix) +This beta release follows Beta 8.1.0 and contains fixes needed when importing all the CodeSystems within a FHIR package that contains +potentially duplicate CodeSystem versions. This is required to import some HL7 packages, for example `hl7.terminology.r4-3.1.0.tgz`. + +### Fixes + - ISTO-61 More lenient FHIR package import + - Use the package index file as the source of truth for the url and version of CodeSystems imported. This avoids some duplicates in the HL7 package. + - Automatically skip importing a duplicate CodeSystem version if it has "content:not-present". This case is logged at INFO level. + - Make logging less noisy during FHIR package import + + ## Beta Release 8.1.0 - FHIR Multiple Code Systems and Packages This beta release builds on Beta 8.0.0, adding support for loading code systems from FHIR packages. See [Loading FHIR Packages documentation](https://github.com/IHTSDO/snowstorm-x/blob/snowstorm-x-8.1.0/docs/using-the-fhir-api.md#loading-fhir-packages) for details of how to load HL7 code systems and others. @@ -46,6 +57,7 @@ Please refer to the [updated v8 FHIR API documentation](https://github.com/IHTSD - Minor fixes for FHIR specification conformance + ## 7.9.3 Release (June 2022) ### Breaking - ISTO-16 Add support for ECL 2.0. diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRCodeSystemService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRCodeSystemService.java index 142b4fbf0..b1b00ae8f 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRCodeSystemService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRCodeSystemService.java @@ -58,7 +58,7 @@ public FHIRCodeSystemVersion save(CodeSystem codeSystem) { } wrap(fhirCodeSystemVersion); - logger.info("Saving fhir code system '{}'.", fhirCodeSystemVersion.getId()); + logger.debug("Saving fhir code system '{}'.", fhirCodeSystemVersion.getId()); codeSystemRepository.save(fhirCodeSystemVersion); return fhirCodeSystemVersion; } diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java index 35de836d1..52e094d1a 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java @@ -99,7 +99,9 @@ private void saveAllConceptsOfCodeSystemVersion(FHIRCodeSystemVersion codeSystem } } // Add parent and child properties if missing - Map conceptDisplayMap = concepts.stream().collect(Collectors.toMap(FHIRConcept::getCode, FHIRConcept::getDisplay)); + Map conceptDisplayMap = concepts.stream() + .filter(concept -> concept.getDisplay() != null) + .collect(Collectors.toMap(FHIRConcept::getCode, FHIRConcept::getDisplay)); for (FHIRConcept concept : concepts) { Map> properties = concept.getProperties(); Collection parents = graphBuilder.getNodeParents(concept.getCode()); @@ -139,7 +141,7 @@ private void saveAllConceptsOfCodeSystemVersion(FHIRCodeSystemVersion codeSystem } } conceptRepository.saveAll(batch); - if (percentToLog != null) { + if (concepts.size() > 1000 && percentToLog != null) { logger.info("Saved {}% of '{}' fhir concepts.", Math.round(percentToLog), idWithVersion); percentToLog = null; } @@ -152,9 +154,8 @@ private void saveAllConceptsOfCodeSystemVersion(FHIRCodeSystemVersion codeSystem public void deleteExistingCodes(String idWithVersion) { Page existingConcepts = conceptRepository.findByCodeSystemVersion(idWithVersion, PageRequest.of(0, 1)); long totalExisting = existingConcepts.getTotalElements(); - logger.info("Found {} existing concepts for this code system version {}", totalExisting, idWithVersion); if (totalExisting > 0) { - logger.info("Deleting existing codes..."); + logger.info("Deleting {} existing concepts for this code system version {}", totalExisting, idWithVersion); // Deleting by query often seems to exceed the default 30 second query timeout so we will page through them... Page codesToDelete = conceptRepository.findByCodeSystemVersion(idWithVersion, PageRequest.of(0, DELETE_BATCH_SIZE)); while (!codesToDelete.isEmpty()) { diff --git a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRLoadPackageService.java b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRLoadPackageService.java index 75fd5b008..3bb8363cc 100644 --- a/src/main/java/org/snomed/snowstorm/fhir/services/FHIRLoadPackageService.java +++ b/src/main/java/org/snomed/snowstorm/fhir/services/FHIRLoadPackageService.java @@ -58,19 +58,27 @@ public void uploadPackageResources(File packageFile, Set resourceUrlsToI logger.info("Importing {} resources, found within index of package {}.", filesToImport.size(), packageFile.getName()); for (FHIRPackageIndexFile indexFileToImport : filesToImport) { - if (indexFileToImport.getResourceType().equals("CodeSystem")) { + String id = indexFileToImport.getId(); + String url = indexFileToImport.getUrl(); + if (indexFileToImport.getResourceType().equals("CodeSystem") && id != null && url != null) { CodeSystem codeSystem = extractObject(packageFile, indexFileToImport.getFilename(), CodeSystem.class, jsonParser); + codeSystem.setId(id); + codeSystem.setUrl(url); if (FHIRHelper.isSnomedUri(codeSystem.getUrl())) { logger.info("Skipping import of SNOMED CT code system via package. Please use the native SNOMED-CT API RF2 import."); continue; } - String url = indexFileToImport.getUrl(); String version = indexFileToImport.getVersion(); FHIRCodeSystemVersion existingCodeSystemVersion = codeSystemService.findCodeSystemVersion(new FHIRCodeSystemVersionParams(url).setVersion(version)); if (existingCodeSystemVersion != null) { - throw FHIRHelper.exception(format("Resource %s/%s already exists. Please delete this version before attempting to import.", - "CodeSystem", existingCodeSystemVersion.getId()), - OperationOutcome.IssueType.NOTSUPPORTED, 400); + if (codeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) { + logger.info("Skipping import of CodeSystem %s with 'content:not-present' because a CodeSystem with the same url and version already exists."); + } else { + throw FHIRHelper.exception(format("Resource %s with url '%s' and version '%s' already exists it has id '%s'. " + + "Please delete this version before attempting to import.", + "CodeSystem", url, version, existingCodeSystemVersion.getId()), + OperationOutcome.IssueType.NOTSUPPORTED, 400); + } } List concepts = codeSystem.getConcept(); logger.info("Importing CodeSystem {} with {} concepts from package", codeSystem.getUrl(), concepts != null ? concepts.size() : 0); @@ -80,6 +88,7 @@ public void uploadPackageResources(File packageFile, Set resourceUrlsToI } } } + logger.info("Completed import of package {}.", packageFile.getName()); } private static void validateResources(List filesToImport, Set resourceUrlsToImport, boolean importAll, Set supportedResourceTypes) { @@ -113,7 +122,6 @@ private T extractObject(File packageFile, String archiveEntryName, Class return mapper.readValue(tarIn, clazz); } else { IBaseResource iBaseResource = jsonParser.parseResource(tarIn); - System.out.println(iBaseResource.getClass()); return (T) iBaseResource; } }