ISTO-61 More lenient FHIR package import.

- Use the package index file as the source of truth for the url and version of CodeSystems imported. This avoids some duplicates in the HL7 package.
- Automatically skip importing a duplicate CodeSystem version if it has "content:not-present". This case is logged at INFO level.

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## Beta Fix Release 8.1.1 - FHIR Multiple Code Systems and Packages (fix)
+This beta release follows Beta 8.1.0 and contains fixes needed when importing all the CodeSystems within a FHIR package that contains
+potentially duplicate CodeSystem versions. This is required to import some HL7 packages, for example `hl7.terminology.r4-3.1.0.tgz`. 
+
+### Fixes
+  - ISTO-61 More lenient FHIR package import
+    - Use the package index file as the source of truth for the url and version of CodeSystems imported. This avoids some duplicates in the HL7 package.
+    - Automatically skip importing a duplicate CodeSystem version if it has "content:not-present". This case is logged at INFO level.
+  - Make logging less noisy during FHIR package import
+
+
 ## Beta Release 8.1.0 - FHIR Multiple Code Systems and Packages
 This beta release builds on Beta 8.0.0, adding support for loading code systems from FHIR packages.  
 See [Loading FHIR Packages documentation](https://github.com/IHTSDO/snowstorm-x/blob/snowstorm-x-8.1.0/docs/using-the-fhir-api.md#loading-fhir-packages) for details of how to load HL7 code systems and others.
@@ -46,6 +57,7 @@ Please refer to the [updated v8 FHIR API documentation](https://github.com/IHTSD
 - Minor fixes for FHIR specification conformance
 
 
+
 ## 7.9.3 Release (June 2022)
 ### Breaking
   - ISTO-16 Add support for ECL 2.0.

src/main/java/org/snomed/snowstorm/fhir/services/FHIRCodeSystemService.java

@@ -58,7 +58,7 @@ public FHIRCodeSystemVersion save(CodeSystem codeSystem) {
 		}
 
 		wrap(fhirCodeSystemVersion);
-		logger.info("Saving fhir code system '{}'.", fhirCodeSystemVersion.getId());
+		logger.debug("Saving fhir code system '{}'.", fhirCodeSystemVersion.getId());
 		codeSystemRepository.save(fhirCodeSystemVersion);
 		return fhirCodeSystemVersion;
 	}

src/main/java/org/snomed/snowstorm/fhir/services/FHIRConceptService.java

@@ -99,7 +99,9 @@ private void saveAllConceptsOfCodeSystemVersion(FHIRCodeSystemVersion codeSystem
 				}
 			}
 			// Add parent and child properties if missing
-			Map<String, String> conceptDisplayMap = concepts.stream().collect(Collectors.toMap(FHIRConcept::getCode, FHIRConcept::getDisplay));
+			Map<String, String> conceptDisplayMap = concepts.stream()
+					.filter(concept -> concept.getDisplay() != null)
+					.collect(Collectors.toMap(FHIRConcept::getCode, FHIRConcept::getDisplay));
 			for (FHIRConcept concept : concepts) {
 				Map<String, List<FHIRProperty>> properties = concept.getProperties();
 				Collection<String> parents = graphBuilder.getNodeParents(concept.getCode());
@@ -139,7 +141,7 @@ private void saveAllConceptsOfCodeSystemVersion(FHIRCodeSystemVersion codeSystem
 				}
 			}
 			conceptRepository.saveAll(batch);
-			if (percentToLog != null) {
+			if (concepts.size() > 1000 && percentToLog != null) {
 				logger.info("Saved {}% of '{}' fhir concepts.", Math.round(percentToLog), idWithVersion);
 				percentToLog = null;
 			}
@@ -152,9 +154,8 @@ private void saveAllConceptsOfCodeSystemVersion(FHIRCodeSystemVersion codeSystem
 	public void deleteExistingCodes(String idWithVersion) {
 		Page<FHIRConcept> existingConcepts = conceptRepository.findByCodeSystemVersion(idWithVersion, PageRequest.of(0, 1));
 		long totalExisting = existingConcepts.getTotalElements();
-		logger.info("Found {} existing concepts for this code system version {}", totalExisting, idWithVersion);
 		if (totalExisting > 0) {
-			logger.info("Deleting existing codes...");
+			logger.info("Deleting {} existing concepts for this code system version {}", totalExisting, idWithVersion);
 			// Deleting by query often seems to exceed the default 30 second query timeout so we will page through them...
 			Page<FHIRConcept> codesToDelete = conceptRepository.findByCodeSystemVersion(idWithVersion, PageRequest.of(0, DELETE_BATCH_SIZE));
 			while (!codesToDelete.isEmpty()) {

src/main/java/org/snomed/snowstorm/fhir/services/FHIRLoadPackageService.java

@@ -58,19 +58,27 @@ public void uploadPackageResources(File packageFile, Set<String> resourceUrlsToI
 		logger.info("Importing {} resources, found within index of package {}.", filesToImport.size(), packageFile.getName());
 
 		for (FHIRPackageIndexFile indexFileToImport : filesToImport) {
-			if (indexFileToImport.getResourceType().equals("CodeSystem")) {
+			String id = indexFileToImport.getId();
+			String url = indexFileToImport.getUrl();
+			if (indexFileToImport.getResourceType().equals("CodeSystem") && id != null && url != null) {
 				CodeSystem codeSystem = extractObject(packageFile, indexFileToImport.getFilename(), CodeSystem.class, jsonParser);
+				codeSystem.setId(id);
+				codeSystem.setUrl(url);
 				if (FHIRHelper.isSnomedUri(codeSystem.getUrl())) {
 					logger.info("Skipping import of SNOMED CT code system via package. Please use the native SNOMED-CT API RF2 import.");
 					continue;
 				}
-				String url = indexFileToImport.getUrl();
 				String version = indexFileToImport.getVersion();
 				FHIRCodeSystemVersion existingCodeSystemVersion = codeSystemService.findCodeSystemVersion(new FHIRCodeSystemVersionParams(url).setVersion(version));
 				if (existingCodeSystemVersion != null) {
-					throw FHIRHelper.exception(format("Resource %s/%s already exists. Please delete this version before attempting to import.",
-									"CodeSystem", existingCodeSystemVersion.getId()),
-							OperationOutcome.IssueType.NOTSUPPORTED, 400);
+					if (codeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
+						logger.info("Skipping import of CodeSystem %s with 'content:not-present' because a CodeSystem with the same url and version already exists.");
+					} else {
+						throw FHIRHelper.exception(format("Resource %s with url '%s' and version '%s' already exists it has id '%s'. " +
+												"Please delete this version before attempting to import.",
+										"CodeSystem", url, version, existingCodeSystemVersion.getId()),
+								OperationOutcome.IssueType.NOTSUPPORTED, 400);
+					}
 				}
 				List<CodeSystem.ConceptDefinitionComponent> concepts = codeSystem.getConcept();
 				logger.info("Importing CodeSystem {} with {} concepts from package", codeSystem.getUrl(), concepts != null ? concepts.size() : 0);
@@ -80,6 +88,7 @@ public void uploadPackageResources(File packageFile, Set<String> resourceUrlsToI
 				}
 			}
 		}
+		logger.info("Completed import of package {}.", packageFile.getName());
 	}
 
 	private static void validateResources(List<FHIRPackageIndexFile> filesToImport, Set<String> resourceUrlsToImport, boolean importAll, Set<String> supportedResourceTypes) {
@@ -113,7 +122,6 @@ private <T> T extractObject(File packageFile, String archiveEntryName, Class<T>
 						return mapper.readValue(tarIn, clazz);
 					} else {
 						IBaseResource iBaseResource = jsonParser.parseResource(tarIn);
-						System.out.println(iBaseResource.getClass());
 						return (T) iBaseResource;
 					}
 				}