From d21dfeafee96f7e2a7d60be4c83690c9726ee753 Mon Sep 17 00:00:00 2001 From: Martin Ndegwa Date: Wed, 26 Apr 2023 09:56:48 +0300 Subject: [PATCH 1/2] Bug Fixes and Enhancements - Improved output documentation - Refactored processing per files - Fixed null pointer on processing poorly formatted json files - Optimization by index configuration by filetype(ext) - Bump up release version to version 2.1.0 --- README.md | 4 +- pom.xml | 2 +- src/main/java/org/smartregister/Main.java | 4 +- .../java/org/smartregister/util/FCTUtils.java | 11 ++- .../util/FCTValidationEngine.java | 72 +++++++++++++------ .../util/QuestionnaireProcessor.java | 13 +++- .../util/StructureMapProcessor.java | 5 +- 7 files changed, 79 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 86a3c91..ce72184 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ A command line utility to support FHIRCore content authoring. This tool supports Download the latest release from https://github.com/opensrp/fhircore-tooling/releases -To run it as a java jar by using the command `java -jar efsity-2.0.0.jar -h` . This is the help command and will list the available options. +To run it as a java jar by using the command `java -jar efsity-2.1.0.jar -h` . This is the help command and will list the available options. If you are using a linux environment e.g. bash you can choose to create an _alias_ for this as shown below. _(Remember to reload the terminal)_ -`alias fct='java -jar ~/Downloads/efsity-2.0.0.jar'` +`alias fct='java -jar ~/Downloads/efsity-2.1.0.jar'` To run the previous help command you can then run `fct -h` in your terminal. diff --git a/pom.xml b/pom.xml index fe018c0..d02d9b6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.smartregister fhircore-tooling - 2.0.0 + 2.1.0 efsity diff --git a/src/main/java/org/smartregister/Main.java b/src/main/java/org/smartregister/Main.java index f5e5ca8..c20f694 100644 --- a/src/main/java/org/smartregister/Main.java +++ b/src/main/java/org/smartregister/Main.java @@ -11,7 +11,7 @@ @Command( name = "fct", description = "FHIRCore tooling to make content authoring easier.", - version = "2.0.0", + version = "2.1.0", mixinStandardHelpOptions = true, subcommands = { ConvertCommand.class, @@ -19,7 +19,7 @@ ValidateCommand.class }) public class Main implements Runnable { - public static final String VERSION = "2.0.0"; + public static final String VERSION = "2.1.0"; @CommandLine.Option( names = {"-v"}, diff --git a/src/main/java/org/smartregister/util/FCTUtils.java b/src/main/java/org/smartregister/util/FCTUtils.java index 34d2723..0184d8c 100644 --- a/src/main/java/org/smartregister/util/FCTUtils.java +++ b/src/main/java/org/smartregister/util/FCTUtils.java @@ -14,9 +14,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.apache.commons.io.FilenameUtils; import org.smartregister.domain.FCTFile; public class FCTUtils { @@ -124,8 +126,8 @@ public static void printCompletedInDuration(long startTime) { FCTUtils.getHumanDuration(System.currentTimeMillis() - startTime))); } - public static Map> indexConfigurationFiles(String inputDirectoryPath) - throws IOException { + public static Map> indexConfigurationFiles( + String inputDirectoryPath, String... fileExtensions) throws IOException { Map> filesMap = new HashMap<>(); Path rootDir = Paths.get(inputDirectoryPath); Files.walkFileTree( @@ -133,7 +135,10 @@ public static Map> indexConfigurationFiles(String in new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - if (!Files.isDirectory(file)) { + if (!Files.isDirectory(file) + && (fileExtensions.length == 1 && fileExtensions[0] == "*" + || Arrays.asList(fileExtensions) + .contains(FilenameUtils.getExtension(file.getFileName().toString())))) { String parentDirKey = file.getParent().equals(rootDir) diff --git a/src/main/java/org/smartregister/util/FCTValidationEngine.java b/src/main/java/org/smartregister/util/FCTValidationEngine.java index 7ee9138..72493d8 100644 --- a/src/main/java/org/smartregister/util/FCTValidationEngine.java +++ b/src/main/java/org/smartregister/util/FCTValidationEngine.java @@ -25,12 +25,12 @@ public class FCTValidationEngine { private JSONObject currentParamDataWorkflowJSONObject; private JSONObject lastWorkflowJSONObject; - private JSONObject previousParentJSONObject; private JSONObject parentJSONObject; private String parentJSONObjectKey; private String currentFile; private int configurationFilesCount; - private Set factMapKeys = new HashSet<>(); + + private Map> factMapKeysByFile = new HashMap<>(); private Map fileConfigTypeIdentifierToFilenameMap = new HashMap<>(); private Map> questionnairesToLinkIds; private Map> structureMapToLinkIds; @@ -47,7 +47,6 @@ private void handleValue(String key, Object value, boolean isComposition) { } else if (value instanceof JSONObject) { parentJSONObjectKey = key; - previousParentJSONObject = parentJSONObject; parentJSONObject = (JSONObject) value; if (parentJSONObject != null && parentJSONObject.has(Constants.workflow)) @@ -121,22 +120,24 @@ private void handleValue(String key, Object value, boolean isComposition) { if (value.toString().contains("data.put(")) { String ruleFactKey = StringUtils.substringBetween(value.toString(), "'", "'"); - factMapKeys.add(ruleFactKey); + Set factsByFile = factMapKeysByFile.getOrDefault(currentFile, new HashSet<>()); + factsByFile.add(ruleFactKey); + factMapKeysByFile.put(currentFile, factsByFile); if (errorsMap.getOrDefault(currentFile, new HashMap<>()).containsKey(Constants.Rules)) errorsMap.get(currentFile).get(Constants.Rules).remove(ruleFactKey); } else if (value.toString().contains("@{")) { String ruleFactKey = StringUtils.substringBetween(value.toString(), "@{", "}"); - if (!factMapKeys.contains(ruleFactKey)) { + if (!factMapKeysByFile + .getOrDefault(currentFile, new HashSet<>()) + .contains(ruleFactKey)) { addToErrorMap(Constants.Rules, ruleFactKey); } } - if (Constants.PARAMDATA.equals(value)) { - - if (previousParentJSONObject != null - && previousParentJSONObject.has(Constants.workflow)) - currentParamDataWorkflowJSONObject = previousParentJSONObject; + if (Constants.workflow.equals(key)) + currentParamDataWorkflowJSONObject = lastWorkflowJSONObject; + if (Constants.PARAMDATA.equals(value)) { String configFileIdentifier = currentParamDataWorkflowJSONObject.getString(Constants.ID); @@ -150,8 +151,13 @@ private void handleValue(String key, Object value, boolean isComposition) { // Add to fact-map for this file if (currentFile.equals( - fileConfigTypeIdentifierToFilenameMap.getOrDefault(configFileIdentifier, null))) - factMapKeys.add(parentJSONObject.getString(Constants.KEY)); + fileConfigTypeIdentifierToFilenameMap.getOrDefault(configFileIdentifier, null))) { + + Set factsByFile = + factMapKeysByFile.getOrDefault(currentFile, new HashSet<>()); + factsByFile.add(parentJSONObject.getString(Constants.KEY)); + factMapKeysByFile.put(currentFile, factsByFile); + } } // @@ -190,7 +196,8 @@ private void handleValue(String key, Object value, boolean isComposition) { questionnaireId)); } - if (structureMapToLinkIds.containsKey(structureMapId) + if (structureMapId != null + && structureMapToLinkIds.containsKey(structureMapId) && !structureMapToLinkIds.get(structureMapId).contains(fieldLinkId)) { addToErrorMap( "Prepopulate", @@ -281,7 +288,7 @@ public void process( long startTime = System.currentTimeMillis(); Map> configDirIndexMap = - FCTUtils.indexConfigurationFiles(directoryPath); + FCTUtils.indexConfigurationFiles(directoryPath, "*"); if (questionnairesFolderPath != null) { @@ -296,7 +303,7 @@ public void process( structureMapToLinkIds = new StructureMapProcessor(structureMapsFolderPath).process(); FCTUtils.printNewLine(); - FCTUtils.printInfo("\u001b[36mPRE PARSING VALIDATION\u001b[0m"); + FCTUtils.printInfo("\u001b[36mPreparsing validation\u001b[0m"); // Validate all Structure Map Link ids should be in questionnaire for (var entry : structureMapToLinkIds.entrySet()) { @@ -323,11 +330,31 @@ public void process( } else { FCTUtils.printError( String.format( - "No Questionnaire found for Structure Map with id \u001b[36m%s\u001b[0m", + "No Questionnaire resource was found for the Structure Map with id \u001b[36m%s\u001b[0m", structureMapId)); } } + + // Validate all Structure Map IDs referenced in Questionnaire extensions exist as Structure + // Maps(files) + + for (var entry : questionnaireToStructureMapId.entrySet()) { + + String questionnaireId = entry.getKey(); + String structureMapId = entry.getValue().stream().findFirst().orElse(""); + + if (!structureMapId.isBlank() && !structureMapToLinkIds.containsKey(structureMapId)) { + + FCTUtils.printError( + String.format( + "No Structure Map with id \u001b[36m%s\u001b[0m was found. Defining Questionnaire is id \u001b[36m%s\u001b[0m", + structureMapId, questionnaireId)); + } + } } + + FCTUtils.printInfo("\u001b[36mPreparsing validation complete\u001b[0m"); + // Load Composition currentFile = compositionPath; FCTFile compositionFile = FCTUtils.readFile(compositionPath); @@ -365,9 +392,15 @@ public void process( } } } - - FCTUtils.printToConsole(String.format("%d translation files found", translationsMap.size())); - FCTUtils.printToConsole(String.format("%d configuration files found", configurationFilesCount)); + FCTUtils.printNewLine(); + FCTUtils.printToConsole( + String.format( + "%d translation file" + (translationsMap.size() > 1 ? "s" : "") + " found", + translationsMap.size())); + FCTUtils.printToConsole( + String.format( + "%d configuration file" + (configurationFilesCount > 1 ? "s" : "") + " found", + configurationFilesCount)); printValidationResults(); FCTUtils.printNewLine(); @@ -387,7 +420,6 @@ private String getQuestionnaireIdByStructureMapId(String structureMapId) { private void resetStatePerFile() { currentFile = null; - factMapKeys.clear(); } private void printValidationResults() { diff --git a/src/main/java/org/smartregister/util/QuestionnaireProcessor.java b/src/main/java/org/smartregister/util/QuestionnaireProcessor.java index fec73a1..50acc59 100644 --- a/src/main/java/org/smartregister/util/QuestionnaireProcessor.java +++ b/src/main/java/org/smartregister/util/QuestionnaireProcessor.java @@ -31,7 +31,7 @@ public Map>> process() { try { Map> folderTofilesIndexMap = - FCTUtils.indexConfigurationFiles(directoryPath); + FCTUtils.indexConfigurationFiles(directoryPath, "json"); // Process other configurations for (var entry : folderTofilesIndexMap.entrySet()) { @@ -69,7 +69,7 @@ public Map>> process() { } catch (JSONException jsonException) { FCTUtils.printError(String.format("Error processing file %s", currentFile)); - FCTUtils.printError(String.format("Error message %s", jsonException.getMessage())); + printJsonExceptionMessages(jsonException.getMessage()); } } } @@ -82,6 +82,15 @@ public Map>> process() { return resultsMap; } + private void printJsonExceptionMessages(String message) { + if (message.contains("JSONObject[\"id\"] not found")) { + FCTUtils.printWarning( + "Questionnaire DOES NOT have an id field. Are we expecting it to be generated on the Server?"); + } else { + FCTUtils.printError(String.format("%s", message)); + } + } + private String getStructureMapId(JSONArray extensionJSONArray) { for (int i = 0; i < extensionJSONArray.length(); i++) { diff --git a/src/main/java/org/smartregister/util/StructureMapProcessor.java b/src/main/java/org/smartregister/util/StructureMapProcessor.java index 4033f77..fd93073 100644 --- a/src/main/java/org/smartregister/util/StructureMapProcessor.java +++ b/src/main/java/org/smartregister/util/StructureMapProcessor.java @@ -29,7 +29,7 @@ public Map> process() { try { Map> folderTofilesIndexMap = - FCTUtils.indexConfigurationFiles(directoryPath); + FCTUtils.indexConfigurationFiles(directoryPath, "map", "txt"); // Process other configurations for (var entry : folderTofilesIndexMap.entrySet()) { @@ -82,7 +82,8 @@ public Map> process() { } } - structureMapToLinkIds.put(getStructureMapId(firstLine), linkIds); + if (!firstLine.isBlank()) + structureMapToLinkIds.put(getStructureMapId(firstLine), linkIds); } } From ff52bc6c6fe79b7f9ad47f909807af7cfe14b4ec Mon Sep 17 00:00:00 2001 From: Martin Ndegwa Date: Wed, 26 Apr 2023 12:56:38 +0300 Subject: [PATCH 2/2] =?UTF-8?q?Update=20error=20message=20=F0=9F=92=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/org/smartregister/util/FCTValidationEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/smartregister/util/FCTValidationEngine.java b/src/main/java/org/smartregister/util/FCTValidationEngine.java index 72493d8..00f40f1 100644 --- a/src/main/java/org/smartregister/util/FCTValidationEngine.java +++ b/src/main/java/org/smartregister/util/FCTValidationEngine.java @@ -347,7 +347,7 @@ public void process( FCTUtils.printError( String.format( - "No Structure Map with id \u001b[36m%s\u001b[0m was found. Defining Questionnaire is id \u001b[36m%s\u001b[0m", + "No Structure Map with id \u001b[36m%s\u001b[0m was found. Defining Questionnaire has id \u001b[36m%s\u001b[0m", structureMapId, questionnaireId)); } }