From 7336036ae6c383a560431d7ddc85cb8989d5926e Mon Sep 17 00:00:00 2001 From: Daniel Pozzi Date: Tue, 31 Aug 2021 06:11:00 +0200 Subject: [PATCH 001/204] [#666] custom json import and mapping using jsonPath --- pom.xml | 5 + .../de/bonndan/nivio/input/FileFetcher.java | 26 +++ .../nivio/input/InputFormatHandler.java | 6 +- .../InputFormatHandlerCustomJSON.java | 174 ++++++++++++++++++ .../input/csv/InputFormatHandlerCSVTest.java | 12 +- .../InputFormatHandlerCustomJSONTest.java | 89 +++++++++ src/test/resources/example/example_json.yml | 12 ++ .../example/services/asd_upstream.json | 12 ++ .../resources/example/services/items.json | 15 ++ 9 files changed, 343 insertions(+), 8 deletions(-) create mode 100644 src/main/java/de/bonndan/nivio/input/customjson/InputFormatHandlerCustomJSON.java create mode 100644 src/test/java/de/bonndan/nivio/input/customjson/InputFormatHandlerCustomJSONTest.java create mode 100644 src/test/resources/example/example_json.yml create mode 100644 src/test/resources/example/services/asd_upstream.json create mode 100644 src/test/resources/example/services/items.json diff --git a/pom.xml b/pom.xml index bb0448225..c78d22dcb 100644 --- a/pom.xml +++ b/pom.xml @@ -229,6 +229,11 @@ ${lucene.version} + + com.jayway.jsonpath + json-path + 2.6.0 + diff --git a/src/main/java/de/bonndan/nivio/input/FileFetcher.java b/src/main/java/de/bonndan/nivio/input/FileFetcher.java index 1e32c4fd9..feff0aa7f 100644 --- a/src/main/java/de/bonndan/nivio/input/FileFetcher.java +++ b/src/main/java/de/bonndan/nivio/input/FileFetcher.java @@ -121,6 +121,32 @@ public String get(SourceReference ref) { } } + /** + * @param part url or partial path + * @param baseUrl optional base url + * @return the file/url contents + */ + @Nullable + public String get(@NonNull final String part, @Nullable final URL baseUrl) { + + try { + //we have no base url or source ref has absolute url + if (baseUrl == null || Objects.requireNonNull(part).startsWith("http")) { + return get(new URL(part)); + } + + //assemble new absolute url + String combined = URLHelper.combine(baseUrl, part); + URL url = new URL(combined); + if (URLHelper.isLocal(url)) { + return get(url); + } + return get(url); + } catch (MalformedURLException e) { + throw new ReadingException(String.format("Failed to build URL of %s with base url '%s'", part, baseUrl), e); + } + } + /** * @param ref * @param baseUrl diff --git a/src/main/java/de/bonndan/nivio/input/InputFormatHandler.java b/src/main/java/de/bonndan/nivio/input/InputFormatHandler.java index 8f1236f69..ee4b0b477 100644 --- a/src/main/java/de/bonndan/nivio/input/InputFormatHandler.java +++ b/src/main/java/de/bonndan/nivio/input/InputFormatHandler.java @@ -28,12 +28,14 @@ public interface InputFormatHandler { * @param baseUrl parent config url * @param landscapeDescription the input dto to modify */ - void applyData(@NonNull final SourceReference reference, @Nullable URL baseUrl, LandscapeDescription landscapeDescription); + void applyData(@NonNull final SourceReference reference, + @Nullable final URL baseUrl, + @NonNull final LandscapeDescription landscapeDescription); /** * Returns an observer for the source reference. * - * @param inner an observer for files or urls + * @param inner an observer for files or urls * @param sourceReference the {@link SourceReference} to observe * @return observer that can handle the format or null if no observer is available */ diff --git a/src/main/java/de/bonndan/nivio/input/customjson/InputFormatHandlerCustomJSON.java b/src/main/java/de/bonndan/nivio/input/customjson/InputFormatHandlerCustomJSON.java new file mode 100644 index 000000000..4704a5f1b --- /dev/null +++ b/src/main/java/de/bonndan/nivio/input/customjson/InputFormatHandlerCustomJSON.java @@ -0,0 +1,174 @@ +package de.bonndan.nivio.input.customjson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import com.jayway.jsonpath.spi.json.JsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import com.jayway.jsonpath.spi.mapper.MappingProvider; +import de.bonndan.nivio.input.FileFetcher; +import de.bonndan.nivio.input.InputFormatHandler; +import de.bonndan.nivio.input.dto.ItemDescription; +import de.bonndan.nivio.input.dto.LandscapeDescription; +import de.bonndan.nivio.input.dto.SourceReference; +import de.bonndan.nivio.observation.InputFormatObserver; +import de.bonndan.nivio.util.URLHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Service; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +public class InputFormatHandlerCustomJSON implements InputFormatHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(InputFormatHandlerCustomJSON.class); + + public static final String FORMAT = "customJSON"; + + public static final String ITEMS_PATH_KEY = "items"; + public static final String ITEM_PATH_KEY = "item"; + public static final String FETCH = "fetch"; + + private final FileFetcher fileFetcher; + private final ObjectMapper objectMapper; + + public InputFormatHandlerCustomJSON(FileFetcher fileFetcher, ObjectMapper objectMapper) { + this.fileFetcher = fileFetcher; + this.objectMapper = objectMapper; + + Configuration.setDefaults(new Configuration.Defaults() { + + private final JsonProvider jsonProvider = new JacksonJsonNodeJsonProvider(); + private final MappingProvider mappingProvider = new JacksonMappingProvider(); + + @Override + public JsonProvider jsonProvider() { + return jsonProvider; + } + + @Override + public MappingProvider mappingProvider() { + return mappingProvider; + } + + @Override + public Set