From 18824d0a1afcfa11c350a030200c1e643b10e943 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Thu, 24 Oct 2024 09:47:57 -0500 Subject: [PATCH] Remove composite predicates from transformations Signed-off-by: Andre Kurait --- .../JsonJMESPathPredicateProvider.java | 31 +++------- .../transform/JsonCompositePredicate.java | 40 ------------ .../JsonConditionalTransformerProvider.java | 15 ++--- .../migrations/transform/PredicateLoader.java | 61 ++++++++----------- .../replay/MultipleJoltScriptsTest.java | 2 +- 5 files changed, 40 insertions(+), 109 deletions(-) delete mode 100644 TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositePredicate.java diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java index 5f63b4d60..35d4b85f8 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicateProvider.java @@ -1,11 +1,7 @@ package org.opensearch.migrations.transform; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import org.opensearch.migrations.transform.JsonCompositePredicate.CompositeOperation; - import io.burt.jmespath.BaseRuntime; import io.burt.jmespath.jcf.JcfRuntime; @@ -20,41 +16,30 @@ public JsonJMESPathPredicateProvider() { @Override public IJsonPredicate createPredicate(Object jsonConfig) { - var transformers = new ArrayList(); - var configs = new ArrayList>(); try { if (jsonConfig instanceof Map) { - configs.add((Map) jsonConfig); - } else if (jsonConfig instanceof List) { - for (var c : (List) jsonConfig) { - configs.add((Map) c); - } - } else { - throw new IllegalArgumentException(getConfigUsageStr()); - } - for (var c : configs) { - if (c.size() != 1) { + @SuppressWarnings("unchecked") + var jsonConfigMap = (Map) jsonConfig; + if (jsonConfigMap.size() != 1) { throw new IllegalArgumentException(getConfigUsageStr()); } - var scriptValue = c.get(SCRIPT_KEY); + var scriptValue = jsonConfigMap.get(SCRIPT_KEY); if (!(scriptValue instanceof String)) { throw new IllegalArgumentException(getConfigUsageStr()); } - transformers.add(new JsonJMESPathPredicate(adapterRuntime, (String) scriptValue)); + return new JsonJMESPathPredicate(adapterRuntime, (String) scriptValue); } + throw new IllegalArgumentException(getConfigUsageStr()); } catch (ClassCastException e) { throw new IllegalArgumentException(getConfigUsageStr(), e); } - return new JsonCompositePredicate(CompositeOperation.ALL, - transformers.toArray(IJsonPredicate[]::new)); } private String getConfigUsageStr() { return this.getClass().getName() + " expects the incoming configuration " - + "to be a Map or a List>. " + + "to be a Map. " + "Each of the Maps should have one key-value of \"script\": \"...\". " - + "Script values should be a fully-formed inlined JsonPath queries encoded as a json value. " - + "All of the values within a configuration will be concatenated into one chained Predicate."; + + "Script values should be a fully-formed inlined JsonPath queries."; } } diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositePredicate.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositePredicate.java deleted file mode 100644 index 3fb7a1d06..000000000 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositePredicate.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.opensearch.migrations.transform; - -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; - -import lombok.NonNull; - -public class JsonCompositePredicate implements IJsonPredicate { - List jsonPredicateList; - CompositeOperation operation; - - public enum CompositeOperation { - ALL, - ANY, - NONE, - } - - public JsonCompositePredicate(@NonNull CompositeOperation operation, - IJsonPredicate... jsonPredicates) { - this.operation = operation; - this.jsonPredicateList = List.of(jsonPredicates); - } - - @Override - public boolean test(Map incomingJson) { - var Predicates = jsonPredicateList.stream(); - Predicate predicate = p -> p.test(incomingJson); - switch (operation) { - case ALL: - return Predicates.allMatch(predicate); - case ANY: - return Predicates.anyMatch(predicate); - case NONE: - return Predicates.noneMatch(predicate); - default: - throw new IllegalArgumentException("Unsupported operation: " + operation); - } - } -} diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java index 283a72ed1..c410b9595 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java @@ -3,10 +3,10 @@ import java.util.List; import java.util.stream.Collectors; -import org.opensearch.migrations.transform.JsonCompositePredicate.CompositeOperation; - import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class JsonConditionalTransformerProvider implements IJsonTransformerProvider { public JsonConditionalTransformerProvider() { @@ -21,20 +21,17 @@ public IJsonTransformer createTransformer(Object jsonConfig) { if (configs.size() != 2) { throw new IllegalArgumentException(getConfigUsageStr()); } - var PredicateConfig = configs.get(0); + var predicateConfig = configs.get(0); var transformerConfig = configs.get(1); - @SuppressWarnings("unchecked") - List Predicate = new PredicateLoader() - .getTransformerFactoryFromServiceLoaderParsed((List) PredicateConfig) - .collect(Collectors.toList()); + IJsonPredicate predicate = new PredicateLoader() + .getPredicateFactoryFromServiceLoaderParsed(predicateConfig); @SuppressWarnings("unchecked") List transformer = new TransformationLoader() .getTransformerFactoryFromServiceLoaderParsed((List) transformerConfig) .collect(Collectors.toList()); return new JsonConditionalTransformer( - new JsonCompositePredicate(CompositeOperation.ALL, - Predicate.toArray(IJsonPredicate[]::new)), + predicate, new JsonCompositeTransformer(transformer.toArray(IJsonTransformer[]::new))); } throw new IllegalArgumentException(getConfigUsageStr()); diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java index 5f3c64966..0b8fa3866 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/PredicateLoader.java @@ -7,9 +7,6 @@ import java.util.ServiceLoader; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.opensearch.migrations.transform.JsonCompositePredicate.CompositeOperation; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -20,18 +17,18 @@ @Slf4j public class PredicateLoader { public static final String WRONG_JSON_STRUCTURE_MESSAGE = - "Must specify the top-level configuration list with a sequence of " - + "maps that have only one key each, where the key is the name of the transformer to be configured."; + "Must specify the top-level configuration map" + + " that only has one key, where the key is the name of the Predicate to be configured."; public static final Pattern CLASS_NAME_PATTERN = Pattern.compile("^[^{}]*$"); private final List providers; ObjectMapper objMapper = new ObjectMapper(); public PredicateLoader() { - ServiceLoader transformerProviders = ServiceLoader.load( + ServiceLoader predicateProviders = ServiceLoader.load( IJsonPredicateProvider.class ); var inProgressProviders = new ArrayList(); - for (var provider : transformerProviders) { + for (var provider : predicateProviders) { log.info("Adding IJsonPredicateProvider: " + provider); inProgressProviders.add(provider); } @@ -44,38 +41,34 @@ public PredicateLoader() { .log(); } - List> parseFullConfig(String fullConfig) throws JsonProcessingException { + Map parseFullConfig(String fullConfig) throws JsonProcessingException { if (CLASS_NAME_PATTERN.matcher(fullConfig).matches()) { - return List.of(Collections.singletonMap(fullConfig, null)); + return Collections.singletonMap(fullConfig, null); } else { - return objMapper.readValue(fullConfig, new TypeReference<>() { - }); + return objMapper.readValue(fullConfig, new TypeReference<>() {}); } } - protected Stream getTransformerFactoryFromServiceLoader(String fullConfig) + protected IJsonPredicate getPredicateFactoryFromServiceLoader(String fullConfig) throws JsonProcessingException { - var configList = fullConfig == null ? List.of() : parseFullConfig(fullConfig); - if (configList.isEmpty() || providers.isEmpty()) { - log.warn("No transformer configuration specified. No custom transformations will be performed"); - return Stream.of(); - } else { - return configList.stream().map(c -> configureTransformerFromConfig((Map) c)); - } + Map configMap = fullConfig == null ? Map.of() : parseFullConfig(fullConfig); + return getPredicateFactoryFromServiceLoaderParsed(configMap); } - public Stream getTransformerFactoryFromServiceLoaderParsed(List configList) { - if (configList.isEmpty() || providers.isEmpty()) { - log.warn("No transformer configuration specified. No custom transformations will be performed"); - return Stream.of(); + public IJsonPredicate getPredicateFactoryFromServiceLoaderParsed(Object config) { + @SuppressWarnings("unchecked") + var configMap = (Map) config; + if (configMap.isEmpty() || providers.isEmpty()) { + log.atError().setMessage("No predicate configuration found for configuration {}") + .addArgument(config).log(); + throw new IllegalArgumentException("No predicate configuration found for configuration " + config); } else { - return configList.stream().map(c -> configureTransformerFromConfig((Map) c)); + return configurePredicateFromConfig(configMap); } } - @SneakyThrows // JsonProcessingException should be impossible since the contents are those that were just parsed - private IJsonPredicate configureTransformerFromConfig(Map c) { + private IJsonPredicate configurePredicateFromConfig(Map c) { var keys = c.keySet(); if (keys.size() != 1) { throw new IllegalArgumentException(WRONG_JSON_STRUCTURE_MESSAGE); @@ -84,12 +77,12 @@ private IJsonPredicate configureTransformerFromConfig(Map c) { .findFirst() .orElseThrow(() -> new IllegalArgumentException(WRONG_JSON_STRUCTURE_MESSAGE)); for (var p : providers) { - var transformerName = p.getName(); - if (transformerName.equals(key)) { + var predicateName = p.getName(); + if (predicateName.equals(key)) { var configuration = c.get(key); log.atInfo() .setMessage( - () -> "Creating a transformer through provider=" + p + " with configuration=" + configuration + () -> "Creating a Predicate through provider=" + p + " with configuration=" + configuration ) .log(); return p.createPredicate(configuration); @@ -98,15 +91,11 @@ private IJsonPredicate configureTransformerFromConfig(Map c) { throw new IllegalArgumentException("Could not find a provider named: " + key); } - public IJsonPredicate getTransformerFactoryLoader(String fullConfig) { + public IJsonPredicate getPredicateFactoryLoader(String fullConfig) { try { - var loadedTransformers = getTransformerFactoryFromServiceLoader(fullConfig); - return new JsonCompositePredicate( - CompositeOperation.ALL, - loadedTransformers.toArray(IJsonPredicate[]::new) - ); + return getPredicateFactoryFromServiceLoader(fullConfig); } catch (JsonProcessingException e) { - throw new IllegalArgumentException("Could not parse the transformer configuration as a json list", e); + throw new IllegalArgumentException("Could not parse the Predicate configuration as a json list", e); } } } diff --git a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java index 4bb773b92..a346370b4 100644 --- a/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java +++ b/TrafficCapture/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java @@ -38,7 +38,7 @@ public void testAddGzip() throws Exception { public void testAddGzipAndCustom() throws Exception { final var addGzip = "[" + "{\"JsonConditionalTransformerProvider\": [" - + " [{\"JsonJMESPathPredicateProvider\": { \"script\": \"" + "URI == '/testindex/_search'" + "\"}}]," + + " {\"JsonJMESPathPredicateProvider\": { \"script\": \"" + "URI == '/testindex/_search'" + "\"}}," + " [{\"JsonJoltTransformerProvider\": { \"canned\": \"ADD_GZIP\" }}]" + "]}," + "{ \"JsonJoltTransformerProvider\":"