From 88dad7733bd0ddc0a5562ae0f8f92132cb6ee753 Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Tue, 27 Aug 2024 11:25:24 -0700 Subject: [PATCH 01/23] Fix cohere model input interface cannot validate cohere input issue (#2847) * Fix cohere model input interface cannot validate cohere input issue Signed-off-by: b4sjoo * Clean out comments Signed-off-by: b4sjoo --------- Signed-off-by: b4sjoo --- .../org/opensearch/ml/common/utils/ModelInterfaceUtils.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java b/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java index 5c5cc5fd99..61bba1065d 100644 --- a/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java +++ b/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java @@ -42,10 +42,7 @@ public class ModelInterfaceUtils { + " \"type\": \"object\",\n" + " \"properties\": {\n" + " \"texts\": {\n" - + " \"type\": \"array\",\n" - + " \"items\": {\n" - + " \"type\": \"string\"\n" - + " }\n" + + " \"type\": \"string\"\n" + " }\n" + " },\n" + " \"required\": [\n" From 05eb53f7df6990e9c2388c275a42d84b43767eee Mon Sep 17 00:00:00 2001 From: Ashish Agrawal Date: Wed, 28 Aug 2024 13:13:29 -0700 Subject: [PATCH 02/23] Expose ML Config API (#2850) * Expose ML Config API Signed-off-by: Ashish Agrawal * Add tests for rejected master key Signed-off-by: Ashish Agrawal --------- Signed-off-by: Ashish Agrawal --- .../ml/client/MachineLearningClient.java | 17 +++++++ .../ml/client/MachineLearningNodeClient.java | 22 +++++++++ .../ml/client/MachineLearningClientTest.java | 26 ++++++++++ .../client/MachineLearningNodeClientTest.java | 48 +++++++++++++++++++ .../transport/config/MLConfigGetResponse.java | 2 + .../config/GetConfigTransportAction.java | 6 +++ .../config/GetConfigTransportActionTests.java | 27 +++++++++++ 7 files changed, 148 insertions(+) diff --git a/client/src/main/java/org/opensearch/ml/client/MachineLearningClient.java b/client/src/main/java/org/opensearch/ml/client/MachineLearningClient.java index b115eb91c9..f0d6f08127 100644 --- a/client/src/main/java/org/opensearch/ml/client/MachineLearningClient.java +++ b/client/src/main/java/org/opensearch/ml/client/MachineLearningClient.java @@ -16,6 +16,7 @@ import org.opensearch.common.action.ActionFuture; import org.opensearch.core.action.ActionListener; import org.opensearch.ml.common.FunctionName; +import org.opensearch.ml.common.MLConfig; import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.ToolMetadata; @@ -428,4 +429,20 @@ default ActionFuture getTool(String toolName) { */ void getTool(String toolName, ActionListener listener); + /** + * Get config + * @param configId ML config id + */ + default ActionFuture getConfig(String configId) { + PlainActionFuture actionFuture = PlainActionFuture.newFuture(); + getConfig(configId, actionFuture); + return actionFuture; + } + + /** + * Get config + * @param configId ML config id + * @param listener a listener to be notified of the result + */ + void getConfig(String configId, ActionListener listener); } diff --git a/client/src/main/java/org/opensearch/ml/client/MachineLearningNodeClient.java b/client/src/main/java/org/opensearch/ml/client/MachineLearningNodeClient.java index acf171872d..5819d055e7 100644 --- a/client/src/main/java/org/opensearch/ml/client/MachineLearningNodeClient.java +++ b/client/src/main/java/org/opensearch/ml/client/MachineLearningNodeClient.java @@ -25,6 +25,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.ml.common.FunctionName; +import org.opensearch.ml.common.MLConfig; import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.ToolMetadata; @@ -39,6 +40,9 @@ import org.opensearch.ml.common.transport.agent.MLRegisterAgentAction; import org.opensearch.ml.common.transport.agent.MLRegisterAgentRequest; import org.opensearch.ml.common.transport.agent.MLRegisterAgentResponse; +import org.opensearch.ml.common.transport.config.MLConfigGetAction; +import org.opensearch.ml.common.transport.config.MLConfigGetRequest; +import org.opensearch.ml.common.transport.config.MLConfigGetResponse; import org.opensearch.ml.common.transport.connector.MLConnectorDeleteAction; import org.opensearch.ml.common.transport.connector.MLConnectorDeleteRequest; import org.opensearch.ml.common.transport.connector.MLCreateConnectorAction; @@ -309,6 +313,13 @@ public void getTool(String toolName, ActionListener listener) { client.execute(MLGetToolAction.INSTANCE, mlToolGetRequest, getMlGetToolResponseActionListener(listener)); } + @Override + public void getConfig(String configId, ActionListener listener) { + MLConfigGetRequest mlConfigGetRequest = MLConfigGetRequest.builder().configId(configId).build(); + + client.execute(MLConfigGetAction.INSTANCE, mlConfigGetRequest, getMlGetConfigResponseActionListener(listener)); + } + private ActionListener getMlListToolsResponseActionListener(ActionListener> listener) { ActionListener internalListener = ActionListener.wrap(mlModelListResponse -> { listener.onResponse(mlModelListResponse.getToolMetadataList()); @@ -331,6 +342,17 @@ private ActionListener getMlGetToolResponseActionListener(Act return actionListener; } + private ActionListener getMlGetConfigResponseActionListener(ActionListener listener) { + ActionListener internalListener = ActionListener.wrap(mlConfigGetResponse -> { + listener.onResponse(mlConfigGetResponse.getMlConfig()); + }, listener::onFailure); + ActionListener actionListener = wrapActionListener(internalListener, res -> { + MLConfigGetResponse getResponse = MLConfigGetResponse.fromActionResponse(res); + return getResponse; + }); + return actionListener; + } + private ActionListener getMLRegisterAgentResponseActionListener( ActionListener listener ) { diff --git a/client/src/test/java/org/opensearch/ml/client/MachineLearningClientTest.java b/client/src/test/java/org/opensearch/ml/client/MachineLearningClientTest.java index d7ae5ec334..5100e7cb19 100644 --- a/client/src/test/java/org/opensearch/ml/client/MachineLearningClientTest.java +++ b/client/src/test/java/org/opensearch/ml/client/MachineLearningClientTest.java @@ -12,6 +12,7 @@ import static org.opensearch.ml.common.input.Constants.KMEANS; import static org.opensearch.ml.common.input.Constants.TRAIN; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -28,8 +29,10 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.core.action.ActionListener; import org.opensearch.ml.common.AccessMode; +import org.opensearch.ml.common.Configuration; import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.MLAgentType; +import org.opensearch.ml.common.MLConfig; import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.ToolMetadata; @@ -46,6 +49,7 @@ import org.opensearch.ml.common.output.MLOutput; import org.opensearch.ml.common.output.MLTrainingOutput; import org.opensearch.ml.common.transport.agent.MLRegisterAgentResponse; +import org.opensearch.ml.common.transport.config.MLConfigGetResponse; import org.opensearch.ml.common.transport.connector.MLCreateConnectorInput; import org.opensearch.ml.common.transport.connector.MLCreateConnectorResponse; import org.opensearch.ml.common.transport.deploy.MLDeployModelResponse; @@ -99,9 +103,13 @@ public class MachineLearningClientTest { @Mock MLRegisterAgentResponse registerAgentResponse; + @Mock + MLConfigGetResponse configGetResponse; + private String modekId = "test_model_id"; private MLModel mlModel; private MLTask mlTask; + private MLConfig mlConfig; private ToolMetadata toolMetadata; private List toolsList = new ArrayList<>(); @@ -124,6 +132,14 @@ public void setUp() { .build(); toolsList.add(toolMetadata); + mlConfig = MLConfig + .builder() + .type("dummyType") + .configuration(Configuration.builder().agentId("agentId").build()) + .createTime(Instant.now()) + .lastUpdateTime(Instant.now()) + .build(); + machineLearningClient = new MachineLearningClient() { @Override public void predict(String modelId, MLInput mlInput, ActionListener listener) { @@ -231,6 +247,11 @@ public void registerAgent(MLAgent mlAgent, ActionListener listener) { listener.onResponse(deleteResponse); } + + @Override + public void getConfig(String configId, ActionListener listener) { + listener.onResponse(mlConfig); + } }; } @@ -503,4 +524,9 @@ public void getTool() { public void listTools() { assertEquals(toolMetadata, machineLearningClient.listTools().actionGet().get(0)); } + + @Test + public void getConfig() { + assertEquals(mlConfig, machineLearningClient.getConfig("configId").actionGet()); + } } diff --git a/client/src/test/java/org/opensearch/ml/client/MachineLearningNodeClientTest.java b/client/src/test/java/org/opensearch/ml/client/MachineLearningNodeClientTest.java index ebdcbf9e87..6fea6e8c60 100644 --- a/client/src/test/java/org/opensearch/ml/client/MachineLearningNodeClientTest.java +++ b/client/src/test/java/org/opensearch/ml/client/MachineLearningNodeClientTest.java @@ -14,6 +14,7 @@ import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.verify; +import static org.opensearch.ml.common.CommonValue.MASTER_KEY; import static org.opensearch.ml.common.input.Constants.ACTION; import static org.opensearch.ml.common.input.Constants.ALGORITHM; import static org.opensearch.ml.common.input.Constants.KMEANS; @@ -40,6 +41,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -51,12 +53,15 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.index.Index; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.ml.common.AccessMode; +import org.opensearch.ml.common.Configuration; import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.MLAgentType; +import org.opensearch.ml.common.MLConfig; import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.MLTaskState; @@ -84,6 +89,9 @@ import org.opensearch.ml.common.transport.agent.MLRegisterAgentAction; import org.opensearch.ml.common.transport.agent.MLRegisterAgentRequest; import org.opensearch.ml.common.transport.agent.MLRegisterAgentResponse; +import org.opensearch.ml.common.transport.config.MLConfigGetAction; +import org.opensearch.ml.common.transport.config.MLConfigGetRequest; +import org.opensearch.ml.common.transport.config.MLConfigGetResponse; import org.opensearch.ml.common.transport.connector.MLConnectorDeleteAction; import org.opensearch.ml.common.transport.connector.MLConnectorDeleteRequest; import org.opensearch.ml.common.transport.connector.MLCreateConnectorAction; @@ -206,6 +214,9 @@ public class MachineLearningNodeClientTest { @Mock ActionListener getToolActionListener; + @Mock + ActionListener getMlConfigListener; + @InjectMocks MachineLearningNodeClient machineLearningNodeClient; @@ -951,6 +962,43 @@ public void listTools() { assertEquals("Use this tool to search general knowledge on wikipedia.", argumentCaptor.getValue().get(0).getDescription()); } + @Test + public void getConfig() { + MLConfig mlConfig = MLConfig.builder().type("type").configuration(Configuration.builder().agentId("agentId").build()).build(); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + MLConfigGetResponse output = MLConfigGetResponse.builder().mlConfig(mlConfig).build(); + actionListener.onResponse(output); + return null; + }).when(client).execute(eq(MLConfigGetAction.INSTANCE), any(), any()); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(MLConfig.class); + machineLearningNodeClient.getConfig("agentId", getMlConfigListener); + + verify(client).execute(eq(MLConfigGetAction.INSTANCE), isA(MLConfigGetRequest.class), any()); + verify(getMlConfigListener).onResponse(argumentCaptor.capture()); + assertEquals("agentId", argumentCaptor.getValue().getConfiguration().getAgentId()); + assertEquals("type", argumentCaptor.getValue().getType()); + } + + @Test + public void getConfigRejectedMasterKey() { + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + actionListener.onFailure(new OpenSearchStatusException("You are not allowed to access this config doc", RestStatus.FORBIDDEN)); + return null; + }).when(client).execute(eq(MLConfigGetAction.INSTANCE), any(), any()); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + machineLearningNodeClient.getConfig(MASTER_KEY, getMlConfigListener); + + verify(client).execute(eq(MLConfigGetAction.INSTANCE), isA(MLConfigGetRequest.class), any()); + verify(getMlConfigListener).onFailure(argumentCaptor.capture()); + assertEquals(RestStatus.FORBIDDEN, argumentCaptor.getValue().status()); + assertEquals("You are not allowed to access this config doc", argumentCaptor.getValue().getLocalizedMessage()); + } + private SearchResponse createSearchResponse(ToXContentObject o) throws IOException { XContentBuilder content = o.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); diff --git a/common/src/main/java/org/opensearch/ml/common/transport/config/MLConfigGetResponse.java b/common/src/main/java/org/opensearch/ml/common/transport/config/MLConfigGetResponse.java index 33e95cc474..f00e7b1085 100644 --- a/common/src/main/java/org/opensearch/ml/common/transport/config/MLConfigGetResponse.java +++ b/common/src/main/java/org/opensearch/ml/common/transport/config/MLConfigGetResponse.java @@ -20,7 +20,9 @@ import org.opensearch.ml.common.MLConfig; import lombok.Builder; +import lombok.Getter; +@Getter public class MLConfigGetResponse extends ActionResponse implements ToXContentObject { MLConfig mlConfig; diff --git a/plugin/src/main/java/org/opensearch/ml/action/config/GetConfigTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/config/GetConfigTransportAction.java index 787198a826..c187a0bc14 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/config/GetConfigTransportAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/config/GetConfigTransportAction.java @@ -6,6 +6,7 @@ package org.opensearch.ml.action.config; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.ml.common.CommonValue.MASTER_KEY; import static org.opensearch.ml.common.CommonValue.ML_CONFIG_INDEX; import static org.opensearch.ml.utils.MLNodeUtils.createXContentParserFromRegistry; @@ -58,6 +59,11 @@ protected void doExecute(Task task, ActionRequest request, ActionListener { log.debug("Completed Get Agent Request, id:{}", configId); diff --git a/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java index 77eccab713..0dbb79ef6c 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java @@ -5,12 +5,14 @@ package org.opensearch.ml.action.config; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.ml.common.CommonValue.MASTER_KEY; import java.io.IOException; import java.time.Instant; @@ -22,6 +24,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.get.GetResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.client.Client; @@ -30,6 +33,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -168,4 +172,27 @@ public GetResponse prepareMLConfig(String configID) throws IOException { GetResponse getResponse = new GetResponse(getResult); return getResponse; } + + @Test + public void testDoExecute_Rejected_MASTER_KEY() throws IOException { + String configID = MASTER_KEY; + GetResponse getResponse = prepareMLConfig(configID); + ActionListener actionListener = mock(ActionListener.class); + MLConfigGetRequest request = new MLConfigGetRequest(configID); + Task task = mock(Task.class); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + + getConfigTransportAction.doExecute(task, request, actionListener); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals(RestStatus.FORBIDDEN, argumentCaptor.getValue().status()); + assertEquals("You are not allowed to access this config doc", argumentCaptor.getValue().getLocalizedMessage()); + + } } From 0a895375b8569e593f45ffac296bd4143c5be7fb Mon Sep 17 00:00:00 2001 From: Sicheng Song Date: Wed, 28 Aug 2024 14:08:00 -0700 Subject: [PATCH 03/23] Add processed function for remote inference input dataset parameters to convert it back to its orignal datatype (#2852) * Add processed function for remote inference input dataset parameters to convert it back to its orignal datatype Signed-off-by: b4sjoo * spotless Signed-off-by: b4sjoo * remove debugging print Signed-off-by: b4sjoo * Add UTs Signed-off-by: b4sjoo * Add UTs Signed-off-by: b4sjoo * Spotless Signed-off-by: b4sjoo --------- Signed-off-by: b4sjoo --- .../ml/common/utils/ModelInterfaceUtils.java | 5 +- .../TransportPredictionTaskAction.java | 9 ++- .../org/opensearch/ml/utils/MLNodeUtils.java | 34 ++++++++++ .../opensearch/ml/utils/MLNodeUtilsTests.java | 62 +++++++++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java b/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java index 61bba1065d..5c5cc5fd99 100644 --- a/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java +++ b/common/src/main/java/org/opensearch/ml/common/utils/ModelInterfaceUtils.java @@ -42,7 +42,10 @@ public class ModelInterfaceUtils { + " \"type\": \"object\",\n" + " \"properties\": {\n" + " \"texts\": {\n" - + " \"type\": \"string\"\n" + + " \"type\": \"array\",\n" + + " \"items\": {\n" + + " \"type\": \"string\"\n" + + " }\n" + " }\n" + " },\n" + " \"required\": [\n" diff --git a/plugin/src/main/java/org/opensearch/ml/action/prediction/TransportPredictionTaskAction.java b/plugin/src/main/java/org/opensearch/ml/action/prediction/TransportPredictionTaskAction.java index 4cf957c499..a0e5018ad4 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/prediction/TransportPredictionTaskAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/prediction/TransportPredictionTaskAction.java @@ -242,11 +242,10 @@ public void validateInputSchema(String modelId, MLInput mlInput) { if (modelCacheHelper.getModelInterface(modelId) != null && modelCacheHelper.getModelInterface(modelId).get("input") != null) { String inputSchemaString = modelCacheHelper.getModelInterface(modelId).get("input"); try { - MLNodeUtils - .validateSchema( - inputSchemaString, - mlInput.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString() - ); + String InputString = mlInput.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString(); + // Process the parameters field in the input dataset to convert it back to its original datatype, instead of a string + String processedInputString = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(InputString); + MLNodeUtils.validateSchema(inputSchemaString, processedInputString); } catch (Exception e) { throw new OpenSearchStatusException("Error validating input schema: " + e.getMessage(), RestStatus.BAD_REQUEST); } diff --git a/plugin/src/main/java/org/opensearch/ml/utils/MLNodeUtils.java b/plugin/src/main/java/org/opensearch/ml/utils/MLNodeUtils.java index 86fbfb1605..3cbbc62ef5 100644 --- a/plugin/src/main/java/org/opensearch/ml/utils/MLNodeUtils.java +++ b/plugin/src/main/java/org/opensearch/ml/utils/MLNodeUtils.java @@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SpecVersion.VersionFlag; @@ -89,6 +90,39 @@ public static void validateSchema(String schemaString, String instanceString) th } } + /** + * This method processes the input JSON string and replaces the string values of the parameters with JSON objects if the string is a valid JSON. + * @param inputJson The input JSON string + * @return The processed JSON string + */ + public static String processRemoteInferenceInputDataSetParametersValue(String inputJson) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(inputJson); + + if (rootNode.has("parameters") && rootNode.get("parameters").isObject()) { + ObjectNode parametersNode = (ObjectNode) rootNode.get("parameters"); + + parametersNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode value = entry.getValue(); + + if (value.isTextual()) { + String textValue = value.asText(); + try { + // Try to parse the string as JSON + JsonNode parsedValue = mapper.readTree(textValue); + // If successful, replace the string with the parsed JSON + parametersNode.set(key, parsedValue); + } catch (IOException e) { + // If parsing fails, it's not a valid JSON string, so keep it as is + parametersNode.set(key, value); + } + } + }); + } + return mapper.writeValueAsString(rootNode); + } + public static void checkOpenCircuitBreaker(MLCircuitBreakerService mlCircuitBreakerService, MLStats mlStats) { ThresholdCircuitBreaker openCircuitBreaker = mlCircuitBreakerService.checkOpenCB(); if (openCircuitBreaker != null) { diff --git a/plugin/src/test/java/org/opensearch/ml/utils/MLNodeUtilsTests.java b/plugin/src/test/java/org/opensearch/ml/utils/MLNodeUtilsTests.java index 7838308834..5b12e73d3c 100644 --- a/plugin/src/test/java/org/opensearch/ml/utils/MLNodeUtilsTests.java +++ b/plugin/src/test/java/org/opensearch/ml/utils/MLNodeUtilsTests.java @@ -26,6 +26,8 @@ import org.opensearch.ml.common.MLTask; import org.opensearch.test.OpenSearchTestCase; +import com.fasterxml.jackson.core.JsonParseException; + public class MLNodeUtilsTests extends OpenSearchTestCase { public void testIsMLNode() { @@ -63,4 +65,64 @@ public void testValidateSchema() throws IOException { String json = "{\"key1\": \"foo\", \"key2\": 123}"; MLNodeUtils.validateSchema(schema, json); } + + @Test + public void testProcessRemoteInferenceInputDataSetParametersValueNoParameters() throws IOException { + String json = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(json, processedJson); + } + + @Test + public void testProcessRemoteInferenceInputDataSetInvalidJson() { + String json = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"a\"}}"; + assertThrows(JsonParseException.class, () -> MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json)); + } + + @Test + public void testProcessRemoteInferenceInputDataSetEmptyParameters() throws IOException { + String json = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{}}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(json, processedJson); + } + + @Test + public void testProcessRemoteInferenceInputDataSetParametersValueParametersWrongType() throws IOException { + String json = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":[\"Hello\",\"world\"]}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(json, processedJson); + } + + @Test + public void testProcessRemoteInferenceInputDataSetParametersValueWithParametersProcessArray() throws IOException { + String json = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"texts\":\"[\\\"Hello\\\",\\\"world\\\"]\"}}"; + String expectedJson = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"texts\":[\"Hello\",\"world\"]}}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(expectedJson, processedJson); + } + + @Test + public void testProcessRemoteInferenceInputDataSetParametersValueWithParametersProcessObject() throws IOException { + String json = + "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"messages\":\"{\\\"role\\\":\\\"system\\\",\\\"foo\\\":\\\"{\\\\\\\"a\\\\\\\": \\\\\\\"b\\\\\\\"}\\\",\\\"content\\\":{\\\"a\\\":\\\"b\\\"}}\"}}}"; + String expectedJson = + "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"messages\":{\"role\":\"system\",\"foo\":\"{\\\"a\\\": \\\"b\\\"}\",\"content\":{\"a\":\"b\"}}}}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(expectedJson, processedJson); + } + + @Test + public void testProcessRemoteInferenceInputDataSetParametersValueWithParametersNoProcess() throws IOException { + String json = "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"key1\":\"foo\",\"key2\":123,\"key3\":true}}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(json, processedJson); + } + + @Test + public void testProcessRemoteInferenceInputDataSetParametersValueWithParametersInvalidJson() throws IOException { + String json = + "{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"parameters\":{\"key1\":\"foo\",\"key2\":123,\"key3\":true,\"texts\":\"[\\\"Hello\\\",\\\"world\\\"\"}}"; + String processedJson = MLNodeUtils.processRemoteInferenceInputDataSetParametersValue(json); + assertEquals(json, processedJson); + } } From 7ecff1aaa17d8ee04fc0a408b49384624db0b93b Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 28 Aug 2024 16:32:08 -0700 Subject: [PATCH 04/23] use local_regex as default type for guardrails (#2853) * use local_regex as default type for guardrails Signed-off-by: Jing Zhang * add UT for model type Signed-off-by: Jing Zhang --------- Signed-off-by: Jing Zhang --- .../ml/common/model/Guardrails.java | 3 + .../ml/common/model/GuardrailsTests.java | 45 ++++++++++ .../ml/rest/RestMLGuardrailsIT.java | 85 +++++++++++++++++++ 3 files changed, 133 insertions(+) diff --git a/common/src/main/java/org/opensearch/ml/common/model/Guardrails.java b/common/src/main/java/org/opensearch/ml/common/model/Guardrails.java index b6f0017878..8e2885c506 100644 --- a/common/src/main/java/org/opensearch/ml/common/model/Guardrails.java +++ b/common/src/main/java/org/opensearch/ml/common/model/Guardrails.java @@ -123,6 +123,9 @@ public static Guardrails parse(XContentParser parser) throws IOException { break; } } + if (type == null) { + type = "local_regex"; + } if (!validateType(type)) { throw new IllegalArgumentException("The type of guardrails is required, can not be null."); } diff --git a/common/src/test/java/org/opensearch/ml/common/model/GuardrailsTests.java b/common/src/test/java/org/opensearch/ml/common/model/GuardrailsTests.java index b4429ebdf8..c6e365f485 100644 --- a/common/src/test/java/org/opensearch/ml/common/model/GuardrailsTests.java +++ b/common/src/test/java/org/opensearch/ml/common/model/GuardrailsTests.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Map; import org.junit.Assert; import org.junit.Before; @@ -27,6 +28,8 @@ public class GuardrailsTests { String[] regex; LocalRegexGuardrail inputLocalRegexGuardrail; LocalRegexGuardrail outputLocalRegexGuardrail; + ModelGuardrail inputModelGuardrail; + ModelGuardrail outputModelGuardrail; @Before public void setUp() { @@ -34,6 +37,8 @@ public void setUp() { regex = List.of("regex1").toArray(new String[0]); inputLocalRegexGuardrail = new LocalRegexGuardrail(List.of(stopWords), regex); outputLocalRegexGuardrail = new LocalRegexGuardrail(List.of(stopWords), regex); + inputModelGuardrail = new ModelGuardrail(Map.of("model_id", "guardrail_model_id", "response_validation_regex", "accept")); + outputModelGuardrail = new ModelGuardrail(Map.of("model_id", "guardrail_model_id", "response_validation_regex", "accept")); } @Test @@ -83,4 +88,44 @@ public void parse() throws IOException { Assert.assertEquals(guardrails.getInputGuardrail(), inputLocalRegexGuardrail); Assert.assertEquals(guardrails.getOutputGuardrail(), outputLocalRegexGuardrail); } + + @Test + public void parseNonType() throws IOException { + String jsonStr = "{" + + "\"input_guardrail\":{\"stop_words\":[{\"index_name\":\"test_index\",\"source_fields\":[\"test_field\"]}],\"regex\":[\"regex1\"]}," + + "\"output_guardrail\":{\"stop_words\":[{\"index_name\":\"test_index\",\"source_fields\":[\"test_field\"]}],\"regex\":[\"regex1\"]}}"; + XContentParser parser = XContentType.JSON + .xContent() + .createParser( + new NamedXContentRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()), + null, + jsonStr + ); + parser.nextToken(); + Guardrails guardrails = Guardrails.parse(parser); + + Assert.assertEquals(guardrails.getType(), "local_regex"); + Assert.assertEquals(guardrails.getInputGuardrail(), inputLocalRegexGuardrail); + Assert.assertEquals(guardrails.getOutputGuardrail(), outputLocalRegexGuardrail); + } + + @Test + public void parseModelType() throws IOException { + String jsonStr = "{\"type\":\"model\"," + + "\"input_guardrail\":{\"model_id\":\"guardrail_model_id\",\"response_validation_regex\":\"accept\"}," + + "\"output_guardrail\":{\"model_id\":\"guardrail_model_id\",\"response_validation_regex\":\"accept\"}}"; + XContentParser parser = XContentType.JSON + .xContent() + .createParser( + new NamedXContentRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()), + null, + jsonStr + ); + parser.nextToken(); + Guardrails guardrails = Guardrails.parse(parser); + + Assert.assertEquals(guardrails.getType(), "model"); + Assert.assertEquals(guardrails.getInputGuardrail(), inputModelGuardrail); + Assert.assertEquals(guardrails.getOutputGuardrail(), outputModelGuardrail); + } } diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLGuardrailsIT.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLGuardrailsIT.java index 6ed0fc4118..fbabf1dbb7 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLGuardrailsIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLGuardrailsIT.java @@ -177,6 +177,31 @@ public void testPredictRemoteModelFailed() throws IOException, InterruptedExcept predictRemoteModel(modelId, predictInput); } + public void testPredictRemoteModelFailedNonType() throws IOException, InterruptedException { + // Skip test if key is null + if (OPENAI_KEY == null) { + return; + } + exceptionRule.expect(ResponseException.class); + exceptionRule.expectMessage("guardrails triggered for user input"); + Response response = createConnector(completionModelConnectorEntity); + Map responseMap = parseResponseToMap(response); + String connectorId = (String) responseMap.get("connector_id"); + response = registerRemoteModelNonTypeGuardrails("openAI-GPT-3.5 completions", connectorId); + responseMap = parseResponseToMap(response); + String taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + response = getTask(taskId); + responseMap = parseResponseToMap(response); + String modelId = (String) responseMap.get("model_id"); + response = deployRemoteModel(modelId); + responseMap = parseResponseToMap(response); + taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + String predictInput = "{\n" + " \"parameters\": {\n" + " \"prompt\": \"Say this is a test of stop word.\"\n" + " }\n" + "}"; + predictRemoteModel(modelId, predictInput); + } + public void testPredictRemoteModelSuccessWithModelGuardrail() throws IOException, InterruptedException { // Skip test if key is null if (OPENAI_KEY == null) { @@ -437,6 +462,66 @@ protected Response registerRemoteModelWithLocalRegexGuardrails(String name, Stri .makeRequest(client(), "POST", "/_plugins/_ml/models/_register", null, TestHelper.toHttpEntity(registerModelEntity), null); } + protected Response registerRemoteModelNonTypeGuardrails(String name, String connectorId) throws IOException { + String registerModelGroupEntity = "{\n" + + " \"name\": \"remote_model_group\",\n" + + " \"description\": \"This is an example description\"\n" + + "}"; + Response response = TestHelper + .makeRequest( + client(), + "POST", + "/_plugins/_ml/model_groups/_register", + null, + TestHelper.toHttpEntity(registerModelGroupEntity), + null + ); + Map responseMap = parseResponseToMap(response); + assertEquals((String) responseMap.get("status"), "CREATED"); + String modelGroupId = (String) responseMap.get("model_group_id"); + + String registerModelEntity = "{\n" + + " \"name\": \"" + + name + + "\",\n" + + " \"function_name\": \"remote\",\n" + + " \"model_group_id\": \"" + + modelGroupId + + "\",\n" + + " \"version\": \"1.0.0\",\n" + + " \"description\": \"test model\",\n" + + " \"connector_id\": \"" + + connectorId + + "\",\n" + + " \"guardrails\": {\n" + + " \"input_guardrail\": {\n" + + " \"stop_words\": [\n" + + " {" + + " \"index_name\": \"stop_words\",\n" + + " \"source_fields\": [\"title\"]\n" + + " }" + + " ],\n" + + " \"regex\": [\"regex1\", \"regex2\"]\n" + + " },\n" + + " \"output_guardrail\": {\n" + + " \"stop_words\": [\n" + + " {" + + " \"index_name\": \"stop_words\",\n" + + " \"source_fields\": [\"title\"]\n" + + " }" + + " ],\n" + + " \"regex\": [\"regex1\", \"regex2\"]\n" + + " }\n" + + "},\n" + + " \"interface\": {\n" + + " \"input\": {},\n" + + " \"output\": {}\n" + + " }\n" + + "}"; + return TestHelper + .makeRequest(client(), "POST", "/_plugins/_ml/models/_register", null, TestHelper.toHttpEntity(registerModelEntity), null); + } + protected Response registerRemoteModelWithModelGuardrails(String name, String connectorId, String guardrailModelId) throws IOException { String registerModelGroupEntity = "{\n" From 1da79ce53754d11ea93ee242e9114ad71f3a406b Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Fri, 30 Aug 2024 10:56:05 -0700 Subject: [PATCH 05/23] send response in xcontent, if any exception, use plain text (#2858) Signed-off-by: Jing Zhang --- .../ml/rest/RestMLExecuteAction.java | 17 +++++- .../ml/rest/RestMLExecuteActionTests.java | 56 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/java/org/opensearch/ml/rest/RestMLExecuteAction.java b/plugin/src/main/java/org/opensearch/ml/rest/RestMLExecuteAction.java index 3284da09b7..90caee44c5 100644 --- a/plugin/src/main/java/org/opensearch/ml/rest/RestMLExecuteAction.java +++ b/plugin/src/main/java/org/opensearch/ml/rest/RestMLExecuteAction.java @@ -21,6 +21,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.input.Input; @@ -132,7 +133,21 @@ private void sendResponse(RestChannel channel, MLExecuteTaskResponse response) t private void reportError(final RestChannel channel, final Exception e, final RestStatus status) { ErrorMessage errorMessage = ErrorMessageFactory.createErrorMessage(e, status.getStatus()); - channel.sendResponse(new BytesRestResponse(RestStatus.fromCode(errorMessage.getStatus()), errorMessage.toString())); + try { + XContentBuilder builder = channel.newBuilder(); + builder.startObject(); + builder.field("status", errorMessage.getStatus()); + builder.startObject("error"); + builder.field("type", errorMessage.getType()); + builder.field("reason", errorMessage.getReason()); + builder.field("details", errorMessage.getDetails()); + builder.endObject(); + builder.endObject(); + channel.sendResponse(new BytesRestResponse(RestStatus.fromCode(errorMessage.getStatus()), builder)); + } catch (Exception exception) { + log.error("Failed to build xContent for an error response, so reply with a plain string.", exception); + channel.sendResponse(new BytesRestResponse(RestStatus.fromCode(errorMessage.getStatus()), errorMessage.toString())); + } } private boolean isClientError(Exception e) { diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLExecuteActionTests.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLExecuteActionTests.java index ac570a6a4d..acbaacab0b 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLExecuteActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLExecuteActionTests.java @@ -28,6 +28,7 @@ import org.mockito.MockitoAnnotations; import org.opensearch.client.node.NodeClient; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; import org.opensearch.core.rest.RestStatus; @@ -281,4 +282,59 @@ public void testPrepareRequestSystemException() throws Exception { "{\"error\":{\"reason\":\"System Error\",\"details\":\"System Exception\",\"type\":\"RuntimeException\"},\"status\":500}"; assertEquals(expectedError, response.content().utf8ToString()); } + + public void testAgentExecutionResponseXContent() throws Exception { + RestRequest request = getExecuteAgentRestRequest(); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + actionListener + .onFailure( + new RemoteTransportException("Remote Transport Exception", new IllegalArgumentException("Illegal Argument Exception")) + ); + return null; + }).when(client).execute(eq(MLExecuteTaskAction.INSTANCE), any(), any()); + doNothing().when(channel).sendResponse(any()); + when(channel.newBuilder()).thenReturn(XContentFactory.jsonBuilder()); + restMLExecuteAction.handleRequest(request, channel, client); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(MLExecuteTaskRequest.class); + verify(client, times(1)).execute(eq(MLExecuteTaskAction.INSTANCE), argumentCaptor.capture(), any()); + Input input = argumentCaptor.getValue().getInput(); + assertEquals(FunctionName.AGENT, input.getFunctionName()); + ArgumentCaptor restResponseArgumentCaptor = ArgumentCaptor.forClass(RestResponse.class); + verify(channel, times(1)).sendResponse(restResponseArgumentCaptor.capture()); + BytesRestResponse response = (BytesRestResponse) restResponseArgumentCaptor.getValue(); + assertEquals(RestStatus.BAD_REQUEST, response.status()); + assertEquals("application/json; charset=UTF-8", response.contentType()); + String expectedError = + "{\"status\":400,\"error\":{\"type\":\"IllegalArgumentException\",\"reason\":\"Invalid Request\",\"details\":\"Illegal Argument Exception\"}}"; + assertEquals(expectedError, response.content().utf8ToString()); + } + + public void testAgentExecutionResponsePlainText() throws Exception { + RestRequest request = getExecuteAgentRestRequest(); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + actionListener + .onFailure( + new RemoteTransportException("Remote Transport Exception", new IllegalArgumentException("Illegal Argument Exception")) + ); + return null; + }).when(client).execute(eq(MLExecuteTaskAction.INSTANCE), any(), any()); + doNothing().when(channel).sendResponse(any()); + restMLExecuteAction.handleRequest(request, channel, client); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(MLExecuteTaskRequest.class); + verify(client, times(1)).execute(eq(MLExecuteTaskAction.INSTANCE), argumentCaptor.capture(), any()); + Input input = argumentCaptor.getValue().getInput(); + assertEquals(FunctionName.AGENT, input.getFunctionName()); + ArgumentCaptor restResponseArgumentCaptor = ArgumentCaptor.forClass(RestResponse.class); + verify(channel, times(1)).sendResponse(restResponseArgumentCaptor.capture()); + BytesRestResponse response = (BytesRestResponse) restResponseArgumentCaptor.getValue(); + assertEquals(RestStatus.BAD_REQUEST, response.status()); + assertEquals("text/plain; charset=UTF-8", response.contentType()); + String expectedError = + "{\"error\":{\"reason\":\"Invalid Request\",\"details\":\"Illegal Argument Exception\",\"type\":\"IllegalArgumentException\"},\"status\":400}"; + assertEquals(expectedError, response.content().utf8ToString()); + } } From 88fd3e709bec003f55ee329f66d01a0df73253e3 Mon Sep 17 00:00:00 2001 From: zane-neo Date: Sat, 31 Aug 2024 08:04:21 +0800 Subject: [PATCH 06/23] Add refresh policy for writing operation (#2785) Signed-off-by: zane-neo --- .../java/org/opensearch/ml/memory/index/InteractionsIndex.java | 3 ++- .../ml/action/models/UpdateModelTransportAction.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java b/memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java index ad31485e50..3084a8621f 100644 --- a/memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java +++ b/memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java @@ -40,6 +40,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.Client; @@ -490,7 +491,7 @@ public void deleteConversation(String conversationId, ActionListener li internalListener.onResponse(true); return; } - BulkRequest request = Requests.bulkRequest(); + BulkRequest request = Requests.bulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); for (Interaction interaction : interactions) { DeleteRequest delRequest = Requests.deleteRequest(INTERACTIONS_INDEX_NAME).id(interaction.getId()); request.add(delRequest); diff --git a/plugin/src/main/java/org/opensearch/ml/action/models/UpdateModelTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/models/UpdateModelTransportAction.java index 2037996ffe..c4ecd57805 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/models/UpdateModelTransportAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/models/UpdateModelTransportAction.java @@ -326,7 +326,7 @@ private void updateModelWithRegisteringToAnotherModelGroup( ActionListener wrappedListener, boolean isUpdateModelCache ) { - UpdateRequest updateRequest = new UpdateRequest(ML_MODEL_INDEX, modelId); + UpdateRequest updateRequest = new UpdateRequest(ML_MODEL_INDEX, modelId).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); if (newModelGroupId != null) { modelAccessControlHelper .validateModelGroupAccess(user, newModelGroupId, client, ActionListener.wrap(hasNewModelGroupPermission -> { From 49d4a01d930b0236a1f373c50e7f7a9430bbf856 Mon Sep 17 00:00:00 2001 From: Mingshi Liu Date: Sun, 1 Sep 2024 18:47:34 -0700 Subject: [PATCH 07/23] Fix custom prompt substitute with List issue in ml inference search response processor (#2871) --- .../ml/common/connector/HttpConnector.java | 31 +-- .../ml/common/utils/StringUtils.java | 46 ++++ .../common/connector/HttpConnectorTest.java | 109 ---------- .../ml/common/utils/StringUtilsTest.java | 205 ++++++++++++++++++ .../MLInferenceSearchResponseProcessor.java | 4 +- ...InferenceSearchResponseProcessorTests.java | 100 ++++++++- ...tMLInferenceSearchResponseProcessorIT.java | 136 ++++++++++-- 7 files changed, 473 insertions(+), 158 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/connector/HttpConnector.java b/common/src/main/java/org/opensearch/ml/common/connector/HttpConnector.java index 287fbb8127..edf26b954d 100644 --- a/common/src/main/java/org/opensearch/ml/common/connector/HttpConnector.java +++ b/common/src/main/java/org/opensearch/ml/common/connector/HttpConnector.java @@ -10,6 +10,7 @@ import static org.opensearch.ml.common.connector.ConnectorProtocols.validateProtocol; import static org.opensearch.ml.common.utils.StringUtils.getParameterMap; import static org.opensearch.ml.common.utils.StringUtils.isJson; +import static org.opensearch.ml.common.utils.StringUtils.parseParameters; import java.io.IOException; import java.time.Instant; @@ -322,40 +323,16 @@ public T createPayload(String action, Map parameters) { if (connectorAction.isPresent() && connectorAction.get().getRequestBody() != null) { String payload = connectorAction.get().getRequestBody(); payload = fillNullParameters(parameters, payload); + parseParameters(parameters); StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); payload = substitutor.replace(payload); + if (!isJson(payload)) { - String payloadAfterEscape = connectorAction.get().getRequestBody(); - Map escapedParameters = escapeMapValues(parameters); - StringSubstitutor escapedSubstitutor = new StringSubstitutor(escapedParameters, "${parameters.", "}"); - payloadAfterEscape = escapedSubstitutor.replace(payloadAfterEscape); - if (!isJson(payloadAfterEscape)) { - throw new IllegalArgumentException("Invalid payload: " + payload); - } else { - payload = payloadAfterEscape; - } + throw new IllegalArgumentException("Invalid payload: " + payload); } return (T) payload; } return (T) parameters.get("http_body"); - - } - - public static Map escapeMapValues(Map parameters) { - Map escapedMap = new HashMap<>(); - if (parameters != null) { - for (Map.Entry entry : parameters.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - String escapedValue = escapeValue(value); - escapedMap.put(key, escapedValue); - } - } - return escapedMap; - } - - private static String escapeValue(String value) { - return value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t"); } protected String fillNullParameters(Map parameters, String payload) { diff --git a/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java index e71636e01b..57c24c22fd 100644 --- a/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java @@ -50,6 +50,7 @@ public class StringUtils { static { gson = new Gson(); } + public static final String TO_STRING_FUNCTION_NAME = ".toString()"; public static boolean isValidJsonString(String Json) { try { @@ -233,4 +234,49 @@ public static String getErrorMessage(String errorMessage, String modelId, Boolea return errorMessage + " Model ID: " + modelId; } } + + /** + * Collects the prefixes of the toString() method calls present in the values of the given map. + * + * @param map A map containing key-value pairs where the values may contain toString() method calls. + * @return A list of prefixes for the toString() method calls found in the map values. + */ + public static List collectToStringPrefixes(Map map) { + List prefixes = new ArrayList<>(); + for (String key : map.keySet()) { + String value = map.get(key); + if (value != null) { + Pattern pattern = Pattern.compile("\\$\\{parameters\\.(.+?)\\.toString\\(\\)\\}"); + Matcher matcher = pattern.matcher(value); + while (matcher.find()) { + String prefix = matcher.group(1); + prefixes.add(prefix); + } + } + } + return prefixes; + } + + /** + * Parses the given parameters map and processes the values containing toString() method calls. + * + * @param parameters A map containing key-value pairs where the values may contain toString() method calls. + * @return A new map with the processed values for the toString() method calls. + */ + public static Map parseParameters(Map parameters) { + if (parameters != null) { + List toStringParametersPrefixes = collectToStringPrefixes(parameters); + + if (!toStringParametersPrefixes.isEmpty()) { + for (String prefix : toStringParametersPrefixes) { + String value = parameters.get(prefix); + if (value != null) { + parameters.put(prefix + TO_STRING_FUNCTION_NAME, processTextDoc(value)); + } + } + } + } + return parameters; + } + } diff --git a/common/src/test/java/org/opensearch/ml/common/connector/HttpConnectorTest.java b/common/src/test/java/org/opensearch/ml/common/connector/HttpConnectorTest.java index a84652791f..0115ac1376 100644 --- a/common/src/test/java/org/opensearch/ml/common/connector/HttpConnectorTest.java +++ b/common/src/test/java/org/opensearch/ml/common/connector/HttpConnectorTest.java @@ -6,7 +6,6 @@ package org.opensearch.ml.common.connector; import static org.opensearch.ml.common.connector.ConnectorAction.ActionType.PREDICT; -import static org.opensearch.ml.common.utils.StringUtils.toJson; import java.io.IOException; import java.util.ArrayList; @@ -184,114 +183,6 @@ public void createPayload_InvalidJson() { connector.validatePayload(predictPayload); } - @Test - public void createPayloadWithString() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - parameters.put("context", "document1"); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - Assert.assertEquals("{\"prompt\": \"answer question based on context: document1\"}", predictPayload); - } - - @Test - public void createPayloadWithList() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - ArrayList listOfDocuments = new ArrayList<>(); - listOfDocuments.add("document1"); - listOfDocuments.add("document2"); - parameters.put("context", toJson(listOfDocuments)); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - } - - @Test - public void createPayloadWithNestedList() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - ArrayList listOfDocuments = new ArrayList<>(); - listOfDocuments.add("document1"); - ArrayList NestedListOfDocuments = new ArrayList<>(); - NestedListOfDocuments.add("document2"); - listOfDocuments.add(toJson(NestedListOfDocuments)); - parameters.put("context", toJson(listOfDocuments)); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - } - - @Test - public void createPayloadWithMap() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - Map mapOfDocuments = new HashMap<>(); - mapOfDocuments.put("name", "John"); - parameters.put("context", toJson(mapOfDocuments)); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - } - - @Test - public void createPayloadWithNestedMapOfString() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - Map mapOfDocuments = new HashMap<>(); - mapOfDocuments.put("name", "John"); - Map nestedMapOfDocuments = new HashMap<>(); - nestedMapOfDocuments.put("city", "New York"); - mapOfDocuments.put("hometown", toJson(nestedMapOfDocuments)); - parameters.put("context", toJson(mapOfDocuments)); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - } - - @Test - public void createPayloadWithNestedMapOfObject() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - Map mapOfDocuments = new HashMap<>(); - mapOfDocuments.put("name", "John"); - Map nestedMapOfDocuments = new HashMap<>(); - nestedMapOfDocuments.put("city", "New York"); - mapOfDocuments.put("hometown", nestedMapOfDocuments); - parameters.put("context", toJson(mapOfDocuments)); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - } - - @Test - public void createPayloadWithNestedListOfMapOfObject() { - String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; - HttpConnector connector = createHttpConnectorWithRequestBody(requestBody); - Map parameters = new HashMap<>(); - parameters.put("prompt", "answer question based on context: ${parameters.context}"); - ArrayList listOfDocuments = new ArrayList<>(); - listOfDocuments.add("document1"); - ArrayList NestedListOfDocuments = new ArrayList<>(); - Map mapOfDocuments = new HashMap<>(); - mapOfDocuments.put("name", "John"); - Map nestedMapOfDocuments = new HashMap<>(); - nestedMapOfDocuments.put("city", "New York"); - mapOfDocuments.put("hometown", nestedMapOfDocuments); - listOfDocuments.add(toJson(NestedListOfDocuments)); - parameters.put("context", toJson(listOfDocuments)); - String predictPayload = connector.createPayload(PREDICT.name(), parameters); - connector.validatePayload(predictPayload); - } - @Test public void createPayload() { HttpConnector connector = createHttpConnector(); diff --git a/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java b/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java index cf112d6ca3..a4b1460f39 100644 --- a/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java +++ b/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java @@ -6,7 +6,12 @@ package org.opensearch.ml.common.utils; import static org.junit.Assert.assertEquals; +import static org.opensearch.ml.common.utils.StringUtils.TO_STRING_FUNCTION_NAME; +import static org.opensearch.ml.common.utils.StringUtils.collectToStringPrefixes; +import static org.opensearch.ml.common.utils.StringUtils.parseParameters; +import static org.opensearch.ml.common.utils.StringUtils.toJson; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -14,6 +19,7 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.text.StringSubstitutor; import org.junit.Assert; import org.junit.Test; @@ -218,4 +224,203 @@ public void testGetErrorMessageWhenHiddenNull() { // Assert assertEquals(expected, result); } + + /** + * Tests the collectToStringPrefixes method with a map containing toString() method calls + * in the values. Verifies that the method correctly extracts the prefixes of the toString() + * method calls. + */ + @Test + public void testGetToStringPrefix() { + Map parameters = new HashMap<>(); + parameters + .put( + "prompt", + "answer question based on context: ${parameters.context.toString()} and conversation history based on history: ${parameters.history.toString()}" + ); + parameters.put("context", "${parameters.text.toString()}"); + + List prefixes = collectToStringPrefixes(parameters); + List expectPrefixes = new ArrayList<>(); + expectPrefixes.add("text"); + expectPrefixes.add("context"); + expectPrefixes.add("history"); + assertEquals(prefixes, expectPrefixes); + } + + /** + * Tests the parseParameters method with a map containing a list of strings as the value + * for the "context" key. Verifies that the method correctly processes the list and adds + * the processed value to the map with the expected key. Also tests the string substitution + * using the processed values. + */ + @Test + public void testParseParametersListToString() { + Map parameters = new HashMap<>(); + parameters.put("prompt", "answer question based on context: ${parameters.context.toString()}"); + ArrayList listOfDocuments = new ArrayList<>(); + listOfDocuments.add("document1"); + parameters.put("context", toJson(listOfDocuments)); + + parseParameters(parameters); + assertEquals(parameters.get("context" + TO_STRING_FUNCTION_NAME), "[\\\"document1\\\"]"); + + String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; + StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); + requestBody = substitutor.replace(requestBody); + assertEquals(requestBody, "{\"prompt\": \"answer question based on context: [\\\"document1\\\"]\"}"); + } + + /** + * Tests the parseParameters method with a map containing a list of strings as the value + * for the "context" key, and the "prompt" value containing escaped characters. Verifies + * that the method correctly processes the list and adds the processed value to the map + * with the expected key. Also tests the string substitution using the processed values. + */ + @Test + public void testParseParametersListToStringWithEscapedPrompt() { + Map parameters = new HashMap<>(); + parameters + .put( + "prompt", + "\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: ${parameters.context.toString()}. \\n\\n Human: please summarize the documents \\n\\n Assistant:" + ); + ArrayList listOfDocuments = new ArrayList<>(); + listOfDocuments.add("document1"); + parameters.put("context", toJson(listOfDocuments)); + + parseParameters(parameters); + assertEquals(parameters.get("context" + TO_STRING_FUNCTION_NAME), "[\\\"document1\\\"]"); + + String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; + StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); + requestBody = substitutor.replace(requestBody); + assertEquals( + requestBody, + "{\"prompt\": \"\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: [\\\"document1\\\"]. \\n\\n Human: please summarize the documents \\n\\n Assistant:\"}" + ); + } + + /** + * Tests the parseParameters method with a map containing a list of strings as the value + * for the "context" key, and the "prompt" value containing escaped characters. Verifies + * that the method correctly processes the list and adds the processed value to the map + * with the expected key. Also tests the string substitution using the processed values. + */ + @Test + public void testParseParametersListToStringModelConfig() { + Map parameters = new HashMap<>(); + parameters + .put( + "prompt", + "\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: ${parameters.model_config.context.toString()}. \\n\\n Human: please summarize the documents \\n\\n Assistant:" + ); + ArrayList listOfDocuments = new ArrayList<>(); + listOfDocuments.add("document1"); + parameters.put("model_config.context", toJson(listOfDocuments)); + + parseParameters(parameters); + assertEquals(parameters.get("model_config.context" + TO_STRING_FUNCTION_NAME), "[\\\"document1\\\"]"); + + String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; + StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); + requestBody = substitutor.replace(requestBody); + assertEquals( + requestBody, + "{\"prompt\": \"\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: [\\\"document1\\\"]. \\n\\n Human: please summarize the documents \\n\\n Assistant:\"}" + ); + } + + /** + * Tests the parseParameters method with a map containing a nested list of strings as the + * value for the "context" key. Verifies that the method correctly processes the nested + * list and adds the processed value to the map with the expected key. Also tests the + * string substitution using the processed values. + */ + @Test + public void testParseParametersNestedListToString() { + Map parameters = new HashMap<>(); + parameters.put("prompt", "answer question based on context: ${parameters.context.toString()}"); + ArrayList listOfDocuments = new ArrayList<>(); + listOfDocuments.add("document1"); + ArrayList NestedListOfDocuments = new ArrayList<>(); + NestedListOfDocuments.add("document2"); + listOfDocuments.add(toJson(NestedListOfDocuments)); + parameters.put("context", toJson(listOfDocuments)); + + parseParameters(parameters); + assertEquals(parameters.get("context" + TO_STRING_FUNCTION_NAME), "[\\\"document1\\\",\\\"[\\\\\\\"document2\\\\\\\"]\\\"]"); + + String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; + StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); + requestBody = substitutor.replace(requestBody); + assertEquals( + requestBody, + "{\"prompt\": \"answer question based on context: [\\\"document1\\\",\\\"[\\\\\\\"document2\\\\\\\"]\\\"]\"}" + ); + } + + /** + * Tests the parseParameters method with a map containing a map of strings as the value + * for the "context" key. Verifies that the method correctly processes the map and adds + * the processed value to the map with the expected key. Also tests the string substitution + * using the processed values. + */ + @Test + public void testParseParametersMapToString() { + Map parameters = new HashMap<>(); + parameters + .put( + "prompt", + "answer question based on context: ${parameters.context.toString()} and conversation history based on history: ${parameters.history.toString()}" + ); + Map mapOfDocuments = new HashMap<>(); + mapOfDocuments.put("name", "John"); + parameters.put("context", toJson(mapOfDocuments)); + parameters.put("history", "hello\n"); + parseParameters(parameters); + assertEquals(parameters.get("context" + TO_STRING_FUNCTION_NAME), "{\\\"name\\\":\\\"John\\\"}"); + String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; + StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); + requestBody = substitutor.replace(requestBody); + assertEquals( + requestBody, + "{\"prompt\": \"answer question based on context: {\\\"name\\\":\\\"John\\\"} and conversation history based on history: hello\\n\"}" + ); + } + + /** + * Tests the parseParameters method with a map containing a nested map of strings as the + * value for the "context" key. Verifies that the method correctly processes the nested + * map and adds the processed value to the map with the expected key. Also tests the + * string substitution using the processed values. + */ + @Test + public void testParseParametersNestedMapToString() { + Map parameters = new HashMap<>(); + parameters + .put( + "prompt", + "answer question based on context: ${parameters.context.toString()} and conversation history based on history: ${parameters.history.toString()}" + ); + Map mapOfDocuments = new HashMap<>(); + mapOfDocuments.put("name", "John"); + Map nestedMapOfDocuments = new HashMap<>(); + nestedMapOfDocuments.put("city", "New York"); + mapOfDocuments.put("hometown", toJson(nestedMapOfDocuments)); + parameters.put("context", toJson(mapOfDocuments)); + parameters.put("history", "hello\n"); + parseParameters(parameters); + assertEquals( + parameters.get("context" + TO_STRING_FUNCTION_NAME), + "{\\\"hometown\\\":\\\"{\\\\\\\"city\\\\\\\":\\\\\\\"New York\\\\\\\"}\\\",\\\"name\\\":\\\"John\\\"}" + ); + String requestBody = "{\"prompt\": \"${parameters.prompt}\"}"; + StringSubstitutor substitutor = new StringSubstitutor(parameters, "${parameters.", "}"); + requestBody = substitutor.replace(requestBody); + assertEquals( + requestBody, + "{\"prompt\": \"answer question based on context: {\\\"hometown\\\":\\\"{\\\\\\\"city\\\\\\\":\\\\\\\"New York\\\\\\\"}\\\",\\\"name\\\":\\\"John\\\"} and conversation history based on history: hello\\n\"}" + ); + } } diff --git a/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java b/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java index f3da7c77bc..38e62528f3 100644 --- a/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java +++ b/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java @@ -384,8 +384,8 @@ private void processPredictions( } } } - - modelParameters = StringUtils.getParameterMap(modelInputParameters); + Map modelParametersInString = StringUtils.getParameterMap(modelInputParameters); + modelParameters.putAll(modelParametersInString); Set inputMapKeys = new HashSet<>(modelParameters.keySet()); inputMapKeys.removeAll(modelConfigs.keySet()); diff --git a/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java b/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java index 850e466ba6..62b397f84b 100644 --- a/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java +++ b/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java @@ -169,10 +169,10 @@ public void onFailure(Exception e) { /** * Tests create processor with one_to_one is true * with custom prompt - * with many to one prediction, 5 documents in hits are calling 1 prediction tasks + * with one to one prediction, 5 documents in hits are calling 5 prediction tasks * @throws Exception if an error occurs during the test */ - public void testProcessResponseManyToOneWithCustomPrompt() throws Exception { + public void testProcessResponseOneToOneWithCustomPrompt() throws Exception { String newDocumentField = "context"; String modelOutputField = "response"; @@ -202,6 +202,102 @@ public void testProcessResponseManyToOneWithCustomPrompt() throws Exception { "{ \"prompt\": \"${model_config.prompt}\"}", client, TEST_XCONTENT_REGISTRY_FOR_QUERY, + true + ); + + SearchRequest request = getSearchRequest(); + String fieldName = "text"; + SearchResponse response = getSearchResponse(5, true, fieldName); + + ModelTensor modelTensor = ModelTensor.builder().dataAsMap(ImmutableMap.of("response", "there is 1 value")).build(); + ModelTensors modelTensors = ModelTensors.builder().mlModelTensors(Arrays.asList(modelTensor)).build(); + ModelTensorOutput mlModelTensorOutput = ModelTensorOutput.builder().mlModelOutputs(Arrays.asList(modelTensors)).build(); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + actionListener.onResponse(MLTaskResponse.builder().output(mlModelTensorOutput).build()); + return null; + }).when(client).execute(any(), any(), any()); + + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(SearchResponse newSearchResponse) { + assertEquals(newSearchResponse.getHits().getHits().length, 5); + assertEquals( + newSearchResponse.getHits().getHits()[0].getSourceAsMap().get(newDocumentField).toString(), + "there is 1 value" + ); + assertEquals( + newSearchResponse.getHits().getHits()[1].getSourceAsMap().get(newDocumentField).toString(), + "there is 1 value" + ); + assertEquals( + newSearchResponse.getHits().getHits()[2].getSourceAsMap().get(newDocumentField).toString(), + "there is 1 value" + ); + assertEquals( + newSearchResponse.getHits().getHits()[3].getSourceAsMap().get(newDocumentField).toString(), + "there is 1 value" + ); + assertEquals( + newSearchResponse.getHits().getHits()[4].getSourceAsMap().get(newDocumentField).toString(), + "there is 1 value" + ); + } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } + + }; + responseProcessor.processResponseAsync(request, response, responseContext, listener); + verify(client, times(5)).execute(any(), any(), any()); + } + + /** + * Tests create processor with one_to_one is false + * with custom prompt + * with many to one prediction, 5 documents in hits are calling 1 prediction tasks + * @throws Exception if an error occurs during the test + */ + public void testProcessResponseManyToOneWithCustomPrompt() throws Exception { + + String documentField = "text"; + String modelInputField = "context"; + List> inputMap = new ArrayList<>(); + Map input = new HashMap<>(); + input.put(modelInputField, documentField); + inputMap.add(input); + + String newDocumentField = "llm_response"; + String modelOutputField = "response"; + List> outputMap = new ArrayList<>(); + Map output = new HashMap<>(); + output.put(newDocumentField, modelOutputField); + outputMap.add(output); + Map modelConfig = new HashMap<>(); + modelConfig + .put( + "prompt", + "\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: ${parameters.context}. \\n\\n Human: please summarize the documents \\n\\n Assistant:" + ); + MLInferenceSearchResponseProcessor responseProcessor = new MLInferenceSearchResponseProcessor( + "model1", + inputMap, + outputMap, + modelConfig, + DEFAULT_MAX_PREDICTION_TASKS, + PROCESSOR_TAG, + DESCRIPTION, + false, + "remote", + false, + false, + false, + "{ \"parameters\": ${ml_inference.parameters} }", + client, + TEST_XCONTENT_REGISTRY_FOR_QUERY, false ); diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java index 64a9306691..9c82547623 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java @@ -36,6 +36,7 @@ public class RestMLInferenceSearchResponseProcessorIT extends MLCommonsRestTestC private String openAIChatModelId; private String bedrockEmbeddingModelId; private String localModelId; + private String bedrockClaudeModelId; private final String completionModelConnectorEntity = "{\n" + " \"name\": \"OpenAI text embedding model Connector\",\n" + " \"description\": \"The connector to public OpenAI text embedding model service\",\n" @@ -106,6 +107,47 @@ public class RestMLInferenceSearchResponseProcessorIT extends MLCommonsRestTestC + " ]\n" + "}"; + private final String bedrockClaudeModelConnectorEntity = "{\n" + + " \"name\": \"BedRock Claude instant-v1 Connector\",\n" + + " \"description\": \"The connector to bedrock for claude model\",\n" + + " \"version\": 1,\n" + + " \"protocol\": \"aws_sigv4\",\n" + + " \"parameters\": {\n" + + " \"region\": \"" + + GITHUB_CI_AWS_REGION + + "\",\n" + + " \"service_name\": \"bedrock\",\n" + + " \"anthropic_version\": \"bedrock-2023-05-31\",\n" + + " \"max_tokens_to_sample\": 8000,\n" + + " \"temperature\": 0.0001,\n" + + " \"response_filter\": \"$.completion\",\n" + + " \"stop_sequences\": [\"\\n\\nHuman:\",\"\\nObservation:\",\"\\n\\tObservation:\",\"\\nObservation\",\"\\n\\tObservation\",\"\\n\\nQuestion\"]\n" + + " },\n" + + " \"credential\": {\n" + + " \"access_key\": \"" + + AWS_ACCESS_KEY_ID + + "\",\n" + + " \"secret_key\": \"" + + AWS_SECRET_ACCESS_KEY + + "\",\n" + + " \"session_token\": \"" + + AWS_SESSION_TOKEN + + "\"\n" + + " },\n" + + " \"actions\": [\n" + + " {\n" + + " \"action_type\": \"predict\",\n" + + " \"method\": \"POST\",\n" + + " \"url\": \"https://bedrock-runtime.${parameters.region}.amazonaws.com/model/anthropic.claude-instant-v1/invoke\",\n" + + " \"headers\": {\n" + + " \"content-type\": \"application/json\",\n" + + " \"x-amz-content-sha256\": \"required\"\n" + + " },\n" + + " \"request_body\": \"{\\\"prompt\\\":\\\"${parameters.prompt}\\\", \\\"stop_sequences\\\": ${parameters.stop_sequences}, \\\"max_tokens_to_sample\\\":${parameters.max_tokens_to_sample}, \\\"temperature\\\":${parameters.temperature}, \\\"anthropic_version\\\":\\\"${parameters.anthropic_version}\\\" }\"\n" + + " }\n" + + " ]\n" + + "}"; + /** * Registers two remote models and creates an index and documents before running the tests. * @@ -119,7 +161,8 @@ public void setup() throws Exception { this.openAIChatModelId = registerRemoteModel(completionModelConnectorEntity, openAIChatModelName, true); String bedrockEmbeddingModelName = "bedrock embedding model " + randomAlphaOfLength(5); this.bedrockEmbeddingModelId = registerRemoteModel(bedrockEmbeddingModelConnectorEntity, bedrockEmbeddingModelName, true); - + String bedrockClaudeModelName = "bedrock claude model " + randomAlphaOfLength(5); + this.bedrockClaudeModelId = registerRemoteModel(bedrockClaudeModelConnectorEntity, bedrockClaudeModelName, true); String index_name = "daily_index"; String createIndexRequestBody = "{\n" + " \"mappings\": {\n" @@ -152,13 +195,14 @@ public void setup() throws Exception { /** * Tests the MLInferenceSearchResponseProcessor with a remote model and an object field as input. * It creates a search pipeline with the processor configured to use the remote model, - * performs a search using the pipeline, and verifies the inference results. - * + * performs a search using the pipeline, gathering search documents into context and added in a custom prompt + * Using a toString() in placeholder to specify the context needs to cast as string + * and verifies the inference results. * @throws Exception if any error occurs during the test */ - public void testMLInferenceProcessorRemoteModelObjectField() throws Exception { + public void testMLInferenceProcessorRemoteModelCustomPrompt() throws Exception { // Skip test if key is null - if (OPENAI_KEY == null) { + if (AWS_ACCESS_KEY_ID == null) { return; } String createPipelineRequestBody = "{\n" @@ -168,20 +212,26 @@ public void testMLInferenceProcessorRemoteModelObjectField() throws Exception { + " \"tag\": \"ml_inference\",\n" + " \"description\": \"This processor is going to run ml inference during search request\",\n" + " \"model_id\": \"" - + this.openAIChatModelId + + this.bedrockClaudeModelId + "\",\n" + + " \"function_name\": \"REMOTE\",\n" + " \"input_map\": [\n" + " {\n" - + " \"input\": \"weather\"\n" + + " \"context\": \"weather\"\n" + " }\n" + " ],\n" + " \"output_map\": [\n" + " {\n" - + " \"weather_embedding\": \"data[*].embedding\"\n" + + " \"llm_response\":\"$.response\"\n" + + " \n" + " }\n" + " ],\n" - + " \"ignore_missing\": false,\n" + + " \"model_config\": {\n" + + " \"prompt\":\"\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: ${parameters.context.toString()}. \\n\\n Human: please summarize the documents \\n\\n Assistant:\"\n" + + " },\n" + + " \"ignore_missing\":false,\n" + " \"ignore_failure\": false\n" + + " \n" + " }\n" + " }\n" + " ]\n" @@ -190,18 +240,13 @@ public void testMLInferenceProcessorRemoteModelObjectField() throws Exception { String query = "{\"query\":{\"term\":{\"weather\":{\"value\":\"sunny\"}}}}"; String index_name = "daily_index"; - String pipelineName = "weather_embedding_pipeline"; + String pipelineName = "qa_pipeline"; createSearchPipelineProcessor(createPipelineRequestBody, pipelineName); Map response = searchWithPipeline(client(), index_name, pipelineName, query); - Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.diary_embedding_size"), "1536"); - Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.weather"), "sunny"); - Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.diary[0]"), "happy"); - Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.diary[1]"), "first day at school"); - List embeddingList = (List) JsonPath.parse(response).read("$.hits.hits[0]._source.weather_embedding"); - Assert.assertEquals(embeddingList.size(), 1536); - Assert.assertEquals((Double) embeddingList.get(0), 0.00020525085, 0.005); - Assert.assertEquals((Double) embeddingList.get(1), -0.0071890163, 0.005); + System.out.println(response); + Assert.assertNotNull(JsonPath.parse(response).read("$.hits.hits[0]._source.llm_response")); + Assert.assertNotNull(JsonPath.parse(response).read("$.hits.hits[1]._source.llm_response")); } /** @@ -312,6 +357,61 @@ public void testMLInferenceProcessorRemoteModelNestedListField() throws Exceptio Assert.assertEquals((Double) embeddingList.get(1), -0.012508746, 0.005); } + /** + * Tests the MLInferenceSearchResponseProcessor with a remote model and an object field as input. + * It creates a search pipeline with the processor configured to use the remote model, + * performs a search using the pipeline, and verifies the inference results. + * + * @throws Exception if any error occurs during the test + */ + public void testMLInferenceProcessorRemoteModelObjectField() throws Exception { + // Skip test if key is null + if (OPENAI_KEY == null) { + return; + } + String createPipelineRequestBody = "{\n" + + " \"response_processors\": [\n" + + " {\n" + + " \"ml_inference\": {\n" + + " \"tag\": \"ml_inference\",\n" + + " \"description\": \"This processor is going to run ml inference during search request\",\n" + + " \"model_id\": \"" + + this.openAIChatModelId + + "\",\n" + + " \"input_map\": [\n" + + " {\n" + + " \"input\": \"weather\"\n" + + " }\n" + + " ],\n" + + " \"output_map\": [\n" + + " {\n" + + " \"weather_embedding\": \"data[*].embedding\"\n" + + " }\n" + + " ],\n" + + " \"ignore_missing\": false,\n" + + " \"ignore_failure\": false\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"; + + String query = "{\"query\":{\"term\":{\"weather\":{\"value\":\"sunny\"}}}}"; + + String index_name = "daily_index"; + String pipelineName = "weather_embedding_pipeline"; + createSearchPipelineProcessor(createPipelineRequestBody, pipelineName); + + Map response = searchWithPipeline(client(), index_name, pipelineName, query); + Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.diary_embedding_size"), "1536"); + Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.weather"), "sunny"); + Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.diary[0]"), "happy"); + Assert.assertEquals(JsonPath.parse(response).read("$.hits.hits[0]._source.diary[1]"), "first day at school"); + List embeddingList = (List) JsonPath.parse(response).read("$.hits.hits[0]._source.weather_embedding"); + Assert.assertEquals(embeddingList.size(), 1536); + Assert.assertEquals((Double) embeddingList.get(0), 0.00020525085, 0.005); + Assert.assertEquals((Double) embeddingList.get(1), -0.0071890163, 0.005); + } + /** * Tests the ML inference processor with a local model. * It registers, deploys, and gets a local model, creates a search pipeline with the ML inference processor From cc402b30d317f0830f9baccdf57c1a663f54a538 Mon Sep 17 00:00:00 2001 From: Bhavana Ramaram Date: Tue, 3 Sep 2024 22:17:00 -0500 Subject: [PATCH 08/23] fix breaking changes in config index fields (#2882) * fix breaking changes in config index fields Signed-off-by: Bhavana Ramaram --- .../org/opensearch/ml/common/CommonValue.java | 1 + .../org/opensearch/ml/common/MLConfig.java | 74 ++++++++++++++++--- .../config/MLConfigGetResponseTest.java | 4 +- .../config/GetConfigTransportActionTests.java | 25 ++++++- 4 files changed, 91 insertions(+), 13 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/CommonValue.java b/common/src/main/java/org/opensearch/ml/common/CommonValue.java index 933f00b5ad..06f917ee9d 100644 --- a/common/src/main/java/org/opensearch/ml/common/CommonValue.java +++ b/common/src/main/java/org/opensearch/ml/common/CommonValue.java @@ -575,6 +575,7 @@ public class CommonValue { public static final Version VERSION_2_12_0 = Version.fromString("2.12.0"); public static final Version VERSION_2_13_0 = Version.fromString("2.13.0"); public static final Version VERSION_2_14_0 = Version.fromString("2.14.0"); + public static final Version VERSION_2_15_0 = Version.fromString("2.15.0"); public static final Version VERSION_2_16_0 = Version.fromString("2.16.0"); public static final Version VERSION_2_17_0 = Version.fromString("2.17.0"); } diff --git a/common/src/main/java/org/opensearch/ml/common/MLConfig.java b/common/src/main/java/org/opensearch/ml/common/MLConfig.java index ccdeed5df3..8204174663 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLConfig.java +++ b/common/src/main/java/org/opensearch/ml/common/MLConfig.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.time.Instant; +import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -28,43 +29,74 @@ public class MLConfig implements ToXContentObject, Writeable { public static final String TYPE_FIELD = "type"; - public static final String CONFIG_TYPE_FIELD = "config_type"; - public static final String CONFIGURATION_FIELD = "configuration"; - public static final String ML_CONFIGURATION_FIELD = "ml_configuration"; - public static final String CREATE_TIME_FIELD = "create_time"; public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + // Adding below three new fields since the original fields, type, configuration, and last_update_time + // are not created with correct data types in config index due to missing schema version bump. + // Starting 2.15, it is suggested that below fields be used for creating new documents in config index + + public static final String CONFIG_TYPE_FIELD = "config_type"; + + public static final String ML_CONFIGURATION_FIELD = "ml_configuration"; + public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time"; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_NEW_CONFIG_FIELDS = CommonValue.VERSION_2_15_0; + @Setter private String type; + @Setter + private String configType; + private Configuration configuration; + private Configuration mlConfiguration; private final Instant createTime; private Instant lastUpdateTime; + private Instant lastUpdatedTime; @Builder(toBuilder = true) - public MLConfig(String type, Configuration configuration, Instant createTime, Instant lastUpdateTime) { + public MLConfig( + String type, + String configType, + Configuration configuration, + Configuration mlConfiguration, + Instant createTime, + Instant lastUpdateTime, + Instant lastUpdatedTime + ) { this.type = type; + this.configType = configType; this.configuration = configuration; + this.mlConfiguration = mlConfiguration; this.createTime = createTime; this.lastUpdateTime = lastUpdateTime; + this.lastUpdatedTime = lastUpdatedTime; } public MLConfig(StreamInput input) throws IOException { + Version streamInputVersion = input.getVersion(); this.type = input.readOptionalString(); if (input.readBoolean()) { configuration = new Configuration(input); } createTime = input.readOptionalInstant(); lastUpdateTime = input.readOptionalInstant(); + if (streamInputVersion.onOrAfter(MINIMAL_SUPPORTED_VERSION_FOR_NEW_CONFIG_FIELDS)) { + this.configType = input.readOptionalString(); + if (input.readBoolean()) { + mlConfiguration = new Configuration(input); + } + lastUpdatedTime = input.readOptionalInstant(); + } } @Override public void writeTo(StreamOutput out) throws IOException { + Version streamOutputVersion = out.getVersion(); out.writeOptionalString(type); if (configuration != null) { out.writeBoolean(true); @@ -74,16 +106,32 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalInstant(createTime); out.writeOptionalInstant(lastUpdateTime); + if (streamOutputVersion.onOrAfter(MINIMAL_SUPPORTED_VERSION_FOR_NEW_CONFIG_FIELDS)) { + out.writeOptionalString(configType); + if (mlConfiguration != null) { + out.writeBoolean(true); + mlConfiguration.writeTo(out); + } else { + out.writeBoolean(false); + } + out.writeOptionalInstant(lastUpdatedTime); + } } @Override public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { XContentBuilder builder = xContentBuilder.startObject(); if (type != null) { - builder.field(CONFIG_TYPE_FIELD, type); + builder.field(TYPE_FIELD, type); + } + if (configType != null) { + builder.field(CONFIG_TYPE_FIELD, configType); } if (configuration != null) { - builder.field(ML_CONFIGURATION_FIELD, configuration); + builder.field(CONFIGURATION_FIELD, configuration); + } + if (mlConfiguration != null) { + builder.field(ML_CONFIGURATION_FIELD, mlConfiguration); } if (createTime != null) { builder.field(CREATE_TIME_FIELD, createTime.toEpochMilli()); @@ -91,6 +139,9 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params if (lastUpdateTime != null) { builder.field(LAST_UPDATE_TIME_FIELD, lastUpdateTime.toEpochMilli()); } + if (lastUpdatedTime != null) { + builder.field(LAST_UPDATED_TIME_FIELD, lastUpdatedTime.toEpochMilli()); + } return builder.endObject(); } @@ -142,10 +193,13 @@ public static MLConfig parse(XContentParser parser) throws IOException { } return MLConfig .builder() - .type(configType == null ? type : configType) - .configuration(mlConfiguration == null ? configuration : mlConfiguration) + .type(type) + .configType(configType) + .configuration(configuration) + .mlConfiguration(mlConfiguration) .createTime(createTime) - .lastUpdateTime(lastUpdatedTime == null ? lastUpdateTime : lastUpdatedTime) + .lastUpdateTime(lastUpdateTime) + .lastUpdatedTime(lastUpdatedTime) .build(); } } diff --git a/common/src/test/java/org/opensearch/ml/common/transport/config/MLConfigGetResponseTest.java b/common/src/test/java/org/opensearch/ml/common/transport/config/MLConfigGetResponseTest.java index b187b4a8c8..5e24bcaa6e 100644 --- a/common/src/test/java/org/opensearch/ml/common/transport/config/MLConfigGetResponseTest.java +++ b/common/src/test/java/org/opensearch/ml/common/transport/config/MLConfigGetResponseTest.java @@ -60,7 +60,7 @@ public void MLConfigGetResponse_Builder() throws IOException { @Test public void writeTo() throws IOException { // create ml agent using mlConfig and mlConfigGetResponse - mlConfig = new MLConfig("olly_agent", new Configuration("agent_id"), Instant.EPOCH, Instant.EPOCH); + mlConfig = new MLConfig("olly_agent", null, new Configuration("agent_id"), null, Instant.EPOCH, Instant.EPOCH, Instant.EPOCH); MLConfigGetResponse mlConfigGetResponse = MLConfigGetResponse.builder().mlConfig(mlConfig).build(); // use write out for both agents BytesStreamOutput output = new BytesStreamOutput(); @@ -76,7 +76,7 @@ public void writeTo() throws IOException { @Test public void toXContent() throws IOException { - mlConfig = new MLConfig(null, null, null, null); + mlConfig = new MLConfig(null, null, null, null, null, null, null); MLConfigGetResponse mlConfigGetResponse = MLConfigGetResponse.builder().mlConfig(mlConfig).build(); XContentBuilder builder = XContentFactory.jsonBuilder(); ToXContent.Params params = EMPTY_PARAMS; diff --git a/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java index 0dbb79ef6c..afa4153a74 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/config/GetConfigTransportActionTests.java @@ -162,9 +162,32 @@ public void testDoExecute_Success() throws IOException { verify(actionListener).onResponse(any(MLConfigGetResponse.class)); } + @Test + public void testDoExecute_Success_ForNewFields() throws IOException { + String configID = "config_id"; + MLConfig mlConfig = new MLConfig(null, "olly_agent", null, new Configuration("agent_id"), Instant.EPOCH, null, Instant.EPOCH); + + XContentBuilder content = mlConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + BytesReference bytesReference = BytesReference.bytes(content); + GetResult getResult = new GetResult("indexName", configID, 111l, 111l, 111l, true, bytesReference, null, null); + GetResponse getResponse = new GetResponse(getResult); + ActionListener actionListener = mock(ActionListener.class); + MLConfigGetRequest request = new MLConfigGetRequest(configID); + Task task = mock(Task.class); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + getConfigTransportAction.doExecute(task, request, actionListener); + verify(actionListener).onResponse(any(MLConfigGetResponse.class)); + } + public GetResponse prepareMLConfig(String configID) throws IOException { - MLConfig mlConfig = new MLConfig("olly_agent", new Configuration("agent_id"), Instant.EPOCH, Instant.EPOCH); + MLConfig mlConfig = new MLConfig("olly_agent", null, new Configuration("agent_id"), null, Instant.EPOCH, Instant.EPOCH, null); XContentBuilder content = mlConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); BytesReference bytesReference = BytesReference.bytes(content); From 33a7c967d1af63d8137ec80514b4d12573a9aa07 Mon Sep 17 00:00:00 2001 From: Xun Zhang Date: Wed, 4 Sep 2024 10:28:15 -0700 Subject: [PATCH 09/23] offline batch ingestion API actions and data ingesters (#2844) * batch ingest API rest and transport actions Signed-off-by: Xun Zhang * add openAI ingester Signed-off-by: Xun Zhang * update batch ingestion field mapping interphase and address comments Signed-off-by: Xun Zhang * support multiple data sources as ingestion inputs Signed-off-by: Xun Zhang * use dedicated thread pool for ingestion Signed-off-by: Xun Zhang --------- Signed-off-by: Xun Zhang --- .../org/opensearch/ml/common/MLTaskType.java | 3 +- .../batch/MLBatchIngestionAction.java | 18 ++ .../batch/MLBatchIngestionInput.java | 152 ++++++++++ .../batch/MLBatchIngestionRequest.java | 82 ++++++ .../batch/MLBatchIngestionResponse.java | 81 ++++++ .../ml/common/utils/StringUtils.java | 14 + .../common/input/nlp/TextDocsMLInputTest.java | 1 - .../batch/MLBatchIngestionInputTests.java | 138 +++++++++ .../batch/MLBatchIngestionRequestTests.java | 113 ++++++++ .../batch/MLBatchIngestionResponseTests.java | 83 ++++++ .../ml/common/utils/StringUtilsTest.java | 34 +++ .../memory/index/InteractionsIndexTests.java | 1 - ml-algorithms/build.gradle | 6 +- .../ml/engine/MLEngineClassLoader.java | 21 +- .../ml/engine/annotation/Ingester.java | 17 ++ .../ml/engine/ingest/AbstractIngestion.java | 215 ++++++++++++++ .../ml/engine/ingest/Ingestable.java | 19 ++ .../ml/engine/ingest/OpenAIDataIngestion.java | 131 +++++++++ .../ml/engine/ingest/S3DataIngestion.java | 206 ++++++++++++++ .../MLSdkAsyncHttpResponseHandlerTest.java | 2 - .../engine/ingest/AbstractIngestionTests.java | 262 ++++++++++++++++++ .../engine/ingest/S3DataIngestionTests.java | 62 +++++ .../batch/TransportBatchIngestionAction.java | 178 ++++++++++++ .../ml/plugin/MachineLearningPlugin.java | 10 +- .../ml/rest/RestMLBatchIngestAction.java | 65 +++++ .../ml/rest/RestMLPredictionAction.java | 2 - .../opensearch/ml/utils/RestActionUtils.java | 1 - .../plugin-metadata/plugin-security.policy | 6 + .../TransportBatchIngestionActionTests.java | 254 +++++++++++++++++ .../rest/RestMLBatchIngestionActionTests.java | 126 +++++++++ .../org/opensearch/ml/utils/TestHelper.java | 23 ++ 31 files changed, 2314 insertions(+), 12 deletions(-) create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionAction.java create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequest.java create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponse.java create mode 100644 common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInputTests.java create mode 100644 common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequestTests.java create mode 100644 common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponseTests.java create mode 100644 ml-algorithms/src/main/java/org/opensearch/ml/engine/annotation/Ingester.java create mode 100644 ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java create mode 100644 ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/Ingestable.java create mode 100644 ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/OpenAIDataIngestion.java create mode 100644 ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/S3DataIngestion.java create mode 100644 ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java create mode 100644 ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java create mode 100644 plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java create mode 100644 plugin/src/main/java/org/opensearch/ml/rest/RestMLBatchIngestAction.java create mode 100644 plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java create mode 100644 plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java diff --git a/common/src/main/java/org/opensearch/ml/common/MLTaskType.java b/common/src/main/java/org/opensearch/ml/common/MLTaskType.java index db2f67f369..e17b36a4dd 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLTaskType.java +++ b/common/src/main/java/org/opensearch/ml/common/MLTaskType.java @@ -15,5 +15,6 @@ public enum MLTaskType { @Deprecated LOAD_MODEL, REGISTER_MODEL, - DEPLOY_MODEL + DEPLOY_MODEL, + BATCH_INGEST } diff --git a/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionAction.java b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionAction.java new file mode 100644 index 0000000000..3e0d39a692 --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionAction.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import org.opensearch.action.ActionType; + +public class MLBatchIngestionAction extends ActionType { + public static MLBatchIngestionAction INSTANCE = new MLBatchIngestionAction(); + public static final String NAME = "cluster:admin/opensearch/ml/batch_ingestion"; + + private MLBatchIngestionAction() { + super(NAME, MLBatchIngestionResponse::new); + } + +} diff --git a/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java new file mode 100644 index 0000000000..e7050f0bd2 --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java @@ -0,0 +1,152 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import lombok.Builder; +import lombok.Getter; + +/** + * ML batch ingestion data: index, field mapping and input and out files. + */ +public class MLBatchIngestionInput implements ToXContentObject, Writeable { + + public static final String INDEX_NAME_FIELD = "index_name"; + public static final String FIELD_MAP_FIELD = "field_map"; + public static final String DATA_SOURCE_FIELD = "data_source"; + public static final String CONNECTOR_CREDENTIAL_FIELD = "credential"; + @Getter + private String indexName; + @Getter + private Map fieldMapping; + @Getter + private Map dataSources; + @Getter + private Map credential; + + @Builder(toBuilder = true) + public MLBatchIngestionInput( + String indexName, + Map fieldMapping, + Map dataSources, + Map credential + ) { + if (indexName == null) { + throw new IllegalArgumentException( + "The index name for data ingestion is missing. Please provide a valid index name to proceed." + ); + } + if (dataSources == null) { + throw new IllegalArgumentException( + "No data sources were provided for ingestion. Please specify at least one valid data source to proceed." + ); + } + this.indexName = indexName; + this.fieldMapping = fieldMapping; + this.dataSources = dataSources; + this.credential = credential; + } + + public static MLBatchIngestionInput parse(XContentParser parser) throws IOException { + String indexName = null; + Map fieldMapping = null; + Map dataSources = null; + Map credential = new HashMap<>(); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + + switch (fieldName) { + case INDEX_NAME_FIELD: + indexName = parser.text(); + break; + case FIELD_MAP_FIELD: + fieldMapping = parser.map(); + break; + case CONNECTOR_CREDENTIAL_FIELD: + credential = parser.mapStrings(); + break; + case DATA_SOURCE_FIELD: + dataSources = parser.map(); + break; + default: + parser.skipChildren(); + break; + } + } + return new MLBatchIngestionInput(indexName, fieldMapping, dataSources, credential); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (indexName != null) { + builder.field(INDEX_NAME_FIELD, indexName); + } + if (fieldMapping != null) { + builder.field(FIELD_MAP_FIELD, fieldMapping); + } + if (credential != null) { + builder.field(CONNECTOR_CREDENTIAL_FIELD, credential); + } + if (dataSources != null) { + builder.field(DATA_SOURCE_FIELD, dataSources); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput output) throws IOException { + output.writeOptionalString(indexName); + if (fieldMapping != null) { + output.writeBoolean(true); + output.writeMap(fieldMapping, StreamOutput::writeString, StreamOutput::writeGenericValue); + } else { + output.writeBoolean(false); + } + if (credential != null) { + output.writeBoolean(true); + output.writeMap(credential, StreamOutput::writeString, StreamOutput::writeString); + } else { + output.writeBoolean(false); + } + if (dataSources != null) { + output.writeBoolean(true); + output.writeMap(dataSources, StreamOutput::writeString, StreamOutput::writeGenericValue); + } else { + output.writeBoolean(false); + } + } + + public MLBatchIngestionInput(StreamInput input) throws IOException { + indexName = input.readOptionalString(); + if (input.readBoolean()) { + fieldMapping = input.readMap(s -> s.readString(), s -> s.readGenericValue()); + } + if (input.readBoolean()) { + credential = input.readMap(s -> s.readString(), s -> s.readString()); + } + if (input.readBoolean()) { + dataSources = input.readMap(s -> s.readString(), s -> s.readGenericValue()); + } + } + +} diff --git a/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequest.java b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequest.java new file mode 100644 index 0000000000..c6bf0de6d6 --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequest.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import static org.opensearch.action.ValidateActions.addValidationError; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import lombok.experimental.FieldDefaults; + +@Getter +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +@ToString +public class MLBatchIngestionRequest extends ActionRequest { + + private MLBatchIngestionInput mlBatchIngestionInput; + + @Builder + public MLBatchIngestionRequest(MLBatchIngestionInput mlBatchIngestionInput) { + this.mlBatchIngestionInput = mlBatchIngestionInput; + } + + public MLBatchIngestionRequest(StreamInput in) throws IOException { + super(in); + this.mlBatchIngestionInput = new MLBatchIngestionInput(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + this.mlBatchIngestionInput.writeTo(out); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException exception = null; + if (mlBatchIngestionInput == null) { + exception = addValidationError("The input for ML batch ingestion cannot be null.", exception); + } + if (mlBatchIngestionInput != null && mlBatchIngestionInput.getCredential() == null) { + exception = addValidationError("The credential for ML batch ingestion cannot be null", exception); + } + if (mlBatchIngestionInput != null && mlBatchIngestionInput.getDataSources() == null) { + exception = addValidationError("The data sources for ML batch ingestion cannot be null", exception); + } + + return exception; + } + + public static MLBatchIngestionRequest fromActionRequest(ActionRequest actionRequest) { + if (actionRequest instanceof MLBatchIngestionRequest) { + return (MLBatchIngestionRequest) actionRequest; + } + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStreamStreamOutput osso = new OutputStreamStreamOutput(baos)) { + actionRequest.writeTo(osso); + try (StreamInput input = new InputStreamStreamInput(new ByteArrayInputStream(baos.toByteArray()))) { + return new MLBatchIngestionRequest(input); + } + } catch (IOException e) { + throw new UncheckedIOException("failed to parse ActionRequest into MLBatchIngestionRequest", e); + } + + } +} diff --git a/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponse.java b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponse.java new file mode 100644 index 0000000000..42ae6857b9 --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponse.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.ml.common.MLTaskType; + +import lombok.Getter; + +@Getter +public class MLBatchIngestionResponse extends ActionResponse implements ToXContentObject { + public static final String TASK_ID_FIELD = "task_id"; + public static final String TASK_TYPE_FIELD = "task_type"; + public static final String STATUS_FIELD = "status"; + + private String taskId; + private MLTaskType taskType; + private String status; + + public MLBatchIngestionResponse(StreamInput in) throws IOException { + super(in); + this.taskId = in.readString(); + this.taskType = in.readEnum(MLTaskType.class); + this.status = in.readString(); + } + + public MLBatchIngestionResponse(String taskId, MLTaskType mlTaskType, String status) { + this.taskId = taskId; + this.taskType = mlTaskType; + this.status = status; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(taskId); + out.writeEnum(taskType); + out.writeString(status); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field(TASK_ID_FIELD, taskId); + if (taskType != null) { + builder.field(TASK_TYPE_FIELD, taskType); + } + builder.field(STATUS_FIELD, status); + builder.endObject(); + return builder; + } + + public static MLBatchIngestionResponse fromActionResponse(ActionResponse actionResponse) { + if (actionResponse instanceof MLBatchIngestionResponse) { + return (MLBatchIngestionResponse) actionResponse; + } + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStreamStreamOutput osso = new OutputStreamStreamOutput(baos)) { + actionResponse.writeTo(osso); + try (StreamInput input = new InputStreamStreamInput(new ByteArrayInputStream(baos.toByteArray()))) { + return new MLBatchIngestionResponse(input); + } + } catch (IOException e) { + throw new UncheckedIOException("failed to parse ActionResponse into MLBatchIngestionResponse", e); + } + } +} diff --git a/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java index 57c24c22fd..4bf74de3a9 100644 --- a/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java @@ -279,4 +279,18 @@ public static Map parseParameters(Map parameters return parameters; } + public static String obtainFieldNameFromJsonPath(String jsonPath) { + String[] parts = jsonPath.split("\\."); + + // Get the last part which is the field name + return parts[parts.length - 1]; + } + + public static String getJsonPath(String jsonPathWithSource) { + // Find the index of the first occurrence of "$." + int startIndex = jsonPathWithSource.indexOf("$."); + + // Extract the substring from the startIndex to the end of the input string + return (startIndex != -1) ? jsonPathWithSource.substring(startIndex) : jsonPathWithSource; + } } diff --git a/common/src/test/java/org/opensearch/ml/common/input/nlp/TextDocsMLInputTest.java b/common/src/test/java/org/opensearch/ml/common/input/nlp/TextDocsMLInputTest.java index 5631071835..4b0947ba15 100644 --- a/common/src/test/java/org/opensearch/ml/common/input/nlp/TextDocsMLInputTest.java +++ b/common/src/test/java/org/opensearch/ml/common/input/nlp/TextDocsMLInputTest.java @@ -57,7 +57,6 @@ public void parseTextDocsMLInput() throws IOException { XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); input.toXContent(builder, ToXContent.EMPTY_PARAMS); String jsonStr = builder.toString(); - System.out.println(jsonStr); parseMLInput(jsonStr, 2); } diff --git a/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInputTests.java b/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInputTests.java new file mode 100644 index 0000000000..abfef0c6f9 --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInputTests.java @@ -0,0 +1,138 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchModule; + +public class MLBatchIngestionInputTests { + + private MLBatchIngestionInput mlBatchIngestionInput; + + private Map dataSource; + + @Rule + public final ExpectedException exceptionRule = ExpectedException.none(); + + private final String expectedInputStr = "{" + + "\"index_name\":\"test index\"," + + "\"field_map\":{" + + "\"chapter\":\"chapter_embedding\"" + + "}," + + "\"credential\":{" + + "\"region\":\"test region\"" + + "}," + + "\"data_source\":{" + + "\"source\":[\"s3://samplebucket/output/sampleresults.json.out\"]," + + "\"type\":\"s3\"" + + "}" + + "}"; + + @Before + public void setUp() { + dataSource = new HashMap<>(); + dataSource.put("type", "s3"); + dataSource.put("source", Arrays.asList("s3://samplebucket/output/sampleresults.json.out")); + + Map credentials = Map.of("region", "test region"); + Map fieldMapping = Map.of("chapter", "chapter_embedding"); + + mlBatchIngestionInput = MLBatchIngestionInput + .builder() + .indexName("test index") + .credential(credentials) + .fieldMapping(fieldMapping) + .dataSources(dataSource) + .build(); + } + + @Test + public void constructorMLBatchIngestionInput_NullName() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("The index name for data ingestion is missing. Please provide a valid index name to proceed."); + + MLBatchIngestionInput.builder().indexName(null).dataSources(dataSource).build(); + } + + @Test + public void constructorMLBatchIngestionInput_NullSource() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule + .expectMessage("No data sources were provided for ingestion. Please specify at least one valid data source to proceed."); + MLBatchIngestionInput.builder().indexName("test index").dataSources(null).build(); + } + + @Test + public void testToXContent_FullFields() throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder(); + mlBatchIngestionInput.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertNotNull(builder); + String jsonStr = builder.toString(); + assertEquals(expectedInputStr, jsonStr); + } + + @Test + public void testParse() throws Exception { + testParseFromJsonString(expectedInputStr, parsedInput -> { + assertEquals("test index", parsedInput.getIndexName()); + assertEquals("test region", parsedInput.getCredential().get("region")); + assertEquals("chapter_embedding", parsedInput.getFieldMapping().get("chapter")); + assertEquals("s3", parsedInput.getDataSources().get("type")); + }); + } + + private void testParseFromJsonString(String expectedInputString, Consumer verify) throws Exception { + XContentParser parser = XContentType.JSON + .xContent() + .createParser( + new NamedXContentRegistry(new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()), + LoggingDeprecationHandler.INSTANCE, + expectedInputString + ); + parser.nextToken(); + MLBatchIngestionInput parsedInput = MLBatchIngestionInput.parse(parser); + verify.accept(parsedInput); + } + + @Test + public void readInputStream_Success() throws IOException { + readInputStream( + mlBatchIngestionInput, + parsedInput -> assertEquals(mlBatchIngestionInput.getIndexName(), parsedInput.getIndexName()) + ); + } + + private void readInputStream(MLBatchIngestionInput input, Consumer verify) throws IOException { + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + input.writeTo(bytesStreamOutput); + StreamInput streamInput = bytesStreamOutput.bytes().streamInput(); + MLBatchIngestionInput parsedInput = new MLBatchIngestionInput(streamInput); + verify.accept(parsedInput); + } +} diff --git a/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequestTests.java b/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequestTests.java new file mode 100644 index 0000000000..ccbc0477c8 --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionRequestTests.java @@ -0,0 +1,113 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; + +public class MLBatchIngestionRequestTests { + private MLBatchIngestionInput mlBatchIngestionInput; + private MLBatchIngestionRequest mlBatchIngestionRequest; + + @Before + public void setUp() { + mlBatchIngestionInput = MLBatchIngestionInput + .builder() + .indexName("test_index_name") + .credential(Map.of("region", "test region")) + .fieldMapping(Map.of("chapter", "chapter_embedding")) + .dataSources(Map.of("type", "s3")) + .build(); + mlBatchIngestionRequest = MLBatchIngestionRequest.builder().mlBatchIngestionInput(mlBatchIngestionInput).build(); + } + + @Test + public void writeToSuccess() throws IOException { + BytesStreamOutput output = new BytesStreamOutput(); + mlBatchIngestionRequest.writeTo(output); + MLBatchIngestionRequest parsedRequest = new MLBatchIngestionRequest(output.bytes().streamInput()); + assertEquals( + mlBatchIngestionRequest.getMlBatchIngestionInput().getIndexName(), + parsedRequest.getMlBatchIngestionInput().getIndexName() + ); + assertEquals( + mlBatchIngestionRequest.getMlBatchIngestionInput().getCredential(), + parsedRequest.getMlBatchIngestionInput().getCredential() + ); + assertEquals( + mlBatchIngestionRequest.getMlBatchIngestionInput().getFieldMapping(), + parsedRequest.getMlBatchIngestionInput().getFieldMapping() + ); + assertEquals( + mlBatchIngestionRequest.getMlBatchIngestionInput().getDataSources(), + parsedRequest.getMlBatchIngestionInput().getDataSources() + ); + } + + @Test + public void validateSuccess() { + assertNull(mlBatchIngestionRequest.validate()); + } + + @Test + public void validateWithNullInputException() { + MLBatchIngestionRequest mlBatchIngestionRequest1 = MLBatchIngestionRequest.builder().build(); + ActionRequestValidationException exception = mlBatchIngestionRequest1.validate(); + assertEquals("Validation Failed: 1: The input for ML batch ingestion cannot be null.;", exception.getMessage()); + } + + @Test + public void fromActionRequestWithBatchRequestSuccess() { + assertSame(MLBatchIngestionRequest.fromActionRequest(mlBatchIngestionRequest), mlBatchIngestionRequest); + } + + @Test + public void fromActionRequestWithNonRequestSuccess() { + ActionRequest actionRequest = new ActionRequest() { + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + mlBatchIngestionRequest.writeTo(out); + } + }; + MLBatchIngestionRequest result = MLBatchIngestionRequest.fromActionRequest(actionRequest); + assertNotSame(result, mlBatchIngestionRequest); + assertEquals(mlBatchIngestionRequest.getMlBatchIngestionInput().getIndexName(), result.getMlBatchIngestionInput().getIndexName()); + } + + @Test(expected = UncheckedIOException.class) + public void fromActionRequestIOException() { + ActionRequest actionRequest = new ActionRequest() { + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new IOException(); + } + }; + MLBatchIngestionRequest.fromActionRequest(actionRequest); + } +} diff --git a/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponseTests.java b/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponseTests.java new file mode 100644 index 0000000000..b0b61f04e0 --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionResponseTests.java @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.batch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.junit.Before; +import org.junit.Test; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.ml.common.MLTaskType; +import org.opensearch.ml.common.TestHelper; + +public class MLBatchIngestionResponseTests { + + MLBatchIngestionResponse mlBatchIngestionResponse; + + @Before + public void setUp() { + mlBatchIngestionResponse = new MLBatchIngestionResponse("testId", MLTaskType.BATCH_INGEST, "Created"); + } + + @Test + public void toXContent() throws IOException { + XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); + mlBatchIngestionResponse.toXContent(builder, ToXContent.EMPTY_PARAMS); + String content = TestHelper.xContentBuilderToString(builder); + assertEquals("{\"task_id\":\"testId\",\"task_type\":\"BATCH_INGEST\",\"status\":\"Created\"}", content); + } + + @Test + public void readFromStream() throws IOException { + BytesStreamOutput output = new BytesStreamOutput(); + mlBatchIngestionResponse.writeTo(output); + + MLBatchIngestionResponse response2 = new MLBatchIngestionResponse(output.bytes().streamInput()); + assertEquals("testId", response2.getTaskId()); + assertEquals("Created", response2.getStatus()); + } + + @Test + public void fromActionResponseWithMLBatchIngestionResponseSuccess() { + MLBatchIngestionResponse responseFromActionResponse = MLBatchIngestionResponse.fromActionResponse(mlBatchIngestionResponse); + assertSame(mlBatchIngestionResponse, responseFromActionResponse); + assertEquals(mlBatchIngestionResponse.getTaskType(), responseFromActionResponse.getTaskType()); + } + + @Test + public void fromActionResponseSuccess() { + ActionResponse actionResponse = new ActionResponse() { + @Override + public void writeTo(StreamOutput out) throws IOException { + mlBatchIngestionResponse.writeTo(out); + } + }; + MLBatchIngestionResponse responseFromActionResponse = MLBatchIngestionResponse.fromActionResponse(actionResponse); + assertNotSame(mlBatchIngestionResponse, responseFromActionResponse); + assertEquals(mlBatchIngestionResponse.getTaskType(), responseFromActionResponse.getTaskType()); + } + + @Test(expected = UncheckedIOException.class) + public void fromActionResponseIOException() { + ActionResponse actionResponse = new ActionResponse() { + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new IOException(); + } + }; + MLBatchIngestionResponse.fromActionResponse(actionResponse); + } +} diff --git a/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java b/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java index a4b1460f39..aed76c5658 100644 --- a/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java +++ b/common/src/test/java/org/opensearch/ml/common/utils/StringUtilsTest.java @@ -8,6 +8,8 @@ import static org.junit.Assert.assertEquals; import static org.opensearch.ml.common.utils.StringUtils.TO_STRING_FUNCTION_NAME; import static org.opensearch.ml.common.utils.StringUtils.collectToStringPrefixes; +import static org.opensearch.ml.common.utils.StringUtils.getJsonPath; +import static org.opensearch.ml.common.utils.StringUtils.obtainFieldNameFromJsonPath; import static org.opensearch.ml.common.utils.StringUtils.parseParameters; import static org.opensearch.ml.common.utils.StringUtils.toJson; @@ -423,4 +425,36 @@ public void testParseParametersNestedMapToString() { "{\"prompt\": \"answer question based on context: {\\\"hometown\\\":\\\"{\\\\\\\"city\\\\\\\":\\\\\\\"New York\\\\\\\"}\\\",\\\"name\\\":\\\"John\\\"} and conversation history based on history: hello\\n\"}" ); } + + @Test + public void testObtainFieldNameFromJsonPath_ValidJsonPath() { + // Test with a typical JSONPath + String jsonPath = "$.response.body.data[*].embedding"; + String fieldName = obtainFieldNameFromJsonPath(jsonPath); + assertEquals("embedding", fieldName); + } + + @Test + public void testObtainFieldNameFromJsonPath_WithPrefix() { + // Test with JSONPath that has a prefix + String jsonPath = "source[1].$.response.body.data[*].embedding"; + String fieldName = obtainFieldNameFromJsonPath(jsonPath); + assertEquals("embedding", fieldName); + } + + @Test + public void testGetJsonPath_ValidJsonPathWithSource() { + // Test with a JSONPath that includes a source prefix + String input = "source[1].$.response.body.data[*].embedding"; + String result = getJsonPath(input); + assertEquals("$.response.body.data[*].embedding", result); + } + + @Test + public void testGetJsonPath_ValidJsonPathWithoutSource() { + // Test with a JSONPath that does not include a source prefix + String input = "$.response.body.data[*].embedding"; + String result = getJsonPath(input); + assertEquals("$.response.body.data[*].embedding", result); + } } diff --git a/memory/src/test/java/org/opensearch/ml/memory/index/InteractionsIndexTests.java b/memory/src/test/java/org/opensearch/ml/memory/index/InteractionsIndexTests.java index 4da9f9d68e..042a4a3a91 100644 --- a/memory/src/test/java/org/opensearch/ml/memory/index/InteractionsIndexTests.java +++ b/memory/src/test/java/org/opensearch/ml/memory/index/InteractionsIndexTests.java @@ -750,7 +750,6 @@ public void testGetSg_NoIndex_ThenFail() { interactionsIndex.getInteraction("iid", getListener); ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Exception.class); verify(getListener, times(1)).onFailure(argCaptor.capture()); - System.out.println(argCaptor.getValue().getMessage()); assert (argCaptor .getValue() .getMessage() diff --git a/ml-algorithms/build.gradle b/ml-algorithms/build.gradle index e97849b019..5b7d146da7 100644 --- a/ml-algorithms/build.gradle +++ b/ml-algorithms/build.gradle @@ -7,6 +7,7 @@ import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform plugins { id 'java' + id 'java-library' id 'jacoco' id "io.freefair.lombok" id 'com.diffplug.spotless' version '6.25.0' @@ -62,9 +63,12 @@ dependencies { } implementation platform('software.amazon.awssdk:bom:2.25.40') - implementation 'software.amazon.awssdk:auth' + api 'software.amazon.awssdk:auth:2.25.40' implementation 'software.amazon.awssdk:apache-client' implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1' + implementation group: 'software.amazon.awssdk', name: 'aws-core', version: '2.25.40' + implementation group: 'software.amazon.awssdk', name: 's3', version: '2.25.40' + implementation group: 'software.amazon.awssdk', name: 'regions', version: '2.25.40' implementation 'com.jayway.jsonpath:json-path:2.9.0' implementation group: 'org.json', name: 'json', version: '20231013' implementation group: 'software.amazon.awssdk', name: 'netty-nio-client', version: '2.25.40' diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/MLEngineClassLoader.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/MLEngineClassLoader.java index 9add9a4f9e..7205883e7f 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/MLEngineClassLoader.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/MLEngineClassLoader.java @@ -20,6 +20,7 @@ import org.opensearch.ml.common.exception.MLException; import org.opensearch.ml.engine.annotation.ConnectorExecutor; import org.opensearch.ml.engine.annotation.Function; +import org.opensearch.ml.engine.annotation.Ingester; import org.reflections.Reflections; @SuppressWarnings("removal") @@ -31,6 +32,7 @@ public class MLEngineClassLoader { */ private static Map, Class> mlAlgoClassMap = new HashMap<>(); private static Map> connectorExecutorMap = new HashMap<>(); + private static Map> ingesterMap = new HashMap<>(); /** * This map contains pre-created thread-safe ML objects. @@ -41,6 +43,7 @@ public class MLEngineClassLoader { try { AccessController.doPrivileged((PrivilegedExceptionAction) () -> { loadClassMapping(); + loadIngestClassMapping(); return null; }); } catch (PrivilegedActionException e) { @@ -69,7 +72,7 @@ public static Object deregister(Enum functionName) { return mlObjects.remove(functionName); } - public static void loadClassMapping() { + private static void loadClassMapping() { Reflections reflections = new Reflections("org.opensearch.ml.engine.algorithms"); Set> classes = reflections.getTypesAnnotatedWith(Function.class); @@ -93,6 +96,19 @@ public static void loadClassMapping() { } } + private static void loadIngestClassMapping() { + Reflections reflections = new Reflections("org.opensearch.ml.engine.ingest"); + Set> ingesterClasses = reflections.getTypesAnnotatedWith(Ingester.class); + // Load ingester class + for (Class clazz : ingesterClasses) { + Ingester ingester = clazz.getAnnotation(Ingester.class); + String ingesterSource = ingester.value(); + if (ingesterSource != null) { + ingesterMap.put(ingesterSource, clazz); + } + } + } + @SuppressWarnings("unchecked") public static S initInstance(T type, I in, Class constructorParamClass) { return initInstance(type, in, constructorParamClass, null); @@ -120,6 +136,9 @@ public static S initInstance(T type, I in, Class con if (clazz == null) { clazz = connectorExecutorMap.get(type); } + if (clazz == null) { + clazz = ingesterMap.get(type); + } if (clazz == null) { throw new IllegalArgumentException("Can't find class for type " + type); } diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/annotation/Ingester.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/annotation/Ingester.java new file mode 100644 index 0000000000..6bacf7c76d --- /dev/null +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/annotation/Ingester.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface Ingester { + String value(); +} diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java new file mode 100644 index 0000000000..be61f09e28 --- /dev/null +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java @@ -0,0 +1,215 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.ingest; + +import static org.opensearch.ml.common.utils.StringUtils.getJsonPath; +import static org.opensearch.ml.common.utils.StringUtils.obtainFieldNameFromJsonPath; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.update.UpdateRequest; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.common.utils.StringUtils; + +import com.jayway.jsonpath.JsonPath; + +import lombok.extern.log4j.Log4j2; + +@Log4j2 +public class AbstractIngestion implements Ingestable { + public static final String OUTPUT = "output"; + public static final String INPUT = "input"; + public static final String OUTPUT_FIELD_NAMES = "output_names"; + public static final String INPUT_FIELD_NAMES = "input_names"; + public static final String INGEST_FIELDS = "ingest_fields"; + public static final String ID_FIELD = "id_field"; + + private final Client client; + + public AbstractIngestion(Client client) { + this.client = client; + } + + protected ActionListener getBulkResponseListener( + AtomicInteger successfulBatches, + AtomicInteger failedBatches, + CompletableFuture future + ) { + return ActionListener.wrap(bulkResponse -> { + if (bulkResponse.hasFailures()) { + failedBatches.incrementAndGet(); + future.completeExceptionally(new RuntimeException(bulkResponse.buildFailureMessage())); // Mark the future as completed + // with an exception + return; + } + log.debug("Batch Ingestion successfully"); + successfulBatches.incrementAndGet(); + future.complete(null); // Mark the future as completed successfully + }, e -> { + log.error("Failed to Batch Ingestion", e); + failedBatches.incrementAndGet(); + future.completeExceptionally(e); // Mark the future as completed with an exception + }); + } + + protected double calculateSuccessRate(List successRates) { + return successRates + .stream() + .min(Double::compare) + .orElseThrow( + () -> new OpenSearchStatusException( + "Failed to batch ingest data as not success rate is returned", + RestStatus.INTERNAL_SERVER_ERROR + ) + ); + } + + /** + * Filters fields in the map where the value contains the specified source index as a prefix. + * + * @param mlBatchIngestionInput The MLBatchIngestionInput. + * @param index The source index to filter by. + * @return A new map with only the entries that match the specified source index. + */ + protected Map filterFieldMapping(MLBatchIngestionInput mlBatchIngestionInput, int index) { + Map fieldMap = mlBatchIngestionInput.getFieldMapping(); + int indexInFieldMap = index + 1; + String prefix = "source[" + indexInFieldMap + "]"; + + Map filteredFieldMap = fieldMap.entrySet().stream().filter(entry -> { + Object value = entry.getValue(); + if (value instanceof String) { + return ((String) value).contains(prefix); + } else if (value instanceof List) { + return ((List) value).stream().anyMatch(val -> val.contains(prefix)); + } + return false; + }).collect(Collectors.toMap(Map.Entry::getKey, entry -> { + Object value = entry.getValue(); + if (value instanceof String) { + return value; + } else if (value instanceof List) { + return ((List) value).stream().filter(val -> val.contains(prefix)).collect(Collectors.toList()); + } + return null; + })); + + if (filteredFieldMap.containsKey(OUTPUT)) { + filteredFieldMap.put(OUTPUT_FIELD_NAMES, fieldMap.get(OUTPUT_FIELD_NAMES)); + } + if (filteredFieldMap.containsKey(INPUT)) { + filteredFieldMap.put(INPUT_FIELD_NAMES, fieldMap.get(INPUT_FIELD_NAMES)); + } + return filteredFieldMap; + } + + /** + * Produce the source as a Map to be ingested in to OpenSearch. + * + * @param jsonStr The MLBatchIngestionInput. + * @param fieldMapping The field mapping that includes all the field name and Json Path for the data. + * @return A new map that contains all the fields and data for ingestion. + */ + protected Map processFieldMapping(String jsonStr, Map fieldMapping) { + String inputJsonPath = fieldMapping.containsKey(INPUT) ? getJsonPath((String) fieldMapping.get(INPUT)) : null; + List remoteModelInput = inputJsonPath != null ? (List) JsonPath.read(jsonStr, inputJsonPath) : null; + List inputFieldNames = inputJsonPath != null ? (List) fieldMapping.get(INPUT_FIELD_NAMES) : null; + + String outputJsonPath = fieldMapping.containsKey(OUTPUT) ? getJsonPath((String) fieldMapping.get(OUTPUT)) : null; + List remoteModelOutput = outputJsonPath != null ? (List) JsonPath.read(jsonStr, outputJsonPath) : null; + List outputFieldNames = outputJsonPath != null ? (List) fieldMapping.get(OUTPUT_FIELD_NAMES) : null; + + List ingestFieldsJsonPath = Optional + .ofNullable((List) fieldMapping.get(INGEST_FIELDS)) + .stream() + .flatMap(Collection::stream) + .map(StringUtils::getJsonPath) + .collect(Collectors.toList()); + + Map jsonMap = new HashMap<>(); + + populateJsonMap(jsonMap, inputFieldNames, remoteModelInput); + populateJsonMap(jsonMap, outputFieldNames, remoteModelOutput); + + for (String fieldPath : ingestFieldsJsonPath) { + jsonMap.put(obtainFieldNameFromJsonPath(fieldPath), JsonPath.read(jsonStr, fieldPath)); + } + + if (fieldMapping.containsKey(ID_FIELD)) { + List docIdJsonPath = Optional + .ofNullable((List) fieldMapping.get(ID_FIELD)) + .stream() + .flatMap(Collection::stream) + .map(StringUtils::getJsonPath) + .collect(Collectors.toList()); + if (docIdJsonPath.size() != 1) { + throw new IllegalArgumentException("The Id field must contains only 1 jsonPath for each source"); + } + jsonMap.put("_id", JsonPath.read(jsonStr, docIdJsonPath.get(0))); + } + return jsonMap; + } + + protected void batchIngest( + List sourceLines, + MLBatchIngestionInput mlBatchIngestionInput, + ActionListener bulkResponseListener, + int sourceIndex, + boolean isSoleSource + ) { + BulkRequest bulkRequest = new BulkRequest(); + sourceLines.stream().forEach(jsonStr -> { + Map filteredMapping = isSoleSource + ? mlBatchIngestionInput.getFieldMapping() + : filterFieldMapping(mlBatchIngestionInput, sourceIndex); + Map jsonMap = processFieldMapping(jsonStr, filteredMapping); + if (isSoleSource || sourceIndex == 0) { + IndexRequest indexRequest = new IndexRequest(mlBatchIngestionInput.getIndexName()); + if (jsonMap.containsKey("_id")) { + String id = (String) jsonMap.remove("_id"); + indexRequest.id(id); + } + indexRequest.source(jsonMap); + bulkRequest.add(indexRequest); + } else { + // bulk update docs as they were partially ingested + if (!jsonMap.containsKey("_id")) { + throw new IllegalArgumentException("The id filed must be provided to match documents for multiple sources"); + } + String id = (String) jsonMap.remove("_id"); + UpdateRequest updateRequest = new UpdateRequest(mlBatchIngestionInput.getIndexName(), id).doc(jsonMap).upsert(jsonMap); + bulkRequest.add(updateRequest); + } + }); + client.bulk(bulkRequest, bulkResponseListener); + } + + private void populateJsonMap(Map jsonMap, List fieldNames, List modelData) { + if (modelData != null) { + if (modelData.size() != fieldNames.size()) { + throw new IllegalArgumentException("The fieldMapping and source data do not match"); + } + + for (int index = 0; index < modelData.size(); index++) { + jsonMap.put(fieldNames.get(index), modelData.get(index)); + } + } + } +} diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/Ingestable.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/Ingestable.java new file mode 100644 index 0000000000..e020dcdd60 --- /dev/null +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/Ingestable.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.ingest; + +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; + +public interface Ingestable { + /** + * offline ingest data with given input. + * @param mlBatchIngestionInput batch ingestion input data + * @return successRate (0 - 100) + */ + default double ingest(MLBatchIngestionInput mlBatchIngestionInput) { + throw new IllegalStateException("Ingest is not implemented"); + } +} diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/OpenAIDataIngestion.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/OpenAIDataIngestion.java new file mode 100644 index 0000000000..8dc94894ef --- /dev/null +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/OpenAIDataIngestion.java @@ -0,0 +1,131 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.ingest; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.client.Client; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.engine.annotation.Ingester; + +import lombok.extern.log4j.Log4j2; + +@Log4j2 +@Ingester("openai") +public class OpenAIDataIngestion extends AbstractIngestion { + private static final String API_KEY = "openAI_key"; + private static final String API_URL = "https://api.openai.com/v1/files/"; + public static final String SOURCE = "source"; + + public OpenAIDataIngestion(Client client) { + super(client); + } + + @Override + public double ingest(MLBatchIngestionInput mlBatchIngestionInput) { + List sources = (List) mlBatchIngestionInput.getDataSources().get(SOURCE); + if (Objects.isNull(sources) || sources.isEmpty()) { + return 100; + } + + boolean isSoleSource = sources.size() == 1; + List successRates = Collections.synchronizedList(new ArrayList<>()); + for (int sourceIndex = 0; sourceIndex < sources.size(); sourceIndex++) { + successRates.add(ingestSingleSource(sources.get(sourceIndex), mlBatchIngestionInput, sourceIndex, isSoleSource)); + } + + return calculateSuccessRate(successRates); + } + + private double ingestSingleSource(String fileId, MLBatchIngestionInput mlBatchIngestionInput, int sourceIndex, boolean isSoleSource) { + double successRate = 0; + try { + String apiKey = mlBatchIngestionInput.getCredential().get(API_KEY); + URL url = new URL(API_URL + fileId + "/content"); + + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Authorization", "Bearer " + apiKey); + + try ( + InputStreamReader inputStreamReader = AccessController + .doPrivileged((PrivilegedExceptionAction) () -> new InputStreamReader(connection.getInputStream())); + BufferedReader reader = new BufferedReader(inputStreamReader) + ) { + List linesBuffer = new ArrayList<>(); + String line; + int lineCount = 0; + // Atomic counters for tracking success and failure + AtomicInteger successfulBatches = new AtomicInteger(0); + AtomicInteger failedBatches = new AtomicInteger(0); + // List of CompletableFutures to track batch ingestion operations + List> futures = new ArrayList<>(); + + while ((line = reader.readLine()) != null) { + linesBuffer.add(line); + lineCount++; + + // Process every 100 lines + if (lineCount % 100 == 0) { + // Create a CompletableFuture that will be completed by the bulkResponseListener + CompletableFuture future = new CompletableFuture<>(); + batchIngest( + linesBuffer, + mlBatchIngestionInput, + getBulkResponseListener(successfulBatches, failedBatches, future), + sourceIndex, + isSoleSource + ); + + futures.add(future); + linesBuffer.clear(); + } + } + // Process any remaining lines in the buffer + if (!linesBuffer.isEmpty()) { + CompletableFuture future = new CompletableFuture<>(); + batchIngest( + linesBuffer, + mlBatchIngestionInput, + getBulkResponseListener(successfulBatches, failedBatches, future), + sourceIndex, + isSoleSource + ); + futures.add(future); + } + + reader.close(); + // Combine all futures and wait for completion + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + // Wait for all tasks to complete + allFutures.join(); + int totalBatches = successfulBatches.get() + failedBatches.get(); + successRate = (totalBatches == 0) ? 100 : (double) successfulBatches.get() / totalBatches * 100; + } + } catch (PrivilegedActionException e) { + throw new RuntimeException("Failed to read from OpenAI file API: ", e); + } catch (Exception e) { + log.error(e.getMessage()); + throw new OpenSearchStatusException("Failed to batch ingest: " + e.getMessage(), RestStatus.INTERNAL_SERVER_ERROR); + } + + return successRate; + } +} diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/S3DataIngestion.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/S3DataIngestion.java new file mode 100644 index 0000000000..b6fb3e1226 --- /dev/null +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/S3DataIngestion.java @@ -0,0 +1,206 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.ingest; + +import static org.opensearch.ml.common.connector.AbstractConnector.ACCESS_KEY_FIELD; +import static org.opensearch.ml.common.connector.AbstractConnector.SECRET_KEY_FIELD; +import static org.opensearch.ml.common.connector.AbstractConnector.SESSION_TOKEN_FIELD; +import static org.opensearch.ml.common.connector.HttpConnector.REGION_FIELD; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.client.Client; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.engine.annotation.Ingester; + +import com.google.common.annotations.VisibleForTesting; + +import lombok.extern.log4j.Log4j2; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; + +@Log4j2 +@Ingester("s3") +public class S3DataIngestion extends AbstractIngestion { + public static final String SOURCE = "source"; + + public S3DataIngestion(Client client) { + super(client); + } + + @Override + public double ingest(MLBatchIngestionInput mlBatchIngestionInput) { + S3Client s3 = initS3Client(mlBatchIngestionInput); + + List s3Uris = (List) mlBatchIngestionInput.getDataSources().get(SOURCE); + if (Objects.isNull(s3Uris) || s3Uris.isEmpty()) { + return 100; + } + boolean isSoleSource = s3Uris.size() == 1; + List successRates = Collections.synchronizedList(new ArrayList<>()); + for (int sourceIndex = 0; sourceIndex < s3Uris.size(); sourceIndex++) { + successRates.add(ingestSingleSource(s3, s3Uris.get(sourceIndex), mlBatchIngestionInput, sourceIndex, isSoleSource)); + } + + return calculateSuccessRate(successRates); + } + + public double ingestSingleSource( + S3Client s3, + String s3Uri, + MLBatchIngestionInput mlBatchIngestionInput, + int sourceIndex, + boolean isSoleSource + ) { + String bucketName = getS3BucketName(s3Uri); + String keyName = getS3KeyName(s3Uri); + GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucketName).key(keyName).build(); + double successRate = 0; + + try ( + ResponseInputStream s3is = AccessController + .doPrivileged((PrivilegedExceptionAction>) () -> s3.getObject(getObjectRequest)); + BufferedReader reader = new BufferedReader(new InputStreamReader(s3is, StandardCharsets.UTF_8)) + ) { + List linesBuffer = new ArrayList<>(); + String line; + int lineCount = 0; + // Atomic counters for tracking success and failure + AtomicInteger successfulBatches = new AtomicInteger(0); + AtomicInteger failedBatches = new AtomicInteger(0); + // List of CompletableFutures to track batch ingestion operations + List> futures = new ArrayList<>(); + + while ((line = reader.readLine()) != null) { + linesBuffer.add(line); + lineCount++; + + // Process every 100 lines + if (lineCount % 100 == 0) { + // Create a CompletableFuture that will be completed by the bulkResponseListener + CompletableFuture future = new CompletableFuture<>(); + batchIngest( + linesBuffer, + mlBatchIngestionInput, + getBulkResponseListener(successfulBatches, failedBatches, future), + sourceIndex, + isSoleSource + ); + + futures.add(future); + linesBuffer.clear(); + } + } + // Process any remaining lines in the buffer + if (!linesBuffer.isEmpty()) { + CompletableFuture future = new CompletableFuture<>(); + batchIngest( + linesBuffer, + mlBatchIngestionInput, + getBulkResponseListener(successfulBatches, failedBatches, future), + sourceIndex, + isSoleSource + ); + futures.add(future); + } + + reader.close(); + + // Combine all futures and wait for completion + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + + // Wait for all tasks to complete + allFutures.join(); + + int totalBatches = successfulBatches.get() + failedBatches.get(); + successRate = (totalBatches == 0) ? 100 : (double) successfulBatches.get() / totalBatches * 100; + } catch (S3Exception e) { + log.error("Error reading from S3: " + e.awsErrorDetails().errorMessage()); + throw e; + } catch (PrivilegedActionException e) { + throw new RuntimeException("Failed to get S3 Object: ", e); + } catch (Exception e) { + log.error(e.getMessage()); + throw new OpenSearchStatusException("Failed to batch ingest: " + e.getMessage(), RestStatus.INTERNAL_SERVER_ERROR); + } finally { + s3.close(); + } + + return successRate; + } + + private String getS3BucketName(String s3Uri) { + // Remove the "s3://" prefix + String uriWithoutPrefix = s3Uri.substring(5); + // Find the first slash after the bucket name + int slashIndex = uriWithoutPrefix.indexOf('/'); + // If there is no slash, the entire remaining string is the bucket name + if (slashIndex == -1) { + return uriWithoutPrefix; + } + // Otherwise, the bucket name is the substring up to the first slash + return uriWithoutPrefix.substring(0, slashIndex); + } + + private String getS3KeyName(String s3Uri) { + String uriWithoutPrefix = s3Uri.substring(5); + // Find the first slash after the bucket name + int slashIndex = uriWithoutPrefix.indexOf('/'); + // If there is no slash, it means there is no key, return an empty string or handle as needed + if (slashIndex == -1) { + return ""; + } + // The key name is the substring after the first slash + return uriWithoutPrefix.substring(slashIndex + 1); + } + + @VisibleForTesting + public S3Client initS3Client(MLBatchIngestionInput mlBatchIngestionInput) { + String accessKey = mlBatchIngestionInput.getCredential().get(ACCESS_KEY_FIELD); + String secretKey = mlBatchIngestionInput.getCredential().get(SECRET_KEY_FIELD); + String sessionToken = mlBatchIngestionInput.getCredential().get(SESSION_TOKEN_FIELD); + String region = mlBatchIngestionInput.getCredential().get(REGION_FIELD); + + AwsCredentials credentials = sessionToken == null + ? AwsBasicCredentials.create(accessKey, secretKey) + : AwsSessionCredentials.create(accessKey, secretKey, sessionToken); + + try { + S3Client s3 = AccessController + .doPrivileged( + (PrivilegedExceptionAction) () -> S3Client + .builder() + .region(Region.of(region)) // Specify the region here + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .build() + ); + return s3; + } catch (PrivilegedActionException e) { + throw new RuntimeException("Can't load credentials", e); + } + } +} diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandlerTest.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandlerTest.java index f6c9b76071..44d3f104cb 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandlerTest.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandlerTest.java @@ -326,7 +326,6 @@ public void test_onComplete_error_http_status() { ArgumentCaptor captor = ArgumentCaptor.forClass(Exception.class); verify(actionListener, times(1)).onFailure(captor.capture()); assert captor.getValue() instanceof OpenSearchStatusException; - System.out.println(captor.getValue().getMessage()); assert captor.getValue().getMessage().contains("runtime error"); } @@ -350,7 +349,6 @@ public void test_onComplete_throttle_error_headers() { ArgumentCaptor captor = ArgumentCaptor.forClass(Exception.class); verify(actionListener, times(1)).onFailure(captor.capture()); assert captor.getValue() instanceof OpenSearchStatusException; - System.out.println(captor.getValue().getMessage()); assert captor.getValue().getMessage().contains(REMOTE_SERVICE_ERROR); } diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java new file mode 100644 index 0000000000..d2f66dacbc --- /dev/null +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java @@ -0,0 +1,262 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.ingest; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.ID_FIELD; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INGEST_FIELDS; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT_FIELD_NAMES; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT_FIELD_NAMES; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; + +public class AbstractIngestionTests { + @Mock + Client client; + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + S3DataIngestion s3DataIngestion = new S3DataIngestion(client); + + Map fieldMap; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + s3DataIngestion = new S3DataIngestion(client); + + fieldMap = new HashMap<>(); + fieldMap.put(INPUT, "source[1].$.content"); + fieldMap.put(OUTPUT, "source[1].$.SageMakerOutput"); + fieldMap.put(INPUT_FIELD_NAMES, Arrays.asList("chapter", "title")); + fieldMap.put(OUTPUT_FIELD_NAMES, Arrays.asList("chapter_embedding", "title_embedding")); + fieldMap.put(INGEST_FIELDS, Arrays.asList("source[1].$.id")); + } + + @Test + public void testBulkResponseListener_Success() { + // Arrange + AtomicInteger successfulBatches = new AtomicInteger(0); + AtomicInteger failedBatches = new AtomicInteger(0); + CompletableFuture future = new CompletableFuture<>(); + + // Mock BulkResponse + BulkResponse bulkResponse = mock(BulkResponse.class); + when(bulkResponse.hasFailures()).thenReturn(false); + + S3DataIngestion instance = new S3DataIngestion(client); + + // Act + ActionListener listener = instance.getBulkResponseListener(successfulBatches, failedBatches, future); + listener.onResponse(bulkResponse); + + // Assert + assertFalse(future.isCompletedExceptionally()); + assertEquals(1, successfulBatches.get()); + assertEquals(0, failedBatches.get()); + } + + @Test + public void testBulkResponseListener_Failure() { + // Arrange + AtomicInteger successfulBatches = new AtomicInteger(0); + AtomicInteger failedBatches = new AtomicInteger(0); + CompletableFuture future = new CompletableFuture<>(); + + // Mock BulkResponse + BulkResponse bulkResponse = mock(BulkResponse.class); + when(bulkResponse.hasFailures()).thenReturn(true); + when(bulkResponse.buildFailureMessage()).thenReturn("Failure message"); + + S3DataIngestion instance = new S3DataIngestion(client); + + // Act + ActionListener listener = instance.getBulkResponseListener(successfulBatches, failedBatches, future); + listener.onResponse(bulkResponse); + + // Assert + assertTrue(future.isCompletedExceptionally()); + assertEquals(0, successfulBatches.get()); + assertEquals(1, failedBatches.get()); + } + + @Test + public void testBulkResponseListener_Exception() { + // Arrange + AtomicInteger successfulBatches = new AtomicInteger(0); + AtomicInteger failedBatches = new AtomicInteger(0); + CompletableFuture future = new CompletableFuture<>(); + + // Create an exception + RuntimeException exception = new RuntimeException("Test exception"); + + S3DataIngestion instance = new S3DataIngestion(client); + + // Act + ActionListener listener = instance.getBulkResponseListener(successfulBatches, failedBatches, future); + listener.onFailure(exception); + + // Assert + assertTrue(future.isCompletedExceptionally()); + assertEquals(0, successfulBatches.get()); + assertEquals(1, failedBatches.get()); + assertThrows(Exception.class, () -> future.join()); // Ensure that future throws exception + } + + @Test + public void testCalculateSuccessRate_MultipleValues() { + // Arrange + List successRates = Arrays.asList(90.0, 85.5, 92.0, 88.0); + + // Act + double result = s3DataIngestion.calculateSuccessRate(successRates); + + // Assert + assertEquals(85.5, result, 0.0001); + } + + @Test + public void testCalculateSuccessRate_SingleValue() { + // Arrange + List successRates = Collections.singletonList(99.9); + + // Act + double result = s3DataIngestion.calculateSuccessRate(successRates); + + // Assert + assertEquals(99.9, result, 0.0001); + } + + @Test + public void testFilterFieldMapping_ValidInput_MatchingPrefix() { + // Arrange + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + Map result = s3DataIngestion.filterFieldMapping(mlBatchIngestionInput, 0); + + // Assert + assertEquals(5, result.size()); + assertEquals("source[1].$.content", result.get(INPUT)); + assertEquals("source[1].$.SageMakerOutput", result.get(OUTPUT)); + assertEquals(Arrays.asList("chapter", "title"), result.get(INPUT_FIELD_NAMES)); + assertEquals(Arrays.asList("chapter_embedding", "title_embedding"), result.get(OUTPUT_FIELD_NAMES)); + assertEquals(Arrays.asList("source[1].$.id"), result.get(INGEST_FIELDS)); + } + + @Test + public void testFilterFieldMapping_NoMatchingPrefix() { + // Arrange + Map fieldMap = new HashMap<>(); + fieldMap.put("field1", "source[3].$.response.body.data[*].embedding"); + fieldMap.put("field2", "source[4].$.body.input"); + + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + + // Act + Map result = s3DataIngestion.filterFieldMapping(mlBatchIngestionInput, 0); + + // Assert + assertTrue(result.isEmpty()); + } + + @Test + public void testProcessFieldMapping_ValidInput() { + String jsonStr = + "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}"; + // Arrange + + // Act + Map processedFieldMapping = s3DataIngestion.processFieldMapping(jsonStr, fieldMap); + + // Assert + assertEquals("this is chapter 1", processedFieldMapping.get("chapter")); + assertEquals("harry potter", processedFieldMapping.get("title")); + assertEquals(1, processedFieldMapping.get("id")); + } + + @Test + public void testProcessFieldMapping_NoIdFieldInput() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("The Id field must contains only 1 jsonPath for each source"); + + String jsonStr = + "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}"; + // Arrange + fieldMap.put(ID_FIELD, null); + + // Act + s3DataIngestion.processFieldMapping(jsonStr, fieldMap); + } + + @Test + public void testBatchIngestSuccess_SoleSource() { + doAnswer(invocation -> { + ActionListener bulkResponseListener = invocation.getArgument(1); + bulkResponseListener.onResponse(mock(BulkResponse.class)); + return null; + }).when(client).bulk(any(), any()); + + List sourceLines = Arrays + .asList( + "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}" + ); + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + ActionListener bulkResponseListener = mock(ActionListener.class); + s3DataIngestion.batchIngest(sourceLines, mlBatchIngestionInput, bulkResponseListener, 0, true); + + verify(client).bulk(isA(BulkRequest.class), isA(ActionListener.class)); + verify(bulkResponseListener).onResponse(isA(BulkResponse.class)); + } + + @Test + public void testBatchIngestSuccess_NoIdError() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("The id filed must be provided to match documents for multiple sources"); + + doAnswer(invocation -> { + ActionListener bulkResponseListener = invocation.getArgument(1); + bulkResponseListener.onResponse(mock(BulkResponse.class)); + return null; + }).when(client).bulk(any(), any()); + + List sourceLines = Arrays + .asList( + "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}" + ); + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + ActionListener bulkResponseListener = mock(ActionListener.class); + s3DataIngestion.batchIngest(sourceLines, mlBatchIngestionInput, bulkResponseListener, 1, false); + } +} diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java new file mode 100644 index 0000000000..4bf2ffd58f --- /dev/null +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.engine.ingest; + +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INGEST_FIELDS; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT_FIELD_NAMES; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT_FIELD_NAMES; +import static org.opensearch.ml.engine.ingest.S3DataIngestion.SOURCE; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.client.Client; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; + +import software.amazon.awssdk.services.s3.S3Client; + +public class S3DataIngestionTests { + + private MLBatchIngestionInput mlBatchIngestionInput; + private S3DataIngestion s3DataIngestion; + + @Mock + Client client; + + @Mock + S3Client s3Client; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + s3DataIngestion = new S3DataIngestion(client); + + Map fieldMap = new HashMap<>(); + fieldMap.put("input", "$.content"); + fieldMap.put("output", "$.SageMakerOutput"); + fieldMap.put(INPUT_FIELD_NAMES, Arrays.asList("chapter", "title")); + fieldMap.put(OUTPUT_FIELD_NAMES, Arrays.asList("chapter_embedding", "title_embedding")); + fieldMap.put(INGEST_FIELDS, Arrays.asList("$.id")); + + Map credential = Map + .of("region", "us-east-1", "access_key", "some accesskey", "secret_key", "some secret", "session_token", "some token"); + Map dataSource = new HashMap<>(); + dataSource.put("type", "s3"); + dataSource.put(SOURCE, Arrays.asList("s3://offlinebatch/output/sagemaker_djl_batch_input.json.out")); + + mlBatchIngestionInput = MLBatchIngestionInput + .builder() + .indexName("testIndex") + .fieldMapping(fieldMap) + .credential(credential) + .dataSources(dataSource) + .build(); + } +} diff --git a/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java b/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java new file mode 100644 index 0000000000..cf03d0f11a --- /dev/null +++ b/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java @@ -0,0 +1,178 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.action.batch; + +import static org.opensearch.ml.common.MLTask.ERROR_FIELD; +import static org.opensearch.ml.common.MLTask.STATE_FIELD; +import static org.opensearch.ml.common.MLTaskState.COMPLETED; +import static org.opensearch.ml.common.MLTaskState.FAILED; +import static org.opensearch.ml.plugin.MachineLearningPlugin.TRAIN_THREAD_POOL; +import static org.opensearch.ml.task.MLTaskManager.TASK_SEMAPHORE_TIMEOUT; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.MLTaskState; +import org.opensearch.ml.common.MLTaskType; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionAction; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionRequest; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionResponse; +import org.opensearch.ml.engine.MLEngineClassLoader; +import org.opensearch.ml.engine.ingest.Ingestable; +import org.opensearch.ml.task.MLTaskManager; +import org.opensearch.ml.utils.MLExceptionUtils; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import lombok.extern.log4j.Log4j2; + +@Log4j2 +public class TransportBatchIngestionAction extends HandledTransportAction { + private static final String S3_URI_REGEX = "^s3://([a-zA-Z0-9.-]+)(/.*)?$"; + private static final Pattern S3_URI_PATTERN = Pattern.compile(S3_URI_REGEX); + public static final String TYPE = "type"; + public static final String SOURCE = "source"; + TransportService transportService; + MLTaskManager mlTaskManager; + private final Client client; + private ThreadPool threadPool; + + @Inject + public TransportBatchIngestionAction( + TransportService transportService, + ActionFilters actionFilters, + Client client, + MLTaskManager mlTaskManager, + ThreadPool threadPool + ) { + super(MLBatchIngestionAction.NAME, transportService, actionFilters, MLBatchIngestionRequest::new); + this.transportService = transportService; + this.client = client; + this.mlTaskManager = mlTaskManager; + this.threadPool = threadPool; + } + + @Override + protected void doExecute(Task task, ActionRequest request, ActionListener listener) { + MLBatchIngestionRequest mlBatchIngestionRequest = MLBatchIngestionRequest.fromActionRequest(request); + MLBatchIngestionInput mlBatchIngestionInput = mlBatchIngestionRequest.getMlBatchIngestionInput(); + try { + validateBatchIngestInput(mlBatchIngestionInput); + MLTask mlTask = MLTask + .builder() + .async(true) + .taskType(MLTaskType.BATCH_INGEST) + .createTime(Instant.now()) + .lastUpdateTime(Instant.now()) + .state(MLTaskState.CREATED) + .build(); + + mlTaskManager.createMLTask(mlTask, ActionListener.wrap(response -> { + String taskId = response.getId(); + try { + mlTask.setTaskId(taskId); + mlTaskManager.add(mlTask); + listener.onResponse(new MLBatchIngestionResponse(taskId, MLTaskType.BATCH_INGEST, MLTaskState.CREATED.name())); + String ingestType = (String) mlBatchIngestionInput.getDataSources().get(TYPE); + Ingestable ingestable = MLEngineClassLoader.initInstance(ingestType.toLowerCase(), client, Client.class); + threadPool.executor(TRAIN_THREAD_POOL).execute(() -> { + double successRate = ingestable.ingest(mlBatchIngestionInput); + handleSuccessRate(successRate, taskId); + }); + } catch (Exception ex) { + log.error("Failed in batch ingestion", ex); + mlTaskManager + .updateMLTask( + taskId, + Map.of(STATE_FIELD, FAILED, ERROR_FIELD, MLExceptionUtils.getRootCauseMessage(ex)), + TASK_SEMAPHORE_TIMEOUT, + true + ); + listener.onFailure(ex); + } + }, exception -> { + log.error("Failed to create batch ingestion task", exception); + listener.onFailure(exception); + })); + } catch (IllegalArgumentException e) { + log.error(e.getMessage()); + listener + .onFailure( + new OpenSearchStatusException( + "IllegalArgumentException in the batch ingestion input: " + e.getMessage(), + RestStatus.BAD_REQUEST + ) + ); + } catch (Exception e) { + listener.onFailure(e); + } + } + + protected void handleSuccessRate(double successRate, String taskId) { + if (successRate == 100) { + mlTaskManager.updateMLTask(taskId, Map.of(STATE_FIELD, COMPLETED), 5000, true); + } else if (successRate > 0) { + mlTaskManager + .updateMLTask( + taskId, + Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "batch ingestion successful rate is " + successRate), + TASK_SEMAPHORE_TIMEOUT, + true + ); + } else { + mlTaskManager + .updateMLTask( + taskId, + Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "batch ingestion successful rate is 0"), + TASK_SEMAPHORE_TIMEOUT, + true + ); + } + } + + private void validateBatchIngestInput(MLBatchIngestionInput mlBatchIngestionInput) { + if (mlBatchIngestionInput == null + || mlBatchIngestionInput.getDataSources() == null + || mlBatchIngestionInput.getDataSources().isEmpty()) { + throw new IllegalArgumentException("The batch ingest input data source cannot be null"); + } + Map dataSources = mlBatchIngestionInput.getDataSources(); + if (dataSources.get(TYPE) == null || dataSources.get(SOURCE) == null) { + throw new IllegalArgumentException("The batch ingest input data source is missing data type or source"); + } + if (((String) dataSources.get(TYPE)).toLowerCase() == "s3") { + List s3Uris = (List) dataSources.get(SOURCE); + if (s3Uris == null || s3Uris.isEmpty()) { + throw new IllegalArgumentException("The batch ingest input s3Uris is empty"); + } + + // Partition the list into valid and invalid URIs + Map> partitionedUris = s3Uris + .stream() + .collect(Collectors.partitioningBy(uri -> S3_URI_PATTERN.matcher(uri).matches())); + + List invalidUris = partitionedUris.get(false); + + if (!invalidUris.isEmpty()) { + throw new IllegalArgumentException("The following batch ingest input S3 URIs are invalid: " + invalidUris); + } + } + } +} diff --git a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java index 7e9ab6d940..b4abd328c7 100644 --- a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java +++ b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java @@ -48,6 +48,7 @@ import org.opensearch.ml.action.agents.GetAgentTransportAction; import org.opensearch.ml.action.agents.TransportRegisterAgentAction; import org.opensearch.ml.action.agents.TransportSearchAgentAction; +import org.opensearch.ml.action.batch.TransportBatchIngestionAction; import org.opensearch.ml.action.config.GetConfigTransportAction; import org.opensearch.ml.action.connector.DeleteConnectorTransportAction; import org.opensearch.ml.action.connector.ExecuteConnectorTransportAction; @@ -120,6 +121,7 @@ import org.opensearch.ml.common.transport.agent.MLAgentGetAction; import org.opensearch.ml.common.transport.agent.MLRegisterAgentAction; import org.opensearch.ml.common.transport.agent.MLSearchAgentAction; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionAction; import org.opensearch.ml.common.transport.config.MLConfigGetAction; import org.opensearch.ml.common.transport.connector.MLConnectorDeleteAction; import org.opensearch.ml.common.transport.connector.MLConnectorGetAction; @@ -216,6 +218,7 @@ import org.opensearch.ml.processor.MLInferenceSearchRequestProcessor; import org.opensearch.ml.processor.MLInferenceSearchResponseProcessor; import org.opensearch.ml.repackage.com.google.common.collect.ImmutableList; +import org.opensearch.ml.rest.RestMLBatchIngestAction; import org.opensearch.ml.rest.RestMLCreateConnectorAction; import org.opensearch.ml.rest.RestMLCreateControllerAction; import org.opensearch.ml.rest.RestMLDeleteAgentAction; @@ -440,7 +443,8 @@ public MachineLearningPlugin(Settings settings) { new ActionHandler<>(GetTracesAction.INSTANCE, GetTracesTransportAction.class), new ActionHandler<>(MLListToolsAction.INSTANCE, ListToolsTransportAction.class), new ActionHandler<>(MLGetToolAction.INSTANCE, GetToolTransportAction.class), - new ActionHandler<>(MLConfigGetAction.INSTANCE, GetConfigTransportAction.class) + new ActionHandler<>(MLConfigGetAction.INSTANCE, GetConfigTransportAction.class), + new ActionHandler<>(MLBatchIngestionAction.INSTANCE, TransportBatchIngestionAction.class) ); } @@ -759,6 +763,7 @@ public List getRestHandlers( RestMLListToolsAction restMLListToolsAction = new RestMLListToolsAction(toolFactories); RestMLGetToolAction restMLGetToolAction = new RestMLGetToolAction(toolFactories); RestMLGetConfigAction restMLGetConfigAction = new RestMLGetConfigAction(); + RestMLBatchIngestAction restMLBatchIngestAction = new RestMLBatchIngestAction(); return ImmutableList .of( restMLStatsAction, @@ -811,7 +816,8 @@ public List getRestHandlers( restMLSearchAgentAction, restMLListToolsAction, restMLGetToolAction, - restMLGetConfigAction + restMLGetConfigAction, + restMLBatchIngestAction ); } diff --git a/plugin/src/main/java/org/opensearch/ml/rest/RestMLBatchIngestAction.java b/plugin/src/main/java/org/opensearch/ml/rest/RestMLBatchIngestAction.java new file mode 100644 index 0000000000..cdf2380985 --- /dev/null +++ b/plugin/src/main/java/org/opensearch/ml/rest/RestMLBatchIngestAction.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.rest; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.ml.plugin.MachineLearningPlugin.ML_BASE_URI; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionAction; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionRequest; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import lombok.extern.log4j.Log4j2; + +@Log4j2 +public class RestMLBatchIngestAction extends BaseRestHandler { + private static final String ML_BATCH_INGESTION_ACTION = "ml_batch_ingestion_action"; + + @Override + public String getName() { + return ML_BATCH_INGESTION_ACTION; + } + + @Override + public List routes() { + return ImmutableList.of(new Route(RestRequest.Method.POST, String.format(Locale.ROOT, "%s/_batch_ingestion", ML_BASE_URI))); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + MLBatchIngestionRequest mlBatchIngestTaskRequest = getRequest(request); + return channel -> client.execute(MLBatchIngestionAction.INSTANCE, mlBatchIngestTaskRequest, new RestToXContentListener<>(channel)); + } + + /** + * Creates a MLBatchIngestTaskRequest from a RestRequest + * + * @param request RestRequest + * @return MLBatchIngestTaskRequest + */ + @VisibleForTesting + MLBatchIngestionRequest getRequest(RestRequest request) throws IOException { + if (!request.hasContent()) { + throw new IOException("Batch Ingestion request has empty body"); + } + XContentParser parser = request.contentParser(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + MLBatchIngestionInput mlBatchIngestionInput = MLBatchIngestionInput.parse(parser); + return new MLBatchIngestionRequest(mlBatchIngestionInput); + } +} diff --git a/plugin/src/main/java/org/opensearch/ml/rest/RestMLPredictionAction.java b/plugin/src/main/java/org/opensearch/ml/rest/RestMLPredictionAction.java index 82c72e11a2..72b841eb7b 100644 --- a/plugin/src/main/java/org/opensearch/ml/rest/RestMLPredictionAction.java +++ b/plugin/src/main/java/org/opensearch/ml/rest/RestMLPredictionAction.java @@ -127,13 +127,11 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client @VisibleForTesting MLPredictionTaskRequest getRequest(String modelId, String algorithm, RestRequest request) throws IOException { ActionType actionType = ActionType.from(getActionTypeFromRestRequest(request)); - System.out.println("actionType is " + actionType); if (FunctionName.REMOTE.name().equals(algorithm) && !mlFeatureEnabledSetting.isRemoteInferenceEnabled()) { throw new IllegalStateException(REMOTE_INFERENCE_DISABLED_ERR_MSG); } else if (FunctionName.isDLModel(FunctionName.from(algorithm.toUpperCase())) && !mlFeatureEnabledSetting.isLocalModelEnabled()) { throw new IllegalStateException(LOCAL_MODEL_DISABLED_ERR_MSG); } else if (!ActionType.isValidActionInModelPrediction(actionType)) { - System.out.println(actionType.toString()); throw new IllegalArgumentException("Wrong action type in the rest request path!"); } diff --git a/plugin/src/main/java/org/opensearch/ml/utils/RestActionUtils.java b/plugin/src/main/java/org/opensearch/ml/utils/RestActionUtils.java index 5f5f567eb8..962200c5d0 100644 --- a/plugin/src/main/java/org/opensearch/ml/utils/RestActionUtils.java +++ b/plugin/src/main/java/org/opensearch/ml/utils/RestActionUtils.java @@ -319,7 +319,6 @@ public static void wrapListenerToHandleSearchIndexNotFound(Exception e, ActionLi */ public static String getActionTypeFromRestRequest(RestRequest request) { String path = request.path(); - System.out.println("path is " + path); String[] segments = path.split("/"); String methodName = segments[segments.length - 1]; methodName = methodName.startsWith("_") ? methodName.substring(1) : methodName; diff --git a/plugin/src/main/plugin-metadata/plugin-security.policy b/plugin/src/main/plugin-metadata/plugin-security.policy index 99cf437d24..1914fd5eb2 100644 --- a/plugin/src/main/plugin-metadata/plugin-security.policy +++ b/plugin/src/main/plugin-metadata/plugin-security.policy @@ -24,4 +24,10 @@ grant { // Circuit Breaker permission java.lang.RuntimePermission "getFileSystemAttributes"; + + // s3 client opens socket connections for to access repository + permission java.net.SocketPermission "*", "connect,resolve"; + + // aws credential file access + permission java.io.FilePermission "<>", "read"; }; diff --git a/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java new file mode 100644 index 0000000000..7b3766dadf --- /dev/null +++ b/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java @@ -0,0 +1,254 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.action.batch; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.ml.common.MLTask.ERROR_FIELD; +import static org.opensearch.ml.common.MLTask.STATE_FIELD; +import static org.opensearch.ml.common.MLTaskState.COMPLETED; +import static org.opensearch.ml.common.MLTaskState.FAILED; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INGEST_FIELDS; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT_FIELD_NAMES; +import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT_FIELD_NAMES; +import static org.opensearch.ml.engine.ingest.S3DataIngestion.SOURCE; +import static org.opensearch.ml.task.MLTaskManager.TASK_SEMAPHORE_TIMEOUT; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.index.Index; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionRequest; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionResponse; +import org.opensearch.ml.task.MLTaskManager; +import org.opensearch.tasks.Task; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class TransportBatchIngestionActionTests extends OpenSearchTestCase { + @Mock + private Client client; + @Mock + private TransportService transportService; + @Mock + private MLTaskManager mlTaskManager; + @Mock + private ActionFilters actionFilters; + @Mock + private MLBatchIngestionRequest mlBatchIngestionRequest; + @Mock + private Task task; + @Mock + ActionListener actionListener; + @Mock + ThreadPool threadPool; + + private TransportBatchIngestionAction batchAction; + private MLBatchIngestionInput batchInput; + + @Before + public void setup() { + MockitoAnnotations.openMocks(this); + batchAction = new TransportBatchIngestionAction(transportService, actionFilters, client, mlTaskManager, threadPool); + + Map fieldMap = new HashMap<>(); + fieldMap.put("input", "$.content"); + fieldMap.put("output", "$.SageMakerOutput"); + fieldMap.put(INPUT_FIELD_NAMES, Arrays.asList("chapter", "title")); + fieldMap.put(OUTPUT_FIELD_NAMES, Arrays.asList("chapter_embedding", "title_embedding")); + fieldMap.put(INGEST_FIELDS, Arrays.asList("$.id")); + + Map credential = Map + .of("region", "us-east-1", "access_key", "some accesskey", "secret_key", "some secret", "session_token", "some token"); + Map dataSource = new HashMap<>(); + dataSource.put("type", "s3"); + dataSource.put(SOURCE, Arrays.asList("s3://offlinebatch/output/sagemaker_djl_batch_input.json.out")); + + batchInput = MLBatchIngestionInput + .builder() + .indexName("testIndex") + .fieldMapping(fieldMap) + .credential(credential) + .dataSources(dataSource) + .build(); + when(mlBatchIngestionRequest.getMlBatchIngestionInput()).thenReturn(batchInput); + } + + public void test_doExecute_success() { + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + ShardId shardId = new ShardId(new Index("indexName", "uuid"), 1); + IndexResponse indexResponse = new IndexResponse(shardId, "taskId", 1, 1, 1, true); + listener.onResponse(indexResponse); + return null; + }).when(mlTaskManager).createMLTask(isA(MLTask.class), isA(ActionListener.class)); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + verify(actionListener).onResponse(any(MLBatchIngestionResponse.class)); + } + + public void test_doExecute_handleSuccessRate100() { + batchAction.handleSuccessRate(100, "taskid"); + verify(mlTaskManager).updateMLTask("taskid", Map.of(STATE_FIELD, COMPLETED), 5000, true); + } + + public void test_doExecute_handleSuccessRate50() { + batchAction.handleSuccessRate(50, "taskid"); + verify(mlTaskManager) + .updateMLTask( + "taskid", + Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "batch ingestion successful rate is 50.0"), + TASK_SEMAPHORE_TIMEOUT, + true + ); + } + + public void test_doExecute_handleSuccessRate0() { + batchAction.handleSuccessRate(0, "taskid"); + verify(mlTaskManager) + .updateMLTask( + "taskid", + Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "batch ingestion successful rate is 0"), + TASK_SEMAPHORE_TIMEOUT, + true + ); + } + + public void test_doExecute_noDataSource() { + MLBatchIngestionInput batchInput = MLBatchIngestionInput + .builder() + .indexName("testIndex") + .fieldMapping(new HashMap<>()) + .credential(new HashMap<>()) + .dataSources(new HashMap<>()) + .build(); + when(mlBatchIngestionRequest.getMlBatchIngestionInput()).thenReturn(batchInput); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals( + "IllegalArgumentException in the batch ingestion input: The batch ingest input data source cannot be null", + argumentCaptor.getValue().getMessage() + ); + } + + public void test_doExecute_noTypeInDataSource() { + MLBatchIngestionInput batchInput = MLBatchIngestionInput + .builder() + .indexName("testIndex") + .fieldMapping(new HashMap<>()) + .credential(new HashMap<>()) + .dataSources(Map.of("source", "some url")) + .build(); + when(mlBatchIngestionRequest.getMlBatchIngestionInput()).thenReturn(batchInput); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals( + "IllegalArgumentException in the batch ingestion input: The batch ingest input data source is missing data type or source", + argumentCaptor.getValue().getMessage() + ); + } + + public void test_doExecute_invalidS3DataSource() { + Map dataSource = new HashMap<>(); + dataSource.put("type", "s3"); + dataSource.put(SOURCE, Arrays.asList("s3://offlinebatch/output/sagemaker_djl_batch_input.json.out", "invalid s3")); + + MLBatchIngestionInput batchInput = MLBatchIngestionInput + .builder() + .indexName("testIndex") + .fieldMapping(new HashMap<>()) + .credential(new HashMap<>()) + .dataSources(dataSource) + .build(); + when(mlBatchIngestionRequest.getMlBatchIngestionInput()).thenReturn(batchInput); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals( + "IllegalArgumentException in the batch ingestion input: The following batch ingest input S3 URIs are invalid: [invalid s3]", + argumentCaptor.getValue().getMessage() + ); + } + + public void test_doExecute_emptyS3DataSource() { + Map dataSource = new HashMap<>(); + dataSource.put("type", "s3"); + dataSource.put(SOURCE, new ArrayList<>()); + + MLBatchIngestionInput batchInput = MLBatchIngestionInput + .builder() + .indexName("testIndex") + .fieldMapping(new HashMap<>()) + .credential(new HashMap<>()) + .dataSources(dataSource) + .build(); + when(mlBatchIngestionRequest.getMlBatchIngestionInput()).thenReturn(batchInput); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals( + "IllegalArgumentException in the batch ingestion input: The batch ingest input s3Uris is empty", + argumentCaptor.getValue().getMessage() + ); + } + + public void test_doExecute_mlTaskCreateException() { + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onFailure(new RuntimeException("Failed to create ML Task")); + return null; + }).when(mlTaskManager).createMLTask(isA(MLTask.class), isA(ActionListener.class)); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(RuntimeException.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("Failed to create ML Task", argumentCaptor.getValue().getMessage()); + } + + public void test_doExecute_batchIngestionFailed() { + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + ShardId shardId = new ShardId(new Index("indexName", "uuid"), 1); + IndexResponse indexResponse = new IndexResponse(shardId, "taskId", 1, 1, 1, true); + listener.onResponse(indexResponse); + return null; + }).when(mlTaskManager).createMLTask(isA(MLTask.class), isA(ActionListener.class)); + + doThrow(new OpenSearchStatusException("some error", RestStatus.INTERNAL_SERVER_ERROR)).when(mlTaskManager).add(isA(MLTask.class)); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(OpenSearchStatusException.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("some error", argumentCaptor.getValue().getMessage()); + verify(mlTaskManager).updateMLTask("taskId", Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "some error"), TASK_SEMAPHORE_TIMEOUT, true); + } +} diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java new file mode 100644 index 0000000000..9b6a00c8d7 --- /dev/null +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java @@ -0,0 +1,126 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.rest; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.opensearch.ml.utils.TestHelper.getBatchIngestionRestRequest; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionAction; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionInput; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionRequest; +import org.opensearch.ml.common.transport.batch.MLBatchIngestionResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RestMLBatchIngestionActionTests extends OpenSearchTestCase { + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private RestMLBatchIngestAction restMLBatchIngestAction; + private ThreadPool threadPool; + NodeClient client; + + @Mock + RestChannel channel; + + @Before + public void setup() { + MockitoAnnotations.openMocks(this); + restMLBatchIngestAction = new RestMLBatchIngestAction(); + threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool"); + client = spy(new NodeClient(Settings.EMPTY, threadPool)); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + return null; + }).when(client).execute(eq(MLBatchIngestionAction.INSTANCE), any(), any()); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + client.close(); + } + + public void testConstructor() { + RestMLBatchIngestAction mlBatchIngestAction = new RestMLBatchIngestAction(); + assertNotNull(mlBatchIngestAction); + } + + public void testGetName() { + String actionName = restMLBatchIngestAction.getName(); + assertFalse(Strings.isNullOrEmpty(actionName)); + assertEquals("ml_batch_ingestion_action", actionName); + } + + public void testRoutes() { + List routes = restMLBatchIngestAction.routes(); + assertNotNull(routes); + assertFalse(routes.isEmpty()); + RestHandler.Route route = routes.get(0); + assertEquals(RestRequest.Method.POST, route.getMethod()); + assertEquals("/_plugins/_ml/_batch_ingestion", route.getPath()); + } + + public void testGetRequest() throws IOException { + RestRequest request = getBatchIngestionRestRequest(); + MLBatchIngestionRequest mlBatchIngestionRequest = restMLBatchIngestAction.getRequest(request); + + MLBatchIngestionInput mlBatchIngestionInput = mlBatchIngestionRequest.getMlBatchIngestionInput(); + assertEquals("test batch index", mlBatchIngestionInput.getIndexName()); + assertEquals("$.content", mlBatchIngestionInput.getFieldMapping().get("input")); + assertNotNull(mlBatchIngestionInput.getDataSources().get("source")); + assertNotNull(mlBatchIngestionInput.getCredential()); + } + + public void testPrepareRequest() throws Exception { + RestRequest request = getBatchIngestionRestRequest(); + restMLBatchIngestAction.handleRequest(request, channel, client); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(MLBatchIngestionRequest.class); + verify(client, times(1)).execute(eq(MLBatchIngestionAction.INSTANCE), argumentCaptor.capture(), any()); + MLBatchIngestionInput mlBatchIngestionInput = argumentCaptor.getValue().getMlBatchIngestionInput(); + assertEquals("test batch index", mlBatchIngestionInput.getIndexName()); + assertEquals("$.content", mlBatchIngestionInput.getFieldMapping().get("input")); + assertNotNull(mlBatchIngestionInput.getDataSources().get("source")); + assertNotNull(mlBatchIngestionInput.getCredential()); + } + + public void testPrepareRequest_EmptyContent() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Batch Ingestion request has empty body"); + Map params = new HashMap<>(); + RestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(params).build(); + + restMLBatchIngestAction.handleRequest(request, channel, client); + } +} diff --git a/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java b/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java index ca5046fa0b..26f0afa8fb 100644 --- a/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java +++ b/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java @@ -523,4 +523,27 @@ public static void copyFile(String sourceFile, String destFile) throws IOExcepti FileUtils.copyFile(new File(sourceFile), new File(destFile)); } + public static RestRequest getBatchIngestionRestRequest() { + final String requestContent = "{\n" + + " \"index_name\": \"test batch index\",\n" + + " \"field_map\": {\n" + + " \"input\": \"$.content\",\n" + + " \"output\": \"$.SageMakerOutput\",\n" + + " \"input_names\": [\"chapter\", \"title\"],\n" + + " \"output_names\": [\"chapter_embedding\", \"title_embedding\"],\n" + + " \"ingest_fields\": [\"$.id\"]\n" + + " },\n" + + " \"credential\": {\n" + + " \"region\": \"xxxxxxxx\"\n" + + " },\n" + + " \"data_source\": {\n" + + " \"type\": \"s3\",\n" + + " \"source\": [\"s3://offlinebatch/output/sagemaker_djl_batch_input.json.out\"]\n" + + " }\n" + + "}"; + RestRequest request = new FakeRestRequest.Builder(getXContentRegistry()) + .withContent(new BytesArray(requestContent), XContentType.JSON) + .build(); + return request; + } } From 8da7bd235d27f4eef59583a7ac7f8b8aee6d856d Mon Sep 17 00:00:00 2001 From: Bhavana Ramaram Date: Thu, 5 Sep 2024 12:27:47 -0500 Subject: [PATCH 10/23] support get batch transform job status in get task API (#2825) * support get batch transform job status in get task API Signed-off-by: Bhavana Ramaram * add cancel batch prediction job API for offline inference Signed-off-by: Bhavana Ramaram * add unit tests and address comments Signed-off-by: Bhavana Ramaram * stash context for get model Signed-off-by: Bhavana Ramaram * apply spotlessJava and exclude from test coverage Signed-off-by: Bhavana Ramaram --------- Signed-off-by: Bhavana Ramaram --- .../org/opensearch/ml/common/CommonValue.java | 6 +- .../java/org/opensearch/ml/common/MLTask.java | 33 +- .../org/opensearch/ml/common/MLTaskType.java | 1 + .../ml/common/connector/ConnectorAction.java | 4 +- .../ml/common/output/model/ModelTensors.java | 5 + .../task/MLCancelBatchJobAction.java | 17 + .../task/MLCancelBatchJobRequest.java | 70 ++++ .../task/MLCancelBatchJobResponse.java | 64 ++++ .../task/MLCancelBatchJobRequestTest.java | 75 +++++ .../task/MLCancelBatchJobResponseTest.java | 36 ++ .../algorithms/remote/ConnectorUtils.java | 5 +- .../remote/MLSdkAsyncHttpResponseHandler.java | 13 +- plugin/build.gradle | 4 +- .../tasks/CancelBatchJobTransportAction.java | 241 ++++++++++++++ .../action/tasks/GetTaskTransportAction.java | 198 ++++++++++- .../ml/plugin/MachineLearningPlugin.java | 10 +- .../ml/rest/RestMLCancelBatchJobAction.java | 68 ++++ .../ml/task/MLPredictTaskRunner.java | 69 +++- .../CancelBatchJobTransportActionTests.java | 309 ++++++++++++++++++ .../tasks/GetTaskTransportActionTests.java | 228 ++++++++++++- .../rest/RestMLCancelBatchJobActionTests.java | 105 ++++++ .../ml/task/MLPredictTaskRunnerTests.java | 107 ++++++ 22 files changed, 1650 insertions(+), 18 deletions(-) create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequest.java create mode 100644 common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponse.java create mode 100644 common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequestTest.java create mode 100644 common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponseTest.java create mode 100644 plugin/src/main/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportAction.java create mode 100644 plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java create mode 100644 plugin/src/test/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportActionTests.java create mode 100644 plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java diff --git a/common/src/main/java/org/opensearch/ml/common/CommonValue.java b/common/src/main/java/org/opensearch/ml/common/CommonValue.java index 06f917ee9d..edf76dc35e 100644 --- a/common/src/main/java/org/opensearch/ml/common/CommonValue.java +++ b/common/src/main/java/org/opensearch/ml/common/CommonValue.java @@ -66,7 +66,7 @@ public class CommonValue { public static final Integer ML_MODEL_GROUP_INDEX_SCHEMA_VERSION = 2; public static final Integer ML_MODEL_INDEX_SCHEMA_VERSION = 11; public static final String ML_CONNECTOR_INDEX = ".plugins-ml-connector"; - public static final Integer ML_TASK_INDEX_SCHEMA_VERSION = 2; + public static final Integer ML_TASK_INDEX_SCHEMA_VERSION = 3; public static final Integer ML_CONNECTOR_SCHEMA_VERSION = 3; public static final String ML_CONFIG_INDEX = ".plugins-ml-config"; public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 3; @@ -393,6 +393,10 @@ public class CommonValue { + "\" : {\"type\" : \"boolean\"}, \n" + USER_FIELD_MAPPING + " }\n" + + "}" + + MLTask.REMOTE_JOB_FIELD + + "\" : {\"type\": \"flat_object\"}\n" + + " }\n" + "}"; public static final String ML_CONNECTOR_INDEX_MAPPING = "{\n" diff --git a/common/src/main/java/org/opensearch/ml/common/MLTask.java b/common/src/main/java/org/opensearch/ml/common/MLTask.java index a810fa5159..1165628711 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLTask.java +++ b/common/src/main/java/org/opensearch/ml/common/MLTask.java @@ -13,7 +13,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import org.opensearch.Version; import org.opensearch.commons.authuser.User; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -45,6 +47,8 @@ public class MLTask implements ToXContentObject, Writeable { public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; public static final String ERROR_FIELD = "error"; public static final String IS_ASYNC_TASK_FIELD = "is_async"; + public static final String REMOTE_JOB_FIELD = "remote_job"; + public static final Version MINIMAL_SUPPORTED_VERSION_FOR_BATCH_PREDICTION_JOB = CommonValue.VERSION_2_17_0; @Setter private String taskId; @@ -66,6 +70,8 @@ public class MLTask implements ToXContentObject, Writeable { private String error; private User user; // TODO: support document level access control later private boolean async; + @Setter + private Map remoteJob; @Builder(toBuilder = true) public MLTask( @@ -82,7 +88,8 @@ public MLTask( Instant lastUpdateTime, String error, User user, - boolean async + boolean async, + Map remoteJob ) { this.taskId = taskId; this.modelId = modelId; @@ -98,9 +105,11 @@ public MLTask( this.error = error; this.user = user; this.async = async; + this.remoteJob = remoteJob; } public MLTask(StreamInput input) throws IOException { + Version streamInputVersion = input.getVersion(); this.taskId = input.readOptionalString(); this.modelId = input.readOptionalString(); this.taskType = input.readEnum(MLTaskType.class); @@ -123,10 +132,16 @@ public MLTask(StreamInput input) throws IOException { this.user = null; } this.async = input.readBoolean(); + if (streamInputVersion.onOrAfter(MLTask.MINIMAL_SUPPORTED_VERSION_FOR_BATCH_PREDICTION_JOB)) { + if (input.readBoolean()) { + this.remoteJob = input.readMap(s -> s.readString(), s -> s.readGenericValue()); + } + } } @Override public void writeTo(StreamOutput out) throws IOException { + Version streamOutputVersion = out.getVersion(); out.writeOptionalString(taskId); out.writeOptionalString(modelId); out.writeEnum(taskType); @@ -150,6 +165,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } out.writeBoolean(async); + if (streamOutputVersion.onOrAfter(MLTask.MINIMAL_SUPPORTED_VERSION_FOR_BATCH_PREDICTION_JOB)) { + if (remoteJob != null) { + out.writeBoolean(true); + out.writeMap(remoteJob, StreamOutput::writeString, StreamOutput::writeGenericValue); + } else { + out.writeBoolean(false); + } + } } @Override @@ -195,6 +218,9 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params builder.field(USER, user); } builder.field(IS_ASYNC_TASK_FIELD, async); + if (remoteJob != null) { + builder.field(REMOTE_JOB_FIELD, remoteJob); + } return builder.endObject(); } @@ -218,6 +244,7 @@ public static MLTask parse(XContentParser parser) throws IOException { String error = null; User user = null; boolean async = false; + Map remoteJob = null; ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); while (parser.nextToken() != XContentParser.Token.END_OBJECT) { @@ -275,6 +302,9 @@ public static MLTask parse(XContentParser parser) throws IOException { case IS_ASYNC_TASK_FIELD: async = parser.booleanValue(); break; + case REMOTE_JOB_FIELD: + remoteJob = parser.map(); + break; default: parser.skipChildren(); break; @@ -296,6 +326,7 @@ public static MLTask parse(XContentParser parser) throws IOException { .error(error) .user(user) .async(async) + .remoteJob(remoteJob) .build(); } } diff --git a/common/src/main/java/org/opensearch/ml/common/MLTaskType.java b/common/src/main/java/org/opensearch/ml/common/MLTaskType.java index e17b36a4dd..179bf152cd 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLTaskType.java +++ b/common/src/main/java/org/opensearch/ml/common/MLTaskType.java @@ -8,6 +8,7 @@ public enum MLTaskType { TRAINING, PREDICTION, + BATCH_PREDICTION, TRAINING_AND_PREDICTION, EXECUTION, @Deprecated diff --git a/common/src/main/java/org/opensearch/ml/common/connector/ConnectorAction.java b/common/src/main/java/org/opensearch/ml/common/connector/ConnectorAction.java index 93fb5cca57..b62337d49f 100644 --- a/common/src/main/java/org/opensearch/ml/common/connector/ConnectorAction.java +++ b/common/src/main/java/org/opensearch/ml/common/connector/ConnectorAction.java @@ -188,7 +188,9 @@ public static ConnectorAction parse(XContentParser parser) throws IOException { public enum ActionType { PREDICT, EXECUTE, - BATCH_PREDICT; + BATCH_PREDICT, + CANCEL_BATCH_PREDICT, + BATCH_PREDICT_STATUS; public static ActionType from(String value) { try { diff --git a/common/src/main/java/org/opensearch/ml/common/output/model/ModelTensors.java b/common/src/main/java/org/opensearch/ml/common/output/model/ModelTensors.java index c3413a179f..5622057951 100644 --- a/common/src/main/java/org/opensearch/ml/common/output/model/ModelTensors.java +++ b/common/src/main/java/org/opensearch/ml/common/output/model/ModelTensors.java @@ -36,6 +36,11 @@ public ModelTensors(List mlModelTensors) { this.mlModelTensors = mlModelTensors; } + @Builder + public ModelTensors(Integer statusCode) { + this.statusCode = statusCode; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); diff --git a/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java new file mode 100644 index 0000000000..6ea26c9eb3 --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.task; + +import org.opensearch.action.ActionType; + +public class MLCancelBatchJobAction extends ActionType { + public static final MLCancelBatchJobAction INSTANCE = new MLCancelBatchJobAction(); + public static final String NAME = "cluster:admin/opensearch/ml/tasks/cancel_batch_job"; + + private MLCancelBatchJobAction() { + super(NAME, MLCancelBatchJobResponse::new); + } +} diff --git a/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequest.java b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequest.java new file mode 100644 index 0000000000..976ab69bef --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequest.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.task; + +import static org.opensearch.action.ValidateActions.addValidationError; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import lombok.Builder; +import lombok.Getter; + +public class MLCancelBatchJobRequest extends ActionRequest { + @Getter + String taskId; + + @Builder + public MLCancelBatchJobRequest(String taskId) { + this.taskId = taskId; + } + + public MLCancelBatchJobRequest(StreamInput in) throws IOException { + super(in); + this.taskId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(this.taskId); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException exception = null; + + if (this.taskId == null) { + exception = addValidationError("ML task id can't be null", exception); + } + + return exception; + } + + public static MLCancelBatchJobRequest fromActionRequest(ActionRequest actionRequest) { + if (actionRequest instanceof MLCancelBatchJobRequest) { + return (MLCancelBatchJobRequest) actionRequest; + } + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStreamStreamOutput osso = new OutputStreamStreamOutput(baos)) { + actionRequest.writeTo(osso); + try (StreamInput input = new InputStreamStreamInput(new ByteArrayInputStream(baos.toByteArray()))) { + return new MLCancelBatchJobRequest(input); + } + } catch (IOException e) { + throw new UncheckedIOException("failed to parse ActionRequest into MLCancelBatchJobRequest", e); + } + } +} diff --git a/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponse.java b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponse.java new file mode 100644 index 0000000000..6e97eb9647 --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponse.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.transport.task; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.InputStreamStreamInput; +import org.opensearch.core.common.io.stream.OutputStreamStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import lombok.Builder; +import lombok.Getter; + +@Getter +public class MLCancelBatchJobResponse extends ActionResponse implements ToXContentObject { + + RestStatus status; + + @Builder + public MLCancelBatchJobResponse(RestStatus status) { + this.status = status; + } + + public MLCancelBatchJobResponse(StreamInput in) throws IOException { + super(in); + status = in.readEnum(RestStatus.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(status); + } + + public static MLCancelBatchJobResponse fromActionResponse(ActionResponse actionResponse) { + if (actionResponse instanceof MLCancelBatchJobResponse) { + return (MLCancelBatchJobResponse) actionResponse; + } + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStreamStreamOutput osso = new OutputStreamStreamOutput(baos)) { + actionResponse.writeTo(osso); + try (StreamInput input = new InputStreamStreamInput(new ByteArrayInputStream(baos.toByteArray()))) { + return new MLCancelBatchJobResponse(input); + } + } catch (IOException e) { + throw new UncheckedIOException("failed to parse ActionResponse into MLTaskGetResponse", e); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + return xContentBuilder.startObject().field("status", status).endObject(); + } +} diff --git a/common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequestTest.java b/common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequestTest.java new file mode 100644 index 0000000000..e6e1f3838c --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobRequestTest.java @@ -0,0 +1,75 @@ +package org.opensearch.ml.common.transport.task; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + +import java.io.IOException; +import java.io.UncheckedIOException; + +import org.junit.Before; +import org.junit.Test; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamOutput; + +public class MLCancelBatchJobRequestTest { + private String taskId; + + @Before + public void setUp() { + taskId = "test_id"; + } + + @Test + public void writeTo_Success() throws IOException { + MLCancelBatchJobRequest mlCancelBatchJobRequest = MLCancelBatchJobRequest.builder().taskId(taskId).build(); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + mlCancelBatchJobRequest.writeTo(bytesStreamOutput); + MLCancelBatchJobRequest parsedTask = new MLCancelBatchJobRequest(bytesStreamOutput.bytes().streamInput()); + assertEquals(parsedTask.getTaskId(), taskId); + } + + @Test + public void validate_Exception_NullTaskId() { + MLCancelBatchJobRequest mlCancelBatchJobRequest = MLCancelBatchJobRequest.builder().build(); + + ActionRequestValidationException exception = mlCancelBatchJobRequest.validate(); + assertEquals("Validation Failed: 1: ML task id can't be null;", exception.getMessage()); + } + + @Test + public void fromActionRequest_Success() { + MLCancelBatchJobRequest mlCancelBatchJobRequest = MLCancelBatchJobRequest.builder().taskId(taskId).build(); + ActionRequest actionRequest = new ActionRequest() { + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + mlCancelBatchJobRequest.writeTo(out); + } + }; + MLCancelBatchJobRequest result = MLCancelBatchJobRequest.fromActionRequest(actionRequest); + assertNotSame(result, mlCancelBatchJobRequest); + assertEquals(result.getTaskId(), mlCancelBatchJobRequest.getTaskId()); + } + + @Test(expected = UncheckedIOException.class) + public void fromActionRequest_IOException() { + ActionRequest actionRequest = new ActionRequest() { + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new IOException("test"); + } + }; + MLCancelBatchJobRequest.fromActionRequest(actionRequest); + } +} diff --git a/common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponseTest.java b/common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponseTest.java new file mode 100644 index 0000000000..4cb3837df4 --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobResponseTest.java @@ -0,0 +1,36 @@ +package org.opensearch.ml.common.transport.task; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; + +import org.junit.Test; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; + +public class MLCancelBatchJobResponseTest { + + @Test + public void writeTo_Success() throws IOException { + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + MLCancelBatchJobResponse response = MLCancelBatchJobResponse.builder().status(RestStatus.OK).build(); + response.writeTo(bytesStreamOutput); + MLCancelBatchJobResponse parsedResponse = new MLCancelBatchJobResponse(bytesStreamOutput.bytes().streamInput()); + assertEquals(response.getStatus(), parsedResponse.getStatus()); + } + + @Test + public void toXContentTest() throws IOException { + MLCancelBatchJobResponse mlCancelBatchJobResponse1 = MLCancelBatchJobResponse.builder().status(RestStatus.OK).build(); + XContentBuilder builder = MediaTypeRegistry.contentBuilder(XContentType.JSON); + mlCancelBatchJobResponse1.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertNotNull(builder); + String jsonStr = builder.toString(); + assertEquals("{\"status\":\"OK\"}", jsonStr); + } +} diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/ConnectorUtils.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/ConnectorUtils.java index ef4f25c79a..ccceff3d68 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/ConnectorUtils.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/ConnectorUtils.java @@ -6,6 +6,7 @@ package org.opensearch.ml.engine.algorithms.remote; import static org.apache.commons.text.StringEscapeUtils.escapeJson; +import static org.opensearch.ml.common.connector.ConnectorAction.ActionType.CANCEL_BATCH_PREDICT; import static org.opensearch.ml.common.connector.HttpConnector.RESPONSE_FILTER_FIELD; import static org.opensearch.ml.common.connector.MLPreProcessFunction.CONVERT_INPUT_TO_JSON_STRING; import static org.opensearch.ml.common.connector.MLPreProcessFunction.PROCESS_REMOTE_INFERENCE_INPUT; @@ -286,7 +287,9 @@ public static SdkHttpFullRequest buildSdkRequest( } else { requestBody = RequestBody.empty(); } - if (SdkHttpMethod.POST == method && 0 == requestBody.optionalContentLength().get()) { + if (SdkHttpMethod.POST == method + && 0 == requestBody.optionalContentLength().get() + && !action.equals(CANCEL_BATCH_PREDICT.toString())) { log.error("Content length is 0. Aborting request to remote model"); throw new IllegalArgumentException("Content length is 0. Aborting request to remote model"); } diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandler.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandler.java index 6ea03058f0..fdb686dacc 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandler.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/MLSdkAsyncHttpResponseHandler.java @@ -8,6 +8,7 @@ package org.opensearch.ml.engine.algorithms.remote; import static org.opensearch.ml.common.CommonValue.REMOTE_SERVICE_ERROR; +import static org.opensearch.ml.common.connector.ConnectorAction.ActionType.CANCEL_BATCH_PREDICT; import static org.opensearch.ml.engine.algorithms.remote.ConnectorUtils.processOutput; import java.nio.ByteBuffer; @@ -169,13 +170,14 @@ public void onComplete() { } private void response() { + String body = responseBody.toString(); + if (exceptionHolder.get() != null) { actionListener.onFailure(exceptionHolder.get()); return; } - String body = responseBody.toString(); - if (Strings.isBlank(body)) { + if (Strings.isBlank(body) && !action.equals(CANCEL_BATCH_PREDICT.toString())) { log.error("Remote model response body is empty!"); actionListener.onFailure(new OpenSearchStatusException("No response from model", RestStatus.BAD_REQUEST)); return; @@ -187,6 +189,13 @@ private void response() { return; } + if (action.equals(CANCEL_BATCH_PREDICT.toString())) { + ModelTensors tensors = ModelTensors.builder().statusCode(statusCode).build(); + tensors.setStatusCode(statusCode); + actionListener.onResponse(new Tuple<>(executionContext.getSequence(), tensors)); + return; + } + try { ModelTensors tensors = processOutput(action, body, connector, scriptService, parameters, mlGuard); tensors.setStatusCode(statusCode); diff --git a/plugin/build.gradle b/plugin/build.gradle index 1088c544fc..d9f97cf4cf 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -312,7 +312,9 @@ List jacocoExclusions = [ 'org.opensearch.ml.helper.ModelAccessControlHelper', 'org.opensearch.ml.action.models.DeleteModelTransportAction.2', 'org.opensearch.ml.model.MLModelCacheHelper', - 'org.opensearch.ml.model.MLModelCacheHelper.1' + 'org.opensearch.ml.model.MLModelCacheHelper.1', + 'org.opensearch.ml.action.tasks.CancelBatchJobTransportAction' + ] jacocoTestCoverageVerification { diff --git a/plugin/src/main/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportAction.java new file mode 100644 index 0000000000..6a7fd617ae --- /dev/null +++ b/plugin/src/main/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportAction.java @@ -0,0 +1,241 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.action.tasks; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.ml.common.CommonValue.ML_CONNECTOR_INDEX; +import static org.opensearch.ml.common.CommonValue.ML_TASK_INDEX; +import static org.opensearch.ml.common.connector.ConnectorAction.ActionType.CANCEL_BATCH_PREDICT; +import static org.opensearch.ml.utils.MLNodeUtils.createXContentParserFromRegistry; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.apache.hc.core5.http.HttpStatus; +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.ActionRequest; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.ml.common.FunctionName; +import org.opensearch.ml.common.MLModel; +import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.MLTaskType; +import org.opensearch.ml.common.connector.Connector; +import org.opensearch.ml.common.connector.ConnectorAction.ActionType; +import org.opensearch.ml.common.dataset.remote.RemoteInferenceInputDataSet; +import org.opensearch.ml.common.exception.MLResourceNotFoundException; +import org.opensearch.ml.common.input.MLInput; +import org.opensearch.ml.common.output.model.ModelTensorOutput; +import org.opensearch.ml.common.output.model.ModelTensors; +import org.opensearch.ml.common.transport.MLTaskResponse; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobAction; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobRequest; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobResponse; +import org.opensearch.ml.engine.MLEngineClassLoader; +import org.opensearch.ml.engine.algorithms.remote.RemoteConnectorExecutor; +import org.opensearch.ml.engine.encryptor.EncryptorImpl; +import org.opensearch.ml.helper.ConnectorAccessControlHelper; +import org.opensearch.ml.model.MLModelCacheHelper; +import org.opensearch.ml.model.MLModelManager; +import org.opensearch.ml.task.MLTaskManager; +import org.opensearch.script.ScriptService; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import lombok.extern.log4j.Log4j2; + +@Log4j2 +public class CancelBatchJobTransportAction extends HandledTransportAction { + + Client client; + NamedXContentRegistry xContentRegistry; + + ClusterService clusterService; + ScriptService scriptService; + + ConnectorAccessControlHelper connectorAccessControlHelper; + EncryptorImpl encryptor; + MLModelManager mlModelManager; + + MLTaskManager mlTaskManager; + MLModelCacheHelper modelCacheHelper; + + @Inject + public CancelBatchJobTransportAction( + TransportService transportService, + ActionFilters actionFilters, + Client client, + NamedXContentRegistry xContentRegistry, + ClusterService clusterService, + ScriptService scriptService, + ConnectorAccessControlHelper connectorAccessControlHelper, + EncryptorImpl encryptor, + MLTaskManager mlTaskManager, + MLModelManager mlModelManager + ) { + super(MLCancelBatchJobAction.NAME, transportService, actionFilters, MLCancelBatchJobRequest::new); + this.client = client; + this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; + this.scriptService = scriptService; + this.connectorAccessControlHelper = connectorAccessControlHelper; + this.encryptor = encryptor; + this.mlTaskManager = mlTaskManager; + this.mlModelManager = mlModelManager; + } + + @Override + protected void doExecute(Task task, ActionRequest request, ActionListener actionListener) { + MLCancelBatchJobRequest mlCancelBatchJobRequest = MLCancelBatchJobRequest.fromActionRequest(request); + String taskId = mlCancelBatchJobRequest.getTaskId(); + GetRequest getRequest = new GetRequest(ML_TASK_INDEX).id(taskId); + + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + client.get(getRequest, ActionListener.runBefore(ActionListener.wrap(r -> { + log.debug("Completed Get Task Request, id:{}", taskId); + + if (r != null && r.isExists()) { + try (XContentParser parser = createXContentParserFromRegistry(xContentRegistry, r.getSourceAsBytesRef())) { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + MLTask mlTask = MLTask.parse(parser); + + // check if function is remote and task is of type batch prediction + if (mlTask.getTaskType() == MLTaskType.BATCH_PREDICTION && mlTask.getFunctionName() == FunctionName.REMOTE) { + processRemoteBatchPrediction(mlTask, actionListener); + } else { + actionListener + .onFailure(new IllegalArgumentException("The task ID you provided does not have any associated batch job")); + } + } catch (Exception e) { + log.error("Failed to parse ml task " + r.getId(), e); + actionListener.onFailure(e); + } + } else { + actionListener.onFailure(new OpenSearchStatusException("Fail to find task", RestStatus.NOT_FOUND)); + } + }, e -> { + if (e instanceof IndexNotFoundException) { + actionListener.onFailure(new MLResourceNotFoundException("Fail to find task")); + } else { + log.error("Failed to get ML task " + taskId, e); + actionListener.onFailure(e); + } + }), () -> context.restore())); + } catch (Exception e) { + log.error("Failed to get ML task " + taskId, e); + actionListener.onFailure(e); + } + } + + private void processRemoteBatchPrediction(MLTask mlTask, ActionListener actionListener) { + Map remoteJob = mlTask.getRemoteJob(); + + Map parameters = new HashMap<>(); + for (Map.Entry entry : remoteJob.entrySet()) { + if (entry.getValue() instanceof String) { + parameters.put(entry.getKey(), (String) entry.getValue()); + } else { + log.debug("Value for key " + entry.getKey() + " is not a String"); + } + } + + // In sagemaker, to retrieve batch transform job details, we need transformJob name. So retrieving name from the arn + parameters + .computeIfAbsent( + "TransformJobName", + key -> Optional + .ofNullable(parameters.get("TransformJobArn")) + .map(jobArn -> jobArn.substring(jobArn.lastIndexOf("/") + 1)) + .orElse(null) + ); + + RemoteInferenceInputDataSet inferenceInputDataSet = new RemoteInferenceInputDataSet(parameters, ActionType.BATCH_PREDICT_STATUS); + MLInput mlInput = MLInput.builder().algorithm(FunctionName.REMOTE).inputDataset(inferenceInputDataSet).build(); + String modelId = mlTask.getModelId(); + + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + ActionListener getModelListener = ActionListener.wrap(model -> { + if (model.getConnector() != null) { + Connector connector = model.getConnector(); + executeConnector(connector, mlInput, actionListener); + } else if (clusterService.state().metadata().hasIndex(ML_CONNECTOR_INDEX)) { + ActionListener listener = ActionListener + .wrap(connector -> { executeConnector(connector, mlInput, actionListener); }, e -> { + log.error("Failed to get connector " + model.getConnectorId(), e); + actionListener.onFailure(e); + }); + try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + connectorAccessControlHelper + .getConnector(client, model.getConnectorId(), ActionListener.runBefore(listener, threadContext::restore)); + } + } else { + actionListener.onFailure(new ResourceNotFoundException("Can't find connector " + model.getConnectorId())); + } + }, e -> { + log.error("Failed to retrieve the ML model with the given ID", e); + actionListener + .onFailure( + new OpenSearchStatusException("Failed to retrieve the ML model for the given task ID", RestStatus.NOT_FOUND) + ); + }); + mlModelManager.getModel(modelId, null, null, ActionListener.runBefore(getModelListener, context::restore)); + } catch (Exception e) { + log.error("Unable to fetch cancel batch job in ml task ", e); + throw new OpenSearchException("Unable to fetch cancel batch job in ml task " + e.getMessage()); + } + } + + private void executeConnector(Connector connector, MLInput mlInput, ActionListener actionListener) { + if (connectorAccessControlHelper.validateConnectorAccess(client, connector)) { + connector.decrypt(CANCEL_BATCH_PREDICT.name(), (credential) -> encryptor.decrypt(credential)); + RemoteConnectorExecutor connectorExecutor = MLEngineClassLoader + .initInstance(connector.getProtocol(), connector, Connector.class); + connectorExecutor.setScriptService(scriptService); + connectorExecutor.setClusterService(clusterService); + connectorExecutor.setClient(client); + connectorExecutor.setXContentRegistry(xContentRegistry); + connectorExecutor.executeAction(CANCEL_BATCH_PREDICT.name(), mlInput, ActionListener.wrap(taskResponse -> { + processTaskResponse(taskResponse, actionListener); + }, e -> { actionListener.onFailure(e); })); + } else { + actionListener + .onFailure(new OpenSearchStatusException("You don't have permission to access this connector", RestStatus.FORBIDDEN)); + } + } + + private void processTaskResponse(MLTaskResponse taskResponse, ActionListener actionListener) { + try { + ModelTensorOutput tensorOutput = (ModelTensorOutput) taskResponse.getOutput(); + if (tensorOutput != null && tensorOutput.getMlModelOutputs() != null && !tensorOutput.getMlModelOutputs().isEmpty()) { + ModelTensors modelOutput = tensorOutput.getMlModelOutputs().get(0); + if (modelOutput.getStatusCode() != null && modelOutput.getStatusCode().equals(HttpStatus.SC_OK)) { + actionListener.onResponse(new MLCancelBatchJobResponse(RestStatus.OK)); + } else { + log.debug("The status code from remote service is: " + modelOutput.getStatusCode()); + actionListener.onFailure(new OpenSearchException("Couldn't cancel the transform job. Please try again")); + } + } else { + log.debug("ML Model Outputs are null or empty."); + actionListener.onFailure(new ResourceNotFoundException("Couldn't fetch status of the transform job")); + } + } catch (Exception e) { + log.error("Unable to fetch status for ml task ", e); + } + } +} diff --git a/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java index 88c05f71c1..01b4724046 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java @@ -6,15 +6,29 @@ package org.opensearch.ml.action.tasks; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.ml.common.CommonValue.ML_CONNECTOR_INDEX; import static org.opensearch.ml.common.CommonValue.ML_TASK_INDEX; +import static org.opensearch.ml.common.MLTask.REMOTE_JOB_FIELD; +import static org.opensearch.ml.common.MLTask.STATE_FIELD; +import static org.opensearch.ml.common.MLTaskState.CANCELLED; +import static org.opensearch.ml.common.MLTaskState.COMPLETED; +import static org.opensearch.ml.common.connector.ConnectorAction.ActionType.BATCH_PREDICT_STATUS; +import static org.opensearch.ml.utils.MLExceptionUtils.logException; import static org.opensearch.ml.utils.MLNodeUtils.createXContentParserFromRegistry; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionRequest; import org.opensearch.action.get.GetRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; @@ -22,11 +36,29 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.ml.common.FunctionName; +import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.MLTaskType; +import org.opensearch.ml.common.connector.Connector; +import org.opensearch.ml.common.connector.ConnectorAction.ActionType; +import org.opensearch.ml.common.dataset.remote.RemoteInferenceInputDataSet; import org.opensearch.ml.common.exception.MLResourceNotFoundException; +import org.opensearch.ml.common.input.MLInput; +import org.opensearch.ml.common.output.model.ModelTensorOutput; +import org.opensearch.ml.common.output.model.ModelTensors; +import org.opensearch.ml.common.transport.MLTaskResponse; import org.opensearch.ml.common.transport.task.MLTaskGetAction; import org.opensearch.ml.common.transport.task.MLTaskGetRequest; import org.opensearch.ml.common.transport.task.MLTaskGetResponse; +import org.opensearch.ml.engine.MLEngineClassLoader; +import org.opensearch.ml.engine.algorithms.remote.RemoteConnectorExecutor; +import org.opensearch.ml.engine.encryptor.EncryptorImpl; +import org.opensearch.ml.helper.ConnectorAccessControlHelper; +import org.opensearch.ml.model.MLModelCacheHelper; +import org.opensearch.ml.model.MLModelManager; +import org.opensearch.ml.task.MLTaskManager; +import org.opensearch.script.ScriptService; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -38,16 +70,38 @@ public class GetTaskTransportAction extends HandledTransportAction actionListener) { + Map remoteJob = mlTask.getRemoteJob(); + + Map parameters = new HashMap<>(); + for (Map.Entry entry : remoteJob.entrySet()) { + if (entry.getValue() instanceof String) { + parameters.put(entry.getKey(), (String) entry.getValue()); + } else { + log.debug("Value for key " + entry.getKey() + " is not a String"); + } + } + // In sagemaker, to retrieve batch transform job details, we need transformJob name. So retrieving name from the arn + parameters + .computeIfAbsent( + "TransformJobName", + key -> Optional + .ofNullable(parameters.get("TransformJobArn")) + .map(jobArn -> jobArn.substring(jobArn.lastIndexOf("/") + 1)) + .orElse(null) + ); + + RemoteInferenceInputDataSet inferenceInputDataSet = new RemoteInferenceInputDataSet(parameters, ActionType.BATCH_PREDICT_STATUS); + MLInput mlInput = MLInput.builder().algorithm(FunctionName.REMOTE).inputDataset(inferenceInputDataSet).build(); + String modelId = mlTask.getModelId(); + + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + ActionListener getModelListener = ActionListener.wrap(model -> { + if (model.getConnector() != null) { + Connector connector = model.getConnector(); + executeConnector(connector, mlInput, taskId, mlTask, remoteJob, actionListener); + } else if (clusterService.state().metadata().hasIndex(ML_CONNECTOR_INDEX)) { + ActionListener listener = ActionListener.wrap(connector -> { + executeConnector(connector, mlInput, taskId, mlTask, remoteJob, actionListener); + }, e -> { + log.error("Failed to get connector " + model.getConnectorId(), e); + actionListener.onFailure(e); + }); + try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) { + connectorAccessControlHelper + .getConnector(client, model.getConnectorId(), ActionListener.runBefore(listener, threadContext::restore)); + } + } else { + actionListener.onFailure(new ResourceNotFoundException("Can't find connector " + model.getConnectorId())); + } + }, e -> { + log.error("Failed to retrieve the ML model for the given task ID", e); + actionListener + .onFailure( + new OpenSearchStatusException("Failed to retrieve the ML model for the given task ID", RestStatus.NOT_FOUND) + ); + }); + mlModelManager.getModel(modelId, null, null, ActionListener.runBefore(getModelListener, context::restore)); + } catch (Exception e) { + log.error("Unable to fetch status for ml task ", e); + throw new OpenSearchException("Unable to fetch status for ml task " + e.getMessage()); + } + } + + private void executeConnector( + Connector connector, + MLInput mlInput, + String taskId, + MLTask mlTask, + Map transformJob, + ActionListener actionListener + ) { + if (connectorAccessControlHelper.validateConnectorAccess(client, connector)) { + connector.decrypt(BATCH_PREDICT_STATUS.name(), (credential) -> encryptor.decrypt(credential)); + RemoteConnectorExecutor connectorExecutor = MLEngineClassLoader + .initInstance(connector.getProtocol(), connector, Connector.class); + connectorExecutor.setScriptService(scriptService); + connectorExecutor.setClusterService(clusterService); + connectorExecutor.setClient(client); + connectorExecutor.setXContentRegistry(xContentRegistry); + connectorExecutor.executeAction(BATCH_PREDICT_STATUS.name(), mlInput, ActionListener.wrap(taskResponse -> { + processTaskResponse(mlTask, taskId, taskResponse, transformJob, actionListener); + }, e -> { actionListener.onFailure(e); })); + } else { + actionListener + .onFailure(new OpenSearchStatusException("You don't have permission to access this connector", RestStatus.FORBIDDEN)); + } + } + + private void processTaskResponse( + MLTask mlTask, + String taskId, + MLTaskResponse taskResponse, + Map remoteJob, + ActionListener actionListener + ) { + try { + ModelTensorOutput tensorOutput = (ModelTensorOutput) taskResponse.getOutput(); + if (tensorOutput != null && tensorOutput.getMlModelOutputs() != null && !tensorOutput.getMlModelOutputs().isEmpty()) { + ModelTensors modelOutput = tensorOutput.getMlModelOutputs().get(0); + if (modelOutput.getMlModelTensors() != null && !modelOutput.getMlModelTensors().isEmpty()) { + Map remoteJobStatus = (Map) modelOutput.getMlModelTensors().get(0).getDataAsMap(); + if (remoteJobStatus != null) { + remoteJob.putAll(remoteJobStatus); + Map updatedTask = new HashMap<>(); + updatedTask.put(REMOTE_JOB_FIELD, remoteJob); + + if ((remoteJob.containsKey("status") && remoteJob.get("status").equals("completed")) + || (remoteJob.containsKey("TransformJobStatus") && remoteJob.get("TransformJobStatus").equals("Completed"))) { + updatedTask.put(STATE_FIELD, COMPLETED); + mlTask.setState(COMPLETED); + + } else if ((remoteJob.containsKey("status") && remoteJob.get("status").equals("cancelled")) + || (remoteJob.containsKey("TransformJobStatus") && remoteJob.get("TransformJobStatus").equals("Stopped"))) { + updatedTask.put(STATE_FIELD, CANCELLED); + mlTask.setState(CANCELLED); + } + mlTaskManager.updateMLTaskDirectly(taskId, updatedTask, ActionListener.wrap(response -> { + actionListener.onResponse(MLTaskGetResponse.builder().mlTask(mlTask).build()); + }, e -> { + logException("Failed to update task for batch predict model", e, log); + actionListener.onFailure(e); + })); + } else { + log.debug("Transform job status is null."); + actionListener.onFailure(new ResourceNotFoundException("Couldn't fetch status of the transform job")); + } + } else { + log.debug("ML Model Tensors are null or empty."); + actionListener.onFailure(new ResourceNotFoundException("Couldn't fetch status of the transform job")); + } + } else { + log.debug("ML Model Outputs are null or empty."); + actionListener.onFailure(new ResourceNotFoundException("Couldn't fetch status of the transform job")); + } + } catch (Exception e) { + log.error("Unable to fetch status for ml task ", e); + } } } diff --git a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java index b4abd328c7..27f34b35bb 100644 --- a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java +++ b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java @@ -83,6 +83,7 @@ import org.opensearch.ml.action.stats.MLStatsNodesAction; import org.opensearch.ml.action.stats.MLStatsNodesTransportAction; import org.opensearch.ml.action.syncup.TransportSyncUpOnNodeAction; +import org.opensearch.ml.action.tasks.CancelBatchJobTransportAction; import org.opensearch.ml.action.tasks.DeleteTaskTransportAction; import org.opensearch.ml.action.tasks.GetTaskTransportAction; import org.opensearch.ml.action.tasks.SearchTaskTransportAction; @@ -151,6 +152,7 @@ import org.opensearch.ml.common.transport.prediction.MLPredictionTaskAction; import org.opensearch.ml.common.transport.register.MLRegisterModelAction; import org.opensearch.ml.common.transport.sync.MLSyncUpAction; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobAction; import org.opensearch.ml.common.transport.task.MLTaskDeleteAction; import org.opensearch.ml.common.transport.task.MLTaskGetAction; import org.opensearch.ml.common.transport.task.MLTaskSearchAction; @@ -219,6 +221,7 @@ import org.opensearch.ml.processor.MLInferenceSearchResponseProcessor; import org.opensearch.ml.repackage.com.google.common.collect.ImmutableList; import org.opensearch.ml.rest.RestMLBatchIngestAction; +import org.opensearch.ml.rest.RestMLCancelBatchJobAction; import org.opensearch.ml.rest.RestMLCreateConnectorAction; import org.opensearch.ml.rest.RestMLCreateControllerAction; import org.opensearch.ml.rest.RestMLDeleteAgentAction; @@ -444,7 +447,8 @@ public MachineLearningPlugin(Settings settings) { new ActionHandler<>(MLListToolsAction.INSTANCE, ListToolsTransportAction.class), new ActionHandler<>(MLGetToolAction.INSTANCE, GetToolTransportAction.class), new ActionHandler<>(MLConfigGetAction.INSTANCE, GetConfigTransportAction.class), - new ActionHandler<>(MLBatchIngestionAction.INSTANCE, TransportBatchIngestionAction.class) + new ActionHandler<>(MLBatchIngestionAction.INSTANCE, TransportBatchIngestionAction.class), + new ActionHandler<>(MLCancelBatchJobAction.INSTANCE, CancelBatchJobTransportAction.class) ); } @@ -764,6 +768,7 @@ public List getRestHandlers( RestMLGetToolAction restMLGetToolAction = new RestMLGetToolAction(toolFactories); RestMLGetConfigAction restMLGetConfigAction = new RestMLGetConfigAction(); RestMLBatchIngestAction restMLBatchIngestAction = new RestMLBatchIngestAction(); + RestMLCancelBatchJobAction restMLCancelBatchJobAction = new RestMLCancelBatchJobAction(); return ImmutableList .of( restMLStatsAction, @@ -817,7 +822,8 @@ public List getRestHandlers( restMLListToolsAction, restMLGetToolAction, restMLGetConfigAction, - restMLBatchIngestAction + restMLBatchIngestAction, + restMLCancelBatchJobAction ); } diff --git a/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java b/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java new file mode 100644 index 0000000000..33c7314be2 --- /dev/null +++ b/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.rest; + +import static org.opensearch.ml.plugin.MachineLearningPlugin.ML_BASE_URI; +import static org.opensearch.ml.utils.RestActionUtils.PARAMETER_TASK_ID; +import static org.opensearch.ml.utils.RestActionUtils.getParameterId; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobAction; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobRequest; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +public class RestMLCancelBatchJobAction extends BaseRestHandler { + private static final String ML_CANCEL_BATCH_ACTION = "ml_cancel_batch_action"; + + /** + * Constructor + */ + public RestMLCancelBatchJobAction() {} + + @Override + public String getName() { + return ML_CANCEL_BATCH_ACTION; + } + + @Override + public List routes() { + return ImmutableList + .of( + new Route( + RestRequest.Method.POST, + String.format(Locale.ROOT, "%s/tasks/{%s}/_cancel_batch", ML_BASE_URI, PARAMETER_TASK_ID) + ) + ); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + MLCancelBatchJobRequest mlCancelBatchJobRequest = getRequest(request); + return channel -> client.execute(MLCancelBatchJobAction.INSTANCE, mlCancelBatchJobRequest, new RestToXContentListener<>(channel)); + } + + /** + * Creates a MLCancelBatchJobRequest from a RestRequest + * + * @param request RestRequest + * @return MLCancelBatchJobRequest + */ + @VisibleForTesting + MLCancelBatchJobRequest getRequest(RestRequest request) throws IOException { + String taskId = getParameterId(request, PARAMETER_TASK_ID); + + return new MLCancelBatchJobRequest(taskId); + } +} diff --git a/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java b/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java index 72c43bd58f..3b2e70d4b8 100644 --- a/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java +++ b/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java @@ -14,9 +14,12 @@ import static org.opensearch.ml.plugin.MachineLearningPlugin.PREDICT_THREAD_POOL; import static org.opensearch.ml.plugin.MachineLearningPlugin.REMOTE_PREDICT_THREAD_POOL; import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_MODEL_AUTO_DEPLOY_ENABLE; +import static org.opensearch.ml.utils.MLExceptionUtils.logException; import java.time.Instant; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import org.opensearch.OpenSearchException; @@ -48,6 +51,7 @@ import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.MLTaskType; import org.opensearch.ml.common.connector.ConnectorAction; +import org.opensearch.ml.common.connector.ConnectorAction.ActionType; import org.opensearch.ml.common.dataset.MLInputDataType; import org.opensearch.ml.common.dataset.MLInputDataset; import org.opensearch.ml.common.dataset.remote.RemoteInferenceInputDataSet; @@ -55,6 +59,7 @@ import org.opensearch.ml.common.output.MLOutput; import org.opensearch.ml.common.output.MLPredictionOutput; import org.opensearch.ml.common.output.model.ModelTensorOutput; +import org.opensearch.ml.common.output.model.ModelTensors; import org.opensearch.ml.common.transport.MLTaskResponse; import org.opensearch.ml.common.transport.deploy.MLDeployModelAction; import org.opensearch.ml.common.transport.deploy.MLDeployModelRequest; @@ -228,11 +233,18 @@ protected void executeTask(MLPredictionTaskRequest request, ActionListener dataFrameActionListener = ActionListener.wrap(dataSet -> { @@ -336,12 +347,60 @@ private void runPredict( if (mlInput.getAlgorithm() == FunctionName.REMOTE) { long startTime = System.nanoTime(); ActionListener trackPredictDurationListener = ActionListener.wrap(output -> { + if (output.getOutput() instanceof ModelTensorOutput) { validateOutputSchema(modelId, (ModelTensorOutput) output.getOutput()); } - handleAsyncMLTaskComplete(mlTask); - mlModelManager.trackPredictDuration(modelId, startTime); - internalListener.onResponse(output); + if (mlTask.getTaskType().equals(MLTaskType.BATCH_PREDICTION)) { + Map remoteJob = new HashMap<>(); + ModelTensorOutput tensorOutput = (ModelTensorOutput) output.getOutput(); + if (tensorOutput != null + && tensorOutput.getMlModelOutputs() != null + && !tensorOutput.getMlModelOutputs().isEmpty()) { + ModelTensors modelOutput = tensorOutput.getMlModelOutputs().get(0); + if (modelOutput.getMlModelTensors() != null && !modelOutput.getMlModelTensors().isEmpty()) { + Map dataAsMap = (Map) modelOutput + .getMlModelTensors() + .get(0) + .getDataAsMap(); + if (dataAsMap != null + && (dataAsMap.containsKey("TransformJobArn") || dataAsMap.containsKey("id"))) { + remoteJob.putAll(dataAsMap); + mlTask.setRemoteJob(remoteJob); + mlTask.setTaskId(null); + mlTaskManager.createMLTask(mlTask, ActionListener.wrap(response -> { + String taskId = response.getId(); + mlTask.setTaskId(taskId); + MLPredictionOutput outputBuilder = MLPredictionOutput + .builder() + .taskId(taskId) + .status(MLTaskState.CREATED.name()) + .build(); + + MLTaskResponse predictOutput = MLTaskResponse.builder().output(outputBuilder).build(); + internalListener.onResponse(predictOutput); + }, e -> { + logException("Failed to create task for batch predict model", e, log); + internalListener.onFailure(e); + })); + } else { + log.debug("Batch transform job output from remote model did not return the job ID"); + internalListener + .onFailure(new ResourceNotFoundException("Unable to create batch transform job")); + } + } else { + log.debug("ML Model Tensors are null or empty."); + internalListener.onFailure(new ResourceNotFoundException("Unable to create batch transform job")); + } + } else { + log.debug("ML Model Outputs are null or empty."); + internalListener.onFailure(new ResourceNotFoundException("Unable to create batch transform job")); + } + } else { + handleAsyncMLTaskComplete(mlTask); + mlModelManager.trackPredictDuration(modelId, startTime); + internalListener.onResponse(output); + } }, e -> handlePredictFailure(mlTask, internalListener, e, false, modelId, actionName)); predictor.asyncPredict(mlInput, trackPredictDurationListener); } else { diff --git a/plugin/src/test/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportActionTests.java new file mode 100644 index 0000000000..99d9fbf8a1 --- /dev/null +++ b/plugin/src/test/java/org/opensearch/ml/action/tasks/CancelBatchJobTransportActionTests.java @@ -0,0 +1,309 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.action.tasks; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.get.GetResult; +import org.opensearch.ml.common.FunctionName; +import org.opensearch.ml.common.MLModel; +import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.MLTaskType; +import org.opensearch.ml.common.connector.Connector; +import org.opensearch.ml.common.connector.ConnectorAction; +import org.opensearch.ml.common.connector.HttpConnector; +import org.opensearch.ml.common.output.model.ModelTensor; +import org.opensearch.ml.common.output.model.ModelTensorOutput; +import org.opensearch.ml.common.output.model.ModelTensors; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobRequest; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobResponse; +import org.opensearch.ml.engine.encryptor.EncryptorImpl; +import org.opensearch.ml.helper.ConnectorAccessControlHelper; +import org.opensearch.ml.model.MLModelManager; +import org.opensearch.ml.task.MLTaskManager; +import org.opensearch.script.ScriptService; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class CancelBatchJobTransportActionTests extends OpenSearchTestCase { + @Mock + ThreadPool threadPool; + + @Mock + Client client; + + @Mock + NamedXContentRegistry xContentRegistry; + + @Mock + TransportService transportService; + + @Mock + private ClusterService clusterService; + @Mock + private ScriptService scriptService; + @Mock + ClusterState clusterState; + + @Mock + private Metadata metaData; + + @Mock + ActionFilters actionFilters; + @Mock + private ConnectorAccessControlHelper connectorAccessControlHelper; + + @Mock + private EncryptorImpl encryptor; + + @Mock + ActionListener actionListener; + @Mock + private MLModelManager mlModelManager; + + @Mock + private MLTaskManager mlTaskManager; + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + CancelBatchJobTransportAction cancelBatchJobTransportAction; + MLCancelBatchJobRequest mlCancelBatchJobRequest; + ThreadContext threadContext; + + @Before + public void setup() throws IOException { + MockitoAnnotations.openMocks(this); + mlCancelBatchJobRequest = MLCancelBatchJobRequest.builder().taskId("test_id").build(); + + Settings settings = Settings.builder().build(); + threadContext = new ThreadContext(settings); + when(client.threadPool()).thenReturn(threadPool); + when(threadPool.getThreadContext()).thenReturn(threadContext); + + doReturn(clusterState).when(clusterService).state(); + doReturn(metaData).when(clusterState).metadata(); + + doReturn(true).when(metaData).hasIndex(anyString()); + + cancelBatchJobTransportAction = spy( + new CancelBatchJobTransportAction( + transportService, + actionFilters, + client, + xContentRegistry, + clusterService, + scriptService, + connectorAccessControlHelper, + encryptor, + mlTaskManager, + mlModelManager + ) + ); + + MLModel mlModel = mock(MLModel.class); + + Connector connector = HttpConnector + .builder() + .name("test") + .protocol("http") + .version("1") + .credential(Map.of("api_key", "credential_value")) + .parameters(Map.of("param1", "value1")) + .actions( + Arrays + .asList( + ConnectorAction + .builder() + .actionType(ConnectorAction.ActionType.BATCH_PREDICT_STATUS) + .method("POST") + .url("https://api.sagemaker.us-east-1.amazonaws.com/DescribeTransformJob") + .headers(Map.of("Authorization", "Bearer ${credential.api_key}")) + .requestBody("{ \"TransformJobName\" : \"${parameters.TransformJobName}\"}") + .build() + ) + ) + .build(); + + when(mlModel.getConnectorId()).thenReturn("testConnectorID"); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(3); + listener.onResponse(mlModel); + return null; + }).when(mlModelManager).getModel(eq("testModelID"), any(), any(), isA(ActionListener.class)); + + when(connectorAccessControlHelper.validateConnectorAccess(eq(client), any())).thenReturn(true); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onResponse(connector); + return null; + }).when(connectorAccessControlHelper).getConnector(eq(client), anyString(), any()); + + } + + public void testGetTask_NullResponse() { + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse(null); + return null; + }).when(client).get(any(), any()); + cancelBatchJobTransportAction.doExecute(null, mlCancelBatchJobRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("Fail to find task", argumentCaptor.getValue().getMessage()); + } + + public void testGetTask_RuntimeException() { + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onFailure(new RuntimeException("errorMessage")); + return null; + }).when(client).get(any(), any()); + cancelBatchJobTransportAction.doExecute(null, mlCancelBatchJobRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("errorMessage", argumentCaptor.getValue().getMessage()); + } + + public void testGetTask_IndexNotFoundException() { + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onFailure(new IndexNotFoundException("Index Not Found")); + return null; + }).when(client).get(any(), any()); + cancelBatchJobTransportAction.doExecute(null, mlCancelBatchJobRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("Fail to find task", argumentCaptor.getValue().getMessage()); + } + + @Ignore + public void testGetTask_SuccessBatchPredictCancel() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + ModelTensor modelTensor = ModelTensor.builder().name("response").dataAsMap(Map.of("TransformJobStatus", "COMPLETED")).build(); + ModelTensorOutput modelTensorOutput = ModelTensorOutput + .builder() + .mlModelOutputs(List.of(ModelTensors.builder().mlModelTensors(List.of(modelTensor)).build())) + .build(); + + cancelBatchJobTransportAction.doExecute(null, mlCancelBatchJobRequest, actionListener); + verify(actionListener).onResponse(any(MLCancelBatchJobResponse.class)); + } + + public void test_BatchPredictCancel_NoConnector() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + when(connectorAccessControlHelper.validateConnectorAccess(eq(client), any())).thenReturn(false); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + cancelBatchJobTransportAction.doExecute(null, mlCancelBatchJobRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("You don't have permission to access this connector", argumentCaptor.getValue().getMessage()); + } + + public void test_BatchPredictStatus_NoAccessToConnector() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onFailure(new ResourceNotFoundException("Failed to get connector")); + return null; + }).when(connectorAccessControlHelper).getConnector(eq(client), anyString(), any()); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + cancelBatchJobTransportAction.doExecute(null, mlCancelBatchJobRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("Failed to get connector", argumentCaptor.getValue().getMessage()); + } + + public GetResponse prepareMLTask(FunctionName functionName, MLTaskType mlTaskType, Map remoteJob) throws IOException { + MLTask mlTask = MLTask + .builder() + .taskId("taskID") + .modelId("testModelID") + .functionName(functionName) + .taskType(mlTaskType) + .remoteJob(remoteJob) + .build(); + XContentBuilder content = mlTask.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + BytesReference bytesReference = BytesReference.bytes(content); + GetResult getResult = new GetResult("indexName", "111", 111l, 111l, 111l, true, bytesReference, null, null); + GetResponse getResponse = new GetResponse(getResult); + return getResponse; + } +} diff --git a/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java index 83da0f8273..3707c89eae 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java @@ -6,29 +6,63 @@ package org.opensearch.ml.action.tasks; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.rules.ExpectedException; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.get.GetResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexNotFoundException; +import org.opensearch.index.get.GetResult; +import org.opensearch.ml.common.FunctionName; +import org.opensearch.ml.common.MLModel; +import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.MLTaskType; +import org.opensearch.ml.common.connector.Connector; +import org.opensearch.ml.common.connector.ConnectorAction; +import org.opensearch.ml.common.connector.HttpConnector; +import org.opensearch.ml.common.output.model.ModelTensor; +import org.opensearch.ml.common.output.model.ModelTensorOutput; +import org.opensearch.ml.common.output.model.ModelTensors; import org.opensearch.ml.common.transport.task.MLTaskGetRequest; import org.opensearch.ml.common.transport.task.MLTaskGetResponse; +import org.opensearch.ml.engine.encryptor.EncryptorImpl; +import org.opensearch.ml.helper.ConnectorAccessControlHelper; +import org.opensearch.ml.model.MLModelManager; +import org.opensearch.ml.task.MLTaskManager; +import org.opensearch.script.ScriptService; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -46,11 +80,31 @@ public class GetTaskTransportActionTests extends OpenSearchTestCase { @Mock TransportService transportService; + @Mock + private ClusterService clusterService; + @Mock + private ScriptService scriptService; + @Mock + ClusterState clusterState; + + @Mock + private Metadata metaData; + @Mock ActionFilters actionFilters; + @Mock + private ConnectorAccessControlHelper connectorAccessControlHelper; + + @Mock + private EncryptorImpl encryptor; @Mock ActionListener actionListener; + @Mock + private MLModelManager mlModelManager; + + @Mock + private MLTaskManager mlTaskManager; @Rule public ExpectedException exceptionRule = ExpectedException.none(); @@ -64,12 +118,71 @@ public void setup() throws IOException { MockitoAnnotations.openMocks(this); mlTaskGetRequest = MLTaskGetRequest.builder().taskId("test_id").build(); - getTaskTransportAction = spy(new GetTaskTransportAction(transportService, actionFilters, client, xContentRegistry)); - Settings settings = Settings.builder().build(); threadContext = new ThreadContext(settings); when(client.threadPool()).thenReturn(threadPool); when(threadPool.getThreadContext()).thenReturn(threadContext); + + doReturn(clusterState).when(clusterService).state(); + doReturn(metaData).when(clusterState).metadata(); + + doReturn(true).when(metaData).hasIndex(anyString()); + + getTaskTransportAction = spy( + new GetTaskTransportAction( + transportService, + actionFilters, + client, + xContentRegistry, + clusterService, + scriptService, + connectorAccessControlHelper, + encryptor, + mlTaskManager, + mlModelManager + ) + ); + + MLModel mlModel = mock(MLModel.class); + + Connector connector = HttpConnector + .builder() + .name("test") + .protocol("http") + .version("1") + .credential(Map.of("api_key", "credential_value")) + .parameters(Map.of("param1", "value1")) + .actions( + Arrays + .asList( + ConnectorAction + .builder() + .actionType(ConnectorAction.ActionType.BATCH_PREDICT_STATUS) + .method("POST") + .url("https://api.sagemaker.us-east-1.amazonaws.com/DescribeTransformJob") + .headers(Map.of("Authorization", "Bearer ${credential.api_key}")) + .requestBody("{ \"TransformJobName\" : \"${parameters.TransformJobName}\"}") + .build() + ) + ) + .build(); + + when(mlModel.getConnectorId()).thenReturn("testConnectorID"); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(3); + listener.onResponse(mlModel); + return null; + }).when(mlModelManager).getModel(eq("testModelID"), any(), any(), isA(ActionListener.class)); + + when(connectorAccessControlHelper.validateConnectorAccess(eq(client), any())).thenReturn(true); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onResponse(connector); + return null; + }).when(connectorAccessControlHelper).getConnector(eq(client), anyString(), any()); + } public void testGetTask_NullResponse() { @@ -107,4 +220,115 @@ public void testGetTask_IndexNotFoundException() { verify(actionListener).onFailure(argumentCaptor.capture()); assertEquals("Fail to find task", argumentCaptor.getValue().getMessage()); } + + @Ignore + public void testGetTask_SuccessBatchPredictStatus() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + ModelTensor modelTensor = ModelTensor.builder().name("response").dataAsMap(Map.of("TransformJobStatus", "COMPLETED")).build(); + ModelTensorOutput modelTensorOutput = ModelTensorOutput + .builder() + .mlModelOutputs(List.of(ModelTensors.builder().mlModelTensors(List.of(modelTensor)).build())) + .build(); + + getTaskTransportAction.doExecute(null, mlTaskGetRequest, actionListener); + verify(actionListener).onResponse(any(MLTaskGetResponse.class)); + } + + public void test_BatchPredictStatus_NoConnector() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + when(connectorAccessControlHelper.validateConnectorAccess(eq(client), any())).thenReturn(false); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + getTaskTransportAction.doExecute(null, mlTaskGetRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("You don't have permission to access this connector", argumentCaptor.getValue().getMessage()); + } + + public void test_BatchPredictStatus_NoAccessToConnector() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onFailure(new ResourceNotFoundException("Failed to get connector")); + return null; + }).when(connectorAccessControlHelper).getConnector(eq(client), anyString(), any()); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + getTaskTransportAction.doExecute(null, mlTaskGetRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("Failed to get connector", argumentCaptor.getValue().getMessage()); + } + + public void test_BatchPredictStatus_NoModel() throws IOException { + Map remoteJob = new HashMap<>(); + remoteJob.put("Status", "IN PROGRESS"); + remoteJob.put("TransformJobName", "SM-offline-batch-transform13"); + + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onFailure(new ResourceNotFoundException("Failed to get connector")); + return null; + }).when(connectorAccessControlHelper).getConnector(eq(client), anyString(), any()); + + GetResponse getResponse = prepareMLTask(FunctionName.REMOTE, MLTaskType.BATCH_PREDICTION, remoteJob); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(getResponse); + return null; + }).when(client).get(any(), any()); + + getTaskTransportAction.doExecute(null, mlTaskGetRequest, actionListener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(argumentCaptor.capture()); + assertEquals("Failed to get connector", argumentCaptor.getValue().getMessage()); + } + + public GetResponse prepareMLTask(FunctionName functionName, MLTaskType mlTaskType, Map remoteJob) throws IOException { + MLTask mlTask = MLTask + .builder() + .taskId("taskID") + .modelId("testModelID") + .functionName(functionName) + .taskType(mlTaskType) + .remoteJob(remoteJob) + .build(); + XContentBuilder content = mlTask.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS); + BytesReference bytesReference = BytesReference.bytes(content); + GetResult getResult = new GetResult("indexName", "111", 111l, 111l, 111l, true, bytesReference, null, null); + GetResponse getResponse = new GetResponse(getResult); + return getResponse; + } } diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java new file mode 100644 index 0000000000..1498750e6a --- /dev/null +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.rest; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.opensearch.ml.utils.RestActionUtils.PARAMETER_TASK_ID; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobAction; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobRequest; +import org.opensearch.ml.common.transport.task.MLCancelBatchJobResponse; +import org.opensearch.rest.RestChannel; +import org.opensearch.rest.RestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RestMLCancelBatchJobActionTests extends OpenSearchTestCase { + + private RestMLCancelBatchJobAction restMLCancelBatchJobAction; + + NodeClient client; + private ThreadPool threadPool; + + @Mock + RestChannel channel; + + @Before + public void setup() { + restMLCancelBatchJobAction = new RestMLCancelBatchJobAction(); + + threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool"); + client = spy(new NodeClient(Settings.EMPTY, threadPool)); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + return null; + }).when(client).execute(eq(MLCancelBatchJobAction.INSTANCE), any(), any()); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + client.close(); + } + + public void testConstructor() { + RestMLCancelBatchJobAction mlCancelBatchJobAction = new RestMLCancelBatchJobAction(); + assertNotNull(mlCancelBatchJobAction); + } + + public void testGetName() { + String actionName = restMLCancelBatchJobAction.getName(); + assertFalse(Strings.isNullOrEmpty(actionName)); + assertEquals("ml_cancel_batch_action", actionName); + } + + public void testRoutes() { + List routes = restMLCancelBatchJobAction.routes(); + assertNotNull(routes); + assertFalse(routes.isEmpty()); + RestHandler.Route route = routes.get(0); + assertEquals(RestRequest.Method.POST, route.getMethod()); + assertEquals("/_plugins/_ml/tasks/{task_id}/_cancel_batch", route.getPath()); + } + + public void test_PrepareRequest() throws Exception { + RestRequest request = getRestRequest(); + restMLCancelBatchJobAction.handleRequest(request, channel, client); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(MLCancelBatchJobRequest.class); + verify(client, times(1)).execute(eq(MLCancelBatchJobAction.INSTANCE), argumentCaptor.capture(), any()); + String taskId = argumentCaptor.getValue().getTaskId(); + assertEquals(taskId, "test_id"); + } + + private RestRequest getRestRequest() { + Map params = new HashMap<>(); + params.put(PARAMETER_TASK_ID, "test_id"); + RestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withParams(params).build(); + return request; + } +} diff --git a/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java b/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java index cbde703543..064008a9c4 100644 --- a/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java +++ b/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java @@ -25,10 +25,12 @@ import org.junit.rules.ExpectedException; import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.opensearch.OpenSearchStatusException; import org.opensearch.Version; import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexResponse; import org.opensearch.client.Client; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; @@ -49,11 +51,13 @@ import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.connector.ConnectorAction; import org.opensearch.ml.common.dataframe.DataFrame; import org.opensearch.ml.common.dataset.DataFrameInputDataset; import org.opensearch.ml.common.dataset.MLInputDataset; import org.opensearch.ml.common.dataset.SearchQueryInputDataset; import org.opensearch.ml.common.dataset.TextDocsInputDataSet; +import org.opensearch.ml.common.dataset.remote.RemoteInferenceInputDataSet; import org.opensearch.ml.common.input.MLInput; import org.opensearch.ml.common.input.parameter.rcf.BatchRCFParams; import org.opensearch.ml.common.output.MLPredictionOutput; @@ -412,6 +416,109 @@ public void testValidateModelTensorOutputSuccess() { taskRunner.validateOutputSchema("testId", modelTensorOutput); } + public void testValidateBatchPredictionSuccess() throws IOException { + setupMocks(true, false, false, false); + RemoteInferenceInputDataSet remoteInferenceInputDataSet = RemoteInferenceInputDataSet + .builder() + .parameters( + Map + .of( + "messages", + "[{\\\"role\\\":\\\"system\\\",\\\"content\\\":\\\"You are a helpful assistant.\\\"}," + + "{\\\"role\\\":\\\"user\\\",\\\"content\\\":\\\"Hello!\\\"}]" + ) + ) + .actionType(ConnectorAction.ActionType.BATCH_PREDICT) + .build(); + MLPredictionTaskRequest remoteInputRequest = MLPredictionTaskRequest + .builder() + .modelId("test_model") + .mlInput(MLInput.builder().algorithm(FunctionName.REMOTE).inputDataset(remoteInferenceInputDataSet).build()) + .build(); + Predictable predictor = mock(Predictable.class); + when(predictor.isModelReady()).thenReturn(true); + ModelTensor modelTensor = ModelTensor + .builder() + .name("response") + .dataAsMap(Map.of("TransformJobArn", "arn:aws:sagemaker:us-east-1:802041417063:transform-job/batch-transform-01")) + .build(); + Map modelInterface = Map + .of( + "output", + "{\"properties\":{\"inference_results\":{\"description\":\"This is a test description field\"," + "\"type\":\"array\"}}}" + ); + ModelTensorOutput modelTensorOutput = ModelTensorOutput + .builder() + .mlModelOutputs(List.of(ModelTensors.builder().mlModelTensors(List.of(modelTensor)).build())) + .build(); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener.onResponse(MLTaskResponse.builder().output(modelTensorOutput).build()); + return null; + }).when(predictor).asyncPredict(any(), any()); + + IndexResponse indexResponse = mock(IndexResponse.class); + when(indexResponse.getId()).thenReturn("mockTaskId"); + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse(indexResponse); + return null; + }).when(mlTaskManager).createMLTask(any(MLTask.class), Mockito.isA(ActionListener.class)); + + when(mlModelManager.getModelInterface(any())).thenReturn(modelInterface); + + when(mlModelManager.getPredictor(anyString())).thenReturn(predictor); + when(mlModelManager.getWorkerNodes(anyString(), eq(FunctionName.REMOTE), eq(true))).thenReturn(new String[] { "node1" }); + taskRunner.dispatchTask(FunctionName.REMOTE, remoteInputRequest, transportService, listener); + verify(client, never()).get(any(), any()); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(MLTaskResponse.class); + verify(listener).onResponse(argumentCaptor.capture()); + } + + public void testValidateBatchPredictionFailure() throws IOException { + setupMocks(true, false, false, false); + RemoteInferenceInputDataSet remoteInferenceInputDataSet = RemoteInferenceInputDataSet + .builder() + .parameters( + Map + .of( + "messages", + "[{\\\"role\\\":\\\"system\\\",\\\"content\\\":\\\"You are a helpful assistant.\\\"}," + + "{\\\"role\\\":\\\"user\\\",\\\"content\\\":\\\"Hello!\\\"}]" + ) + ) + .actionType(ConnectorAction.ActionType.BATCH_PREDICT) + .build(); + MLPredictionTaskRequest remoteInputRequest = MLPredictionTaskRequest + .builder() + .modelId("test_model") + .mlInput(MLInput.builder().algorithm(FunctionName.REMOTE).inputDataset(remoteInferenceInputDataSet).build()) + .build(); + Predictable predictor = mock(Predictable.class); + when(predictor.isModelReady()).thenReturn(true); + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(1); + actionListener + .onResponse(MLTaskResponse.builder().output(ModelTensorOutput.builder().mlModelOutputs(List.of()).build()).build()); + return null; + }).when(predictor).asyncPredict(any(), any()); + + IndexResponse indexResponse = mock(IndexResponse.class); + when(indexResponse.getId()).thenReturn("mockTaskId"); + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(1); + listener.onResponse(indexResponse); + return null; + }).when(mlTaskManager).createMLTask(any(MLTask.class), Mockito.isA(ActionListener.class)); + + when(mlModelManager.getPredictor(anyString())).thenReturn(predictor); + when(mlModelManager.getWorkerNodes(anyString(), eq(FunctionName.REMOTE), eq(true))).thenReturn(new String[] { "node1" }); + taskRunner.dispatchTask(FunctionName.REMOTE, remoteInputRequest, transportService, listener); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(listener).onFailure(argumentCaptor.capture()); + assertEquals("Unable to create batch transform job", argumentCaptor.getValue().getMessage()); + } + public void testValidateModelTensorOutputFailed() { exceptionRule.expect(OpenSearchStatusException.class); ModelTensor modelTensor = ModelTensor From 5f61df14bbd8c0ce1eeffbae96196d6c01527a26 Mon Sep 17 00:00:00 2001 From: Bhavana Ramaram Date: Thu, 5 Sep 2024 12:35:52 -0500 Subject: [PATCH 11/23] output only old fields in get config API (#2892) Signed-off-by: Bhavana Ramaram --- .../org/opensearch/ml/common/MLConfig.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/MLConfig.java b/common/src/main/java/org/opensearch/ml/common/MLConfig.java index 8204174663..20bc1853d7 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLConfig.java +++ b/common/src/main/java/org/opensearch/ml/common/MLConfig.java @@ -121,26 +121,17 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { XContentBuilder builder = xContentBuilder.startObject(); - if (type != null) { - builder.field(TYPE_FIELD, type); + if (configType != null || type != null) { + builder.field(TYPE_FIELD, configType == null ? type : configType); } - if (configType != null) { - builder.field(CONFIG_TYPE_FIELD, configType); - } - if (configuration != null) { - builder.field(CONFIGURATION_FIELD, configuration); - } - if (mlConfiguration != null) { - builder.field(ML_CONFIGURATION_FIELD, mlConfiguration); + if (configuration != null || mlConfiguration != null) { + builder.field(CONFIGURATION_FIELD, mlConfiguration == null ? configuration : mlConfiguration); } if (createTime != null) { builder.field(CREATE_TIME_FIELD, createTime.toEpochMilli()); } - if (lastUpdateTime != null) { - builder.field(LAST_UPDATE_TIME_FIELD, lastUpdateTime.toEpochMilli()); - } - if (lastUpdatedTime != null) { - builder.field(LAST_UPDATED_TIME_FIELD, lastUpdatedTime.toEpochMilli()); + if (lastUpdateTime != null || lastUpdatedTime != null) { + builder.field(LAST_UPDATE_TIME_FIELD, lastUpdatedTime == null ? lastUpdateTime.toEpochMilli() : lastUpdatedTime.toEpochMilli()); } return builder.endObject(); } From eca963f6040a255c37f700ac673942471c83e1cd Mon Sep 17 00:00:00 2001 From: Xun Zhang Date: Thu, 5 Sep 2024 16:58:21 -0700 Subject: [PATCH 12/23] add 2.17 release note (#2902) * add 2.17 release note Signed-off-by: Xun Zhang * remove reverted commit Signed-off-by: Xun Zhang --------- Signed-off-by: Xun Zhang --- ...search-ml-common.release-notes-2.17.0.0.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 release-notes/opensearch-ml-common.release-notes-2.17.0.0.md diff --git a/release-notes/opensearch-ml-common.release-notes-2.17.0.0.md b/release-notes/opensearch-ml-common.release-notes-2.17.0.0.md new file mode 100644 index 0000000000..c71bad41d1 --- /dev/null +++ b/release-notes/opensearch-ml-common.release-notes-2.17.0.0.md @@ -0,0 +1,37 @@ +## Version 2.17.0.0 Release Notes + +Compatible with OpenSearch 2.17.0 + +### Features +* Offline batch ingestion API actions and data ingesters (#2844)[https://github.com/opensearch-project/ml-commons/pull/2844] +* Support get batch transform job status in get task API (#2825)[https://github.com/opensearch-project/ml-commons/pull/2825] + +### Enhancements +* Adding additional info for memory metadata (#2750)[https://github.com/opensearch-project/ml-commons/pull/2750] +* Support skip_validating_missing_parameters in connector (#2830)[https://github.com/opensearch-project/ml-commons/pull/2830] +* Support one_to_one in ML Inference Search Response Processor (#2801)[https://github.com/opensearch-project/ml-commons/pull/2801] +* Expose ML Config API (#2850)[https://github.com/opensearch-project/ml-commons/pull/2850] + +### Bug Fixes +* Fix delete local model twice quickly get 500 response issue (#2806)[https://github.com/opensearch-project/ml-commons/pull/2806] +* Fix cohere model input interface cannot validate cohere input issue (#2847)[https://github.com/opensearch-project/ml-commons/pull/2847] +* Add processed function for remote inference input dataset parameters to convert it back to its original datatype (#2852)[https://github.com/opensearch-project/ml-commons/pull/2852] +* Use local_regex as default type for guardrails (#2853)[https://github.com/opensearch-project/ml-commons/pull/2853] +* Agent execution error in json format (#2858)[https://github.com/opensearch-project/ml-commons/pull/2858] +* Fix custom prompt substitute with List issue in ml inference search response processor (#2871)[https://github.com/opensearch-project/ml-commons/pull/2871] +* Fix breaking changes in config index fields (#2882)[https://github.com/opensearch-project/ml-commons/pull/2882] +* Output only old fields in get config API (#2892)[https://github.com/opensearch-project/ml-commons/pull/2892] +* Fix http dependency in CancelBatchJobTransportAction (#2898)[https://github.com/opensearch-project/ml-commons/pull/2898] + +### Maintenance +* Applying spotless to common module (#2815)[https://github.com/opensearch-project/ml-commons/pull/2815] +* Fix Cohere test (#2831)[https://github.com/opensearch-project/ml-commons/pull/2831] + +### Infrastructure +* Test: recover search index tool it in multi node cluster (#2407)[https://github.com/opensearch-project/ml-commons/pull/2407] + +### Documentation +* Add tutorial for Bedrock Guardrails (#2695)[https://github.com/opensearch-project/ml-commons/pull/2695] + +### Refactoring +* Code refactor not to occur nullpointer exception (#2816)[https://github.com/opensearch-project/ml-commons/pull/2816] \ No newline at end of file From 17e81ae618bebc72b8d3cb76d57f1556b1d8c8e1 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Fri, 6 Sep 2024 09:30:33 -0700 Subject: [PATCH 13/23] Add support for Bedrock Converse API (Anthropic Messages API, Claude 3.5 Sonnet) (#2851) * Add support for Anthropic Message API (Issue 2826) Signed-off-by: Austin Lee * Fix a bug. Signed-off-by: Austin Lee * Add unit tests, improve coverage, clean up code. Signed-off-by: Austin Lee * Allow pdf and jpg files for IT tests for multimodel conversation API testing. Signed-off-by: Austin Lee * Fix spotless check issues. Signed-off-by: Austin Lee * Update IT to work with session tokens. Signed-off-by: Austin Lee * Fix MLRAGSearchProcessorIT not to extend RestMLRemoteInferenceIT. Signed-off-by: Austin Lee * Use suite specific model group name. Signed-off-by: Austin Lee * Disable tests that require futher investigation. Signed-off-by: Austin Lee * Skip two additional tests with time-outs. Signed-off-by: Austin Lee * Restore a change from RestMLRemoteInferenceIT. Signed-off-by: Austin Lee --------- Signed-off-by: Austin Lee --- plugin/build.gradle | 5 + .../ml/rest/RestMLRAGSearchProcessorIT.java | 701 +++++++++++++++++- .../opensearch/ml/rest/test_data/lincoln.pdf | Bin 0 -> 65220 bytes .../ml/rest/test_data/openai_boardwalk.jpg | Bin 0 -> 103016 bytes .../GenerativeQAResponseProcessor.java | 6 +- .../ext/GenerativeQAParameters.java | 48 +- .../generative/llm/ChatCompletionInput.java | 1 + .../generative/llm/DefaultLlmImpl.java | 34 +- .../questionanswering/generative/llm/Llm.java | 3 +- .../generative/llm/LlmIOUtil.java | 12 +- .../generative/llm/MessageBlock.java | 325 ++++++++ .../generative/prompt/PromptUtil.java | 365 ++++++++- .../ext/GenerativeQAParamExtBuilderTests.java | 16 +- .../ext/GenerativeQAParametersTests.java | 45 +- .../llm/ChatCompletionInputTests.java | 2 + .../generative/llm/DefaultLlmImplTests.java | 70 +- .../generative/llm/MessageBlockTests.java | 103 +++ .../generative/prompt/PromptUtilTests.java | 185 ++++- 18 files changed, 1869 insertions(+), 52 deletions(-) create mode 100644 plugin/src/test/resources/org/opensearch/ml/rest/test_data/lincoln.pdf create mode 100644 plugin/src/test/resources/org/opensearch/ml/rest/test_data/openai_boardwalk.jpg create mode 100644 search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlock.java create mode 100644 search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlockTests.java diff --git a/plugin/build.gradle b/plugin/build.gradle index d9f97cf4cf..bca23cf2e5 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -578,3 +578,8 @@ task bwcTestSuite(type: StandaloneRestIntegTestTask) { dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") dependsOn tasks.named("${baseName}#fullRestartClusterTask") } + +forbiddenPatterns { + exclude '**/*.pdf' + exclude '**/*.jpg' +} diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java index a7e3b9932a..e8154f4c2d 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java @@ -17,19 +17,24 @@ */ package org.opensearch.ml.rest; +import static org.opensearch.ml.rest.RestMLRemoteInferenceIT.createConnector; +import static org.opensearch.ml.rest.RestMLRemoteInferenceIT.deployRemoteModel; import static org.opensearch.ml.utils.TestHelper.makeRequest; import static org.opensearch.ml.utils.TestHelper.toHttpEntity; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Base64; import java.util.Locale; import java.util.Map; import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Before; +import org.junit.Ignore; import org.opensearch.client.Response; import org.opensearch.core.rest.RestStatus; import org.opensearch.ml.common.MLTaskState; @@ -39,7 +44,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { +public class RestMLRAGSearchProcessorIT extends MLCommonsRestTestCase { private static final String OPENAI_KEY = System.getenv("OPENAI_KEY"); private static final String OPENAI_CONNECTOR_BLUEPRINT = "{\n" @@ -70,11 +75,42 @@ public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { + " ]\n" + "}"; + private static final String OPENAI_4o_CONNECTOR_BLUEPRINT = "{\n" + + " \"name\": \"OpenAI Chat Connector\",\n" + + " \"description\": \"The connector to public OpenAI model service for GPT 3.5\",\n" + + " \"version\": 2,\n" + + " \"protocol\": \"http\",\n" + + " \"parameters\": {\n" + + " \"endpoint\": \"api.openai.com\",\n" + + " \"model\": \"gpt-4o-mini\",\n" + + " \"temperature\": 0\n" + + " },\n" + + " \"credential\": {\n" + + " \"openAI_key\": \"" + + OPENAI_KEY + + "\"\n" + + " },\n" + + " \"actions\": [\n" + + " {\n" + + " \"action_type\": \"predict\",\n" + + " \"method\": \"POST\",\n" + + " \"url\": \"https://${parameters.endpoint}/v1/chat/completions\",\n" + + " \"headers\": {\n" + + " \"Authorization\": \"Bearer ${credential.openAI_key}\"\n" + + " },\n" + + " \"request_body\": \"{ \\\"model\\\": \\\"${parameters.model}\\\", \\\"messages\\\": ${parameters.messages}, \\\"temperature\\\": ${parameters.temperature} , \\\"max_tokens\\\": 300 }\"\n" + + " }\n" + + " ]\n" + + "}"; + private static final String AWS_ACCESS_KEY_ID = System.getenv("AWS_ACCESS_KEY_ID"); private static final String AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY"); private static final String AWS_SESSION_TOKEN = System.getenv("AWS_SESSION_TOKEN"); private static final String GITHUB_CI_AWS_REGION = "us-west-2"; + private static final String BEDROCK_ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0"; + private static final String BEDROCK_ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0"; + private static final String BEDROCK_CONNECTOR_BLUEPRINT1 = "{\n" + " \"name\": \"Bedrock Connector: claude2\",\n" + " \"description\": \"The connector to bedrock claude2 model\",\n" @@ -145,10 +181,100 @@ public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { + " ]\n" + "}"; + private static final String BEDROCK_CONVERSE_CONNECTOR_BLUEPRINT2 = "{\n" + + " \"name\": \"Bedrock Connector: claude 3.5\",\n" + + " \"description\": \"The connector to bedrock claude 3.5 model\",\n" + + " \"version\": 1,\n" + + " \"protocol\": \"aws_sigv4\",\n" + + " \"parameters\": {\n" + + " \"region\": \"" + + GITHUB_CI_AWS_REGION + + "\",\n" + + " \"service_name\": \"bedrock\",\n" + + " \"model\": \"" + + BEDROCK_ANTHROPIC_CLAUDE_3_5_SONNET + + "\",\n" + + " \"system_prompt\": \"You are a helpful assistant.\"\n" + + " },\n" + + " \"credential\": {\n" + + " \"access_key\": \"" + + AWS_ACCESS_KEY_ID + + "\",\n" + + " \"secret_key\": \"" + + AWS_SECRET_ACCESS_KEY + + "\",\n" + + " \"session_token\": \"" + + AWS_SESSION_TOKEN + + "\"\n" + + " },\n" + + " \"actions\": [\n" + + " {\n" + + " \"action_type\": \"predict\",\n" + + " \"method\": \"POST\",\n" + + " \"headers\": {\n" + + " \"content-type\": \"application/json\"\n" + + " },\n" + + " \"url\": \"https://bedrock-runtime." + + GITHUB_CI_AWS_REGION + + ".amazonaws.com/model/" + + BEDROCK_ANTHROPIC_CLAUDE_3_5_SONNET + + "/converse\",\n" + + " \"request_body\": \"{ \\\"system\\\": [{\\\"text\\\": \\\"you are a helpful assistant.\\\"}], \\\"messages\\\": ${parameters.messages} , \\\"inferenceConfig\\\": {\\\"temperature\\\": 0.0, \\\"topP\\\": 0.9, \\\"maxTokens\\\": 1000} }\"\n" + + " }\n" + + " ]\n" + + "}"; + + private static final String BEDROCK_DOCUMENT_CONVERSE_CONNECTOR_BLUEPRINT2 = "{\n" + + " \"name\": \"Bedrock Connector: claude 3\",\n" + + " \"description\": \"The connector to bedrock claude 3 model\",\n" + + " \"version\": 1,\n" + + " \"protocol\": \"aws_sigv4\",\n" + + " \"parameters\": {\n" + + " \"region\": \"" + + GITHUB_CI_AWS_REGION + + "\",\n" + + " \"service_name\": \"bedrock\",\n" + + " \"model\": \"" + + BEDROCK_ANTHROPIC_CLAUDE_3_SONNET + + "\",\n" + + " \"system_prompt\": \"You are a helpful assistant.\"\n" + + " },\n" + + " \"credential\": {\n" + + " \"access_key\": \"" + + AWS_ACCESS_KEY_ID + + "\",\n" + + " \"secret_key\": \"" + + AWS_SECRET_ACCESS_KEY + + "\",\n" + + " \"session_token\": \"" + + AWS_SESSION_TOKEN + + "\"\n" + + " },\n" + + " \"actions\": [\n" + + " {\n" + + " \"action_type\": \"predict\",\n" + + " \"method\": \"POST\",\n" + + " \"headers\": {\n" + + " \"content-type\": \"application/json\"\n" + + " },\n" + + " \"url\": \"https://bedrock-runtime." + + GITHUB_CI_AWS_REGION + + ".amazonaws.com/model/" + + BEDROCK_ANTHROPIC_CLAUDE_3_SONNET + + "/converse\",\n" + + " \"request_body\": \"{ \\\"messages\\\": ${parameters.messages} , \\\"inferenceConfig\\\": {\\\"temperature\\\": 0.0, \\\"topP\\\": 0.9, \\\"maxTokens\\\": 1000} }\"\n" + + " }\n" + + " ]\n" + + "}"; + private static final String BEDROCK_CONNECTOR_BLUEPRINT = AWS_SESSION_TOKEN == null ? BEDROCK_CONNECTOR_BLUEPRINT2 : BEDROCK_CONNECTOR_BLUEPRINT1; + private static final String BEDROCK_CONVERSE_CONNECTOR_BLUEPRINT = AWS_SESSION_TOKEN == null + ? BEDROCK_CONVERSE_CONNECTOR_BLUEPRINT2 + : BEDROCK_CONVERSE_CONNECTOR_BLUEPRINT2; + private static final String COHERE_KEY = System.getenv("COHERE_KEY"); private static final String COHERE_CONNECTOR_BLUEPRINT = "{\n" + " \"name\": \"Cohere Chat Model\",\n" @@ -192,6 +318,22 @@ public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { + " ]\n" + "}"; + // In some cases, we do not want a system prompt to be sent to an LLM. + private static final String PIPELINE_TEMPLATE2 = "{\n" + + " \"response_processors\": [\n" + + " {\n" + + " \"retrieval_augmented_generation\": {\n" + + " \"tag\": \"%s\",\n" + + " \"description\": \"%s\",\n" + + " \"model_id\": \"%s\",\n" + // + " \"system_prompt\": \"%s\",\n" + + " \"user_instructions\": \"%s\",\n" + + " \"context_field_list\": [\"%s\"]\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"; + private static final String BM25_SEARCH_REQUEST_TEMPLATE = "{\n" + " \"_source\": [\"%s\"],\n" + " \"query\" : {\n" @@ -210,6 +352,63 @@ public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { + " }\n" + "}"; + private static final String BM25_SEARCH_REQUEST_WITH_IMAGE_TEMPLATE = "{\n" + + " \"_source\": [\"%s\"],\n" + + " \"query\" : {\n" + + " \"match\": {\"%s\": \"%s\"}\n" + + " },\n" + + " \"ext\": {\n" + + " \"generative_qa_parameters\": {\n" + + " \"llm_model\": \"%s\",\n" + + " \"llm_question\": \"%s\",\n" + + " \"system_prompt\": \"%s\",\n" + + " \"user_instructions\": \"%s\",\n" + + " \"context_size\": %d,\n" + + " \"message_size\": %d,\n" + + " \"timeout\": %d,\n" + + " \"llm_messages\": [{ \"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"%s\"}, {\"image\": {\"format\": \"%s\", \"%s\": \"%s\"}}] }]\n" + + " }\n" + + " }\n" + + "}"; + + private static final String BM25_SEARCH_REQUEST_WITH_DOCUMENT_TEMPLATE = "{\n" + + " \"_source\": [\"%s\"],\n" + + " \"query\" : {\n" + + " \"match\": {\"%s\": \"%s\"}\n" + + " },\n" + + " \"ext\": {\n" + + " \"generative_qa_parameters\": {\n" + + " \"llm_model\": \"%s\",\n" + + " \"llm_question\": \"%s\",\n" + // + " \"system_prompt\": \"%s\",\n" + + " \"user_instructions\": \"%s\",\n" + + " \"context_size\": %d,\n" + + " \"message_size\": %d,\n" + + " \"timeout\": %d,\n" + + " \"llm_messages\": [{ \"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"%s\"}, {\"document\": {\"format\": \"%s\", \"name\": \"%s\", \"data\": \"%s\"}}] }]\n" + + " }\n" + + " }\n" + + "}"; + + private static final String BM25_SEARCH_REQUEST_WITH_IMAGE_AND_DOCUMENT_TEMPLATE = "{\n" + + " \"_source\": [\"%s\"],\n" + + " \"query\" : {\n" + + " \"match\": {\"%s\": \"%s\"}\n" + + " },\n" + + " \"ext\": {\n" + + " \"generative_qa_parameters\": {\n" + + " \"llm_model\": \"%s\",\n" + + " \"llm_question\": \"%s\",\n" + + " \"system_prompt\": \"%s\",\n" + + " \"user_instructions\": \"%s\",\n" + + " \"context_size\": %d,\n" + + " \"message_size\": %d,\n" + + " \"timeout\": %d,\n" + + " \"llm_messages\": [{ \"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"%s\"}, {\"image\": {\"format\": \"%s\", \"%s\": \"%s\"}} , {\"document\": {\"format\": \"%s\", \"name\": \"%s\", \"data\": \"%s\"}}] }]\n" + + " }\n" + + " }\n" + + "}"; + private static final String BM25_SEARCH_REQUEST_WITH_CONVO_TEMPLATE = "{\n" + " \"_source\": [\"%s\"],\n" + " \"query\" : {\n" @@ -229,6 +428,26 @@ public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { + " }\n" + "}"; + private static final String BM25_SEARCH_REQUEST_WITH_CONVO_AND_IMAGE_TEMPLATE = "{\n" + + " \"_source\": [\"%s\"],\n" + + " \"query\" : {\n" + + " \"match\": {\"%s\": \"%s\"}\n" + + " },\n" + + " \"ext\": {\n" + + " \"generative_qa_parameters\": {\n" + + " \"llm_model\": \"%s\",\n" + + " \"llm_question\": \"%s\",\n" + + " \"memory_id\": \"%s\",\n" + + " \"system_prompt\": \"%s\",\n" + + " \"user_instructions\": \"%s\",\n" + + " \"context_size\": %d,\n" + + " \"message_size\": %d,\n" + + " \"timeout\": %d,\n" + + " \"llm_messages\": [{ \"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"%s\"}, {\"image\": {\"format\": \"%s\", \"%s\": \"%s\"}}] }]\n" + + " }\n" + + " }\n" + + "}"; + private static final String BM25_SEARCH_REQUEST_WITH_LLM_RESPONSE_FIELD_TEMPLATE = "{\n" + " \"_source\": [\"%s\"],\n" + " \"query\" : {\n" @@ -247,18 +466,28 @@ public class RestMLRAGSearchProcessorIT extends RestMLRemoteInferenceIT { + "}"; private static final String OPENAI_MODEL = "gpt-3.5-turbo"; + private static final String OPENAI_40_MODEL = "gpt-4o-mini"; private static final String BEDROCK_ANTHROPIC_CLAUDE = "bedrock/anthropic-claude"; + private static final String BEDROCK_CONVERSE_ANTHROPIC_CLAUDE = "bedrock-converse/" + BEDROCK_ANTHROPIC_CLAUDE_3_5_SONNET; + private static final String BEDROCK_CONVERSE_ANTHROPIC_CLAUDE_3 = "bedrock-converse/" + BEDROCK_ANTHROPIC_CLAUDE_3_SONNET; private static final String TEST_DOC_PATH = "org/opensearch/ml/rest/test_data/"; private static Set testDocs = Set.of("qa_doc1.json", "qa_doc2.json", "qa_doc3.json"); private static final String DEFAULT_USER_AGENT = "Kibana"; protected ClassLoader classLoader = RestMLRAGSearchProcessorIT.class.getClassLoader(); private static final String INDEX_NAME = "test"; + private static final String ML_RAG_REMOTE_MODEL_GROUP = "rag_remote_model_group"; + // "client" gets initialized by the test framework at the instance level // so we perform this per test case, not via @BeforeClass. @Before public void init() throws Exception { + RestMLRemoteInferenceIT.disableClusterConnectorAccessControl(); + // TODO Do we really need to wait this long? This adds 20s to every test case run. + // Can we instead check the cluster state and move on? + Thread.sleep(20000); + Response response = TestHelper .makeRequest( client(), @@ -307,11 +536,11 @@ public void testBM25WithOpenAI() throws Exception { Response response = createConnector(OPENAI_CONNECTOR_BLUEPRINT); Map responseMap = parseResponseToMap(response); String connectorId = (String) responseMap.get("connector_id"); - response = registerRemoteModel("openAI-GPT-3.5 completions", connectorId); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "openAI-GPT-3.5 completions", connectorId); responseMap = parseResponseToMap(response); String taskId = (String) responseMap.get("task_id"); waitForTask(taskId, MLTaskState.COMPLETED); - response = getTask(taskId); + response = RestMLRemoteInferenceIT.getTask(taskId); responseMap = parseResponseToMap(response); String modelId = (String) responseMap.get("model_id"); response = deployRemoteModel(modelId); @@ -353,6 +582,94 @@ public void testBM25WithOpenAI() throws Exception { assertNotNull(answer); } + @Ignore + public void testBM25WithOpenAIWithImage() throws Exception { + // Skip test if key is null + if (OPENAI_KEY == null) { + return; + } + Response response = createConnector(OPENAI_4o_CONNECTOR_BLUEPRINT); + Map responseMap = parseResponseToMap(response); + String connectorId = (String) responseMap.get("connector_id"); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "openAI-GPT-4o-mini completions", connectorId); + responseMap = parseResponseToMap(response); + String taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + response = RestMLRemoteInferenceIT.getTask(taskId); + responseMap = parseResponseToMap(response); + String modelId = (String) responseMap.get("model_id"); + response = deployRemoteModel(modelId); + responseMap = parseResponseToMap(response); + taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + + PipelineParameters pipelineParameters = new PipelineParameters(); + pipelineParameters.tag = "testBM25WithOpenAIWithImage"; + pipelineParameters.description = "desc"; + pipelineParameters.modelId = modelId; + pipelineParameters.systemPrompt = "You are a helpful assistant"; + pipelineParameters.userInstructions = "none"; + pipelineParameters.context_field = "text"; + Response response1 = createSearchPipeline("pipeline_test", pipelineParameters); + assertEquals(200, response1.getStatusLine().getStatusCode()); + + byte[] rawImage = FileUtils + .readFileToByteArray(Path.of(classLoader.getResource(TEST_DOC_PATH + "openai_boardwalk.jpg").toURI()).toFile()); + String imageContent = Base64.getEncoder().encodeToString(rawImage); + + SearchRequestParameters requestParameters = new SearchRequestParameters(); + requestParameters.source = "text"; + requestParameters.match = "president"; + requestParameters.llmModel = OPENAI_40_MODEL; + requestParameters.llmQuestion = "what is this image"; + requestParameters.systemPrompt = "You are great at answering questions"; + requestParameters.userInstructions = "Follow my instructions as best you can"; + requestParameters.contextSize = 5; + requestParameters.interactionSize = 5; + requestParameters.timeout = 60; + requestParameters.imageFormat = "jpeg"; + requestParameters.imageType = "data"; + requestParameters.imageData = imageContent; + Response response2 = performSearch(INDEX_NAME, "pipeline_test", 5, requestParameters); + assertEquals(200, response2.getStatusLine().getStatusCode()); + + Map responseMap2 = parseResponseToMap(response2); + Map ext = (Map) responseMap2.get("ext"); + assertNotNull(ext); + Map rag = (Map) ext.get("retrieval_augmented_generation"); + assertNotNull(rag); + + // TODO handle errors such as throttling + String answer = (String) rag.get("answer"); + assertNotNull(answer); + + requestParameters = new SearchRequestParameters(); + requestParameters.source = "text"; + requestParameters.match = "president"; + requestParameters.llmModel = OPENAI_40_MODEL; + requestParameters.llmQuestion = "what is this image"; + requestParameters.systemPrompt = "You are great at answering questions"; + requestParameters.userInstructions = "Follow my instructions as best you can"; + requestParameters.contextSize = 5; + requestParameters.interactionSize = 5; + requestParameters.timeout = 60; + requestParameters.imageFormat = "jpeg"; + requestParameters.imageType = "url"; + requestParameters.imageData = + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"; // imageContent; + Response response3 = performSearch(INDEX_NAME, "pipeline_test", 5, requestParameters); + assertEquals(200, response2.getStatusLine().getStatusCode()); + + Map responseMap3 = parseResponseToMap(response3); + ext = (Map) responseMap2.get("ext"); + assertNotNull(ext); + rag = (Map) ext.get("retrieval_augmented_generation"); + assertNotNull(rag); + + answer = (String) rag.get("answer"); + assertNotNull(answer); + } + public void testBM25WithBedrock() throws Exception { // Skip test if key is null if (AWS_ACCESS_KEY_ID == null) { @@ -361,11 +678,11 @@ public void testBM25WithBedrock() throws Exception { Response response = createConnector(BEDROCK_CONNECTOR_BLUEPRINT); Map responseMap = parseResponseToMap(response); String connectorId = (String) responseMap.get("connector_id"); - response = registerRemoteModel("Bedrock Anthropic Claude", connectorId); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Bedrock Anthropic Claude", connectorId); responseMap = parseResponseToMap(response); String taskId = (String) responseMap.get("task_id"); waitForTask(taskId, MLTaskState.COMPLETED); - response = getTask(taskId); + response = RestMLRemoteInferenceIT.getTask(taskId); responseMap = parseResponseToMap(response); String modelId = (String) responseMap.get("model_id"); response = deployRemoteModel(modelId); @@ -374,7 +691,7 @@ public void testBM25WithBedrock() throws Exception { waitForTask(taskId, MLTaskState.COMPLETED); PipelineParameters pipelineParameters = new PipelineParameters(); - pipelineParameters.tag = "testBM25WithOpenAI"; + pipelineParameters.tag = "testBM25WithBedrock"; pipelineParameters.description = "desc"; pipelineParameters.modelId = modelId; pipelineParameters.systemPrompt = "You are a helpful assistant"; @@ -405,6 +722,180 @@ public void testBM25WithBedrock() throws Exception { assertNotNull(answer); } + @Ignore + public void testBM25WithBedrockConverse() throws Exception { + // Skip test if key is null + if (AWS_ACCESS_KEY_ID == null) { + return; + } + Response response = createConnector(BEDROCK_CONVERSE_CONNECTOR_BLUEPRINT); + Map responseMap = parseResponseToMap(response); + String connectorId = (String) responseMap.get("connector_id"); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Bedrock Anthropic Claude", connectorId); + responseMap = parseResponseToMap(response); + String taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + response = RestMLRemoteInferenceIT.getTask(taskId); + responseMap = parseResponseToMap(response); + String modelId = (String) responseMap.get("model_id"); + response = deployRemoteModel(modelId); + responseMap = parseResponseToMap(response); + taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + + PipelineParameters pipelineParameters = new PipelineParameters(); + pipelineParameters.tag = "testBM25WithBedrockConverse"; + pipelineParameters.description = "desc"; + pipelineParameters.modelId = modelId; + pipelineParameters.systemPrompt = "You are a helpful assistant"; + pipelineParameters.userInstructions = "none"; + pipelineParameters.context_field = "text"; + Response response1 = createSearchPipeline("pipeline_test", pipelineParameters); + assertEquals(200, response1.getStatusLine().getStatusCode()); + + SearchRequestParameters requestParameters = new SearchRequestParameters(); + requestParameters.source = "text"; + requestParameters.match = "president"; + requestParameters.llmModel = BEDROCK_CONVERSE_ANTHROPIC_CLAUDE; + requestParameters.llmQuestion = "who is lincoln"; + requestParameters.contextSize = 5; + requestParameters.interactionSize = 5; + requestParameters.timeout = 60; + Response response2 = performSearch(INDEX_NAME, "pipeline_test", 5, requestParameters); + assertEquals(200, response2.getStatusLine().getStatusCode()); + + Map responseMap2 = parseResponseToMap(response2); + Map ext = (Map) responseMap2.get("ext"); + assertNotNull(ext); + Map rag = (Map) ext.get("retrieval_augmented_generation"); + assertNotNull(rag); + + // TODO handle errors such as throttling + String answer = (String) rag.get("answer"); + assertNotNull(answer); + } + + @Ignore + public void testBM25WithBedrockConverseUsingLlmMessages() throws Exception { + // Skip test if key is null + if (AWS_ACCESS_KEY_ID == null) { + return; + } + Response response = createConnector(BEDROCK_CONVERSE_CONNECTOR_BLUEPRINT2); + Map responseMap = parseResponseToMap(response); + String connectorId = (String) responseMap.get("connector_id"); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Bedrock Anthropic Claude", connectorId); + responseMap = parseResponseToMap(response); + String taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + response = RestMLRemoteInferenceIT.getTask(taskId); + responseMap = parseResponseToMap(response); + String modelId = (String) responseMap.get("model_id"); + response = deployRemoteModel(modelId); + responseMap = parseResponseToMap(response); + taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + + PipelineParameters pipelineParameters = new PipelineParameters(); + pipelineParameters.tag = "testBM25WithBedrockConverseUsingLlmMessages"; + pipelineParameters.description = "desc"; + pipelineParameters.modelId = modelId; + pipelineParameters.systemPrompt = "You are a helpful assistant"; + pipelineParameters.userInstructions = "none"; + pipelineParameters.context_field = "text"; + Response response1 = createSearchPipeline("pipeline_test", pipelineParameters); + assertEquals(200, response1.getStatusLine().getStatusCode()); + + byte[] rawImage = FileUtils + .readFileToByteArray(Path.of(classLoader.getResource(TEST_DOC_PATH + "openai_boardwalk.jpg").toURI()).toFile()); + String imageContent = Base64.getEncoder().encodeToString(rawImage); + + SearchRequestParameters requestParameters = new SearchRequestParameters(); + + requestParameters.source = "text"; + requestParameters.match = "president"; + requestParameters.llmModel = BEDROCK_CONVERSE_ANTHROPIC_CLAUDE; + requestParameters.llmQuestion = "describe the image and answer the question: would lincoln have liked this place"; + requestParameters.contextSize = 5; + requestParameters.interactionSize = 5; + requestParameters.timeout = 60; + requestParameters.imageFormat = "jpeg"; + requestParameters.imageType = "data"; // Bedrock does not support URLs + requestParameters.imageData = imageContent; + Response response2 = performSearch(INDEX_NAME, "pipeline_test", 5, requestParameters); + assertEquals(200, response2.getStatusLine().getStatusCode()); + + Map responseMap2 = parseResponseToMap(response2); + Map ext = (Map) responseMap2.get("ext"); + assertNotNull(ext); + Map rag = (Map) ext.get("retrieval_augmented_generation"); + assertNotNull(rag); + + // TODO handle errors such as throttling + String answer = (String) rag.get("answer"); + assertNotNull(answer); + } + + @Ignore + public void testBM25WithBedrockConverseUsingLlmMessagesForDocumentChat() throws Exception { + // Skip test if key is null + if (AWS_ACCESS_KEY_ID == null) { + return; + } + Response response = createConnector(BEDROCK_DOCUMENT_CONVERSE_CONNECTOR_BLUEPRINT2); + Map responseMap = parseResponseToMap(response); + String connectorId = (String) responseMap.get("connector_id"); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Bedrock Anthropic Claude", connectorId); + responseMap = parseResponseToMap(response); + String taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + response = RestMLRemoteInferenceIT.getTask(taskId); + responseMap = parseResponseToMap(response); + String modelId = (String) responseMap.get("model_id"); + response = deployRemoteModel(modelId); + responseMap = parseResponseToMap(response); + taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + + PipelineParameters pipelineParameters = new PipelineParameters(); + pipelineParameters.tag = "testBM25WithBedrockConverseUsingLlmMessagesForDocumentChat"; + pipelineParameters.description = "desc"; + pipelineParameters.modelId = modelId; + // pipelineParameters.systemPrompt = "You are a helpful assistant"; + pipelineParameters.userInstructions = "none"; + pipelineParameters.context_field = "text"; + Response response1 = createSearchPipeline2("pipeline_test", pipelineParameters); + assertEquals(200, response1.getStatusLine().getStatusCode()); + + byte[] docBytes = FileUtils.readFileToByteArray(Path.of(classLoader.getResource(TEST_DOC_PATH + "lincoln.pdf").toURI()).toFile()); + String docContent = Base64.getEncoder().encodeToString(docBytes); + + SearchRequestParameters requestParameters; + requestParameters = new SearchRequestParameters(); + requestParameters.source = "text"; + requestParameters.match = "president"; + requestParameters.llmModel = BEDROCK_CONVERSE_ANTHROPIC_CLAUDE_3; + requestParameters.llmQuestion = "use the information from the attached document to tell me something interesting about lincoln"; + requestParameters.contextSize = 5; + requestParameters.interactionSize = 5; + requestParameters.timeout = 60; + requestParameters.documentFormat = "pdf"; + requestParameters.documentName = "lincoln"; + requestParameters.documentData = docContent; + Response response3 = performSearch(INDEX_NAME, "pipeline_test", 5, requestParameters); + assertEquals(200, response3.getStatusLine().getStatusCode()); + + Map responseMap3 = parseResponseToMap(response3); + Map ext = (Map) responseMap3.get("ext"); + assertNotNull(ext); + Map rag = (Map) ext.get("retrieval_augmented_generation"); + assertNotNull(rag); + + // TODO handle errors such as throttling + String answer = (String) rag.get("answer"); + assertNotNull(answer); + } + public void testBM25WithOpenAIWithConversation() throws Exception { // Skip test if key is null if (OPENAI_KEY == null) { @@ -413,11 +904,11 @@ public void testBM25WithOpenAIWithConversation() throws Exception { Response response = createConnector(OPENAI_CONNECTOR_BLUEPRINT); Map responseMap = parseResponseToMap(response); String connectorId = (String) responseMap.get("connector_id"); - response = registerRemoteModel("openAI-GPT-3.5 completions", connectorId); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "openAI-GPT-3.5 completions", connectorId); responseMap = parseResponseToMap(response); String taskId = (String) responseMap.get("task_id"); waitForTask(taskId, MLTaskState.COMPLETED); - response = getTask(taskId); + response = RestMLRemoteInferenceIT.getTask(taskId); responseMap = parseResponseToMap(response); String modelId = (String) responseMap.get("model_id"); response = deployRemoteModel(modelId); @@ -426,7 +917,7 @@ public void testBM25WithOpenAIWithConversation() throws Exception { waitForTask(taskId, MLTaskState.COMPLETED); PipelineParameters pipelineParameters = new PipelineParameters(); - pipelineParameters.tag = "testBM25WithOpenAI"; + pipelineParameters.tag = "testBM25WithOpenAIWithConversation"; pipelineParameters.description = "desc"; pipelineParameters.modelId = modelId; pipelineParameters.systemPrompt = "You are a helpful assistant"; @@ -462,6 +953,68 @@ public void testBM25WithOpenAIWithConversation() throws Exception { assertNotNull(interactionId); } + @Ignore + public void testBM25WithOpenAIWithConversationAndImage() throws Exception { + // Skip test if key is null + if (OPENAI_KEY == null) { + return; + } + Response response = createConnector(OPENAI_4o_CONNECTOR_BLUEPRINT); + Map responseMap = parseResponseToMap(response); + String connectorId = (String) responseMap.get("connector_id"); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "openAI-GPT-4 completions", connectorId); + responseMap = parseResponseToMap(response); + String taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + response = RestMLRemoteInferenceIT.getTask(taskId); + responseMap = parseResponseToMap(response); + String modelId = (String) responseMap.get("model_id"); + response = deployRemoteModel(modelId); + responseMap = parseResponseToMap(response); + taskId = (String) responseMap.get("task_id"); + waitForTask(taskId, MLTaskState.COMPLETED); + + PipelineParameters pipelineParameters = new PipelineParameters(); + pipelineParameters.tag = "testBM25WithOpenAIWithConversationAndImage"; + pipelineParameters.description = "desc"; + pipelineParameters.modelId = modelId; + pipelineParameters.systemPrompt = "You are a helpful assistant"; + pipelineParameters.userInstructions = "none"; + pipelineParameters.context_field = "text"; + Response response1 = createSearchPipeline("pipeline_test", pipelineParameters); + assertEquals(200, response1.getStatusLine().getStatusCode()); + + String conversationId = createConversation("test_convo_1"); + SearchRequestParameters requestParameters = new SearchRequestParameters(); + requestParameters.source = "text"; + requestParameters.match = "president"; + requestParameters.llmModel = OPENAI_40_MODEL; + requestParameters.llmQuestion = "describe the image and answer the question: can you picture lincoln enjoying himself there"; + requestParameters.contextSize = 5; + requestParameters.interactionSize = 5; + requestParameters.timeout = 60; + requestParameters.conversationId = conversationId; + requestParameters.imageFormat = "jpeg"; + requestParameters.imageType = "url"; + requestParameters.imageData = + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"; + Response response2 = performSearch(INDEX_NAME, "pipeline_test", 5, requestParameters); + assertEquals(200, response2.getStatusLine().getStatusCode()); + + Map responseMap2 = parseResponseToMap(response2); + Map ext = (Map) responseMap2.get("ext"); + assertNotNull(ext); + Map rag = (Map) ext.get("retrieval_augmented_generation"); + assertNotNull(rag); + + // TODO handle errors such as throttling + String answer = (String) rag.get("answer"); + assertNotNull(answer); + + String interactionId = (String) rag.get("message_id"); + assertNotNull(interactionId); + } + public void testBM25WithBedrockWithConversation() throws Exception { // Skip test if key is null if (AWS_ACCESS_KEY_ID == null) { @@ -470,11 +1023,11 @@ public void testBM25WithBedrockWithConversation() throws Exception { Response response = createConnector(BEDROCK_CONNECTOR_BLUEPRINT); Map responseMap = parseResponseToMap(response); String connectorId = (String) responseMap.get("connector_id"); - response = registerRemoteModel("Bedrock", connectorId); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Bedrock", connectorId); responseMap = parseResponseToMap(response); String taskId = (String) responseMap.get("task_id"); waitForTask(taskId, MLTaskState.COMPLETED); - response = getTask(taskId); + response = RestMLRemoteInferenceIT.getTask(taskId); responseMap = parseResponseToMap(response); String modelId = (String) responseMap.get("model_id"); response = deployRemoteModel(modelId); @@ -483,7 +1036,7 @@ public void testBM25WithBedrockWithConversation() throws Exception { waitForTask(taskId, MLTaskState.COMPLETED); PipelineParameters pipelineParameters = new PipelineParameters(); - pipelineParameters.tag = "testBM25WithBedrock"; + pipelineParameters.tag = "testBM25WithBedrockWithConversation"; pipelineParameters.description = "desc"; pipelineParameters.modelId = modelId; pipelineParameters.systemPrompt = "You are a helpful assistant"; @@ -527,11 +1080,11 @@ public void testBM25WithCohere() throws Exception { Response response = createConnector(COHERE_CONNECTOR_BLUEPRINT); Map responseMap = parseResponseToMap(response); String connectorId = (String) responseMap.get("connector_id"); - response = registerRemoteModel("Cohere Chat Completion v1", connectorId); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Cohere Chat Completion v1", connectorId); responseMap = parseResponseToMap(response); String taskId = (String) responseMap.get("task_id"); waitForTask(taskId, MLTaskState.COMPLETED); - response = getTask(taskId); + response = RestMLRemoteInferenceIT.getTask(taskId); responseMap = parseResponseToMap(response); String modelId = (String) responseMap.get("model_id"); response = deployRemoteModel(modelId); @@ -579,11 +1132,11 @@ public void testBM25WithCohereUsingLlmResponseField() throws Exception { Response response = createConnector(COHERE_CONNECTOR_BLUEPRINT); Map responseMap = parseResponseToMap(response); String connectorId = (String) responseMap.get("connector_id"); - response = registerRemoteModel("Cohere Chat Completion v1", connectorId); + response = RestMLRemoteInferenceIT.registerRemoteModel(ML_RAG_REMOTE_MODEL_GROUP, "Cohere Chat Completion v1", connectorId); responseMap = parseResponseToMap(response); String taskId = (String) responseMap.get("task_id"); waitForTask(taskId, MLTaskState.COMPLETED); - response = getTask(taskId); + response = RestMLRemoteInferenceIT.getTask(taskId); responseMap = parseResponseToMap(response); String modelId = (String) responseMap.get("model_id"); response = deployRemoteModel(modelId); @@ -592,7 +1145,7 @@ public void testBM25WithCohereUsingLlmResponseField() throws Exception { waitForTask(taskId, MLTaskState.COMPLETED); PipelineParameters pipelineParameters = new PipelineParameters(); - pipelineParameters.tag = "testBM25WithCohereLlmResponseField"; + pipelineParameters.tag = "testBM25WithCohereUsingLlmResponseField"; pipelineParameters.description = "desc"; pipelineParameters.modelId = modelId; pipelineParameters.systemPrompt = "You are a helpful assistant"; @@ -647,9 +1200,33 @@ private Response createSearchPipeline(String pipeline, PipelineParameters parame ); } + // No system prompt + private Response createSearchPipeline2(String pipeline, PipelineParameters parameters) throws Exception { + return makeRequest( + client(), + "PUT", + String.format(Locale.ROOT, "/_search/pipeline/%s", pipeline), + null, + toHttpEntity( + String + .format( + Locale.ROOT, + PIPELINE_TEMPLATE2, + parameters.tag, + parameters.description, + parameters.modelId, + parameters.userInstructions, + parameters.context_field + ) + ), + ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, DEFAULT_USER_AGENT)) + ); + } + private Response performSearch(String indexName, String pipeline, int size, SearchRequestParameters requestParameters) throws Exception { + // TODO build these templates dynamically String httpEntity = requestParameters.llmResponseField != null ? String .format( @@ -665,6 +1242,90 @@ private Response performSearch(String indexName, String pipeline, int size, Sear requestParameters.timeout, requestParameters.llmResponseField ) + : (requestParameters.documentData != null && requestParameters.imageType != null) + ? String + .format( + Locale.ROOT, + BM25_SEARCH_REQUEST_WITH_IMAGE_AND_DOCUMENT_TEMPLATE, + requestParameters.source, + requestParameters.source, + requestParameters.match, + requestParameters.llmModel, + requestParameters.llmQuestion, + requestParameters.systemPrompt, + requestParameters.userInstructions, + requestParameters.contextSize, + requestParameters.interactionSize, + requestParameters.timeout, + requestParameters.llmQuestion, + requestParameters.imageFormat, + requestParameters.imageType, + requestParameters.imageData, + requestParameters.documentFormat, + requestParameters.documentName, + requestParameters.documentData + ) + : (requestParameters.documentData != null) + ? String + .format( + Locale.ROOT, + BM25_SEARCH_REQUEST_WITH_DOCUMENT_TEMPLATE, + requestParameters.source, + requestParameters.source, + requestParameters.match, + requestParameters.llmModel, + requestParameters.llmQuestion, + // requestParameters.systemPrompt, + requestParameters.userInstructions, + requestParameters.contextSize, + requestParameters.interactionSize, + requestParameters.timeout, + requestParameters.llmQuestion, + requestParameters.documentFormat, + requestParameters.documentName, + requestParameters.documentData + ) + : (requestParameters.conversationId != null && requestParameters.imageType != null) + ? String + .format( + Locale.ROOT, + BM25_SEARCH_REQUEST_WITH_CONVO_AND_IMAGE_TEMPLATE, + requestParameters.source, + requestParameters.source, + requestParameters.match, + requestParameters.llmModel, + requestParameters.llmQuestion, + requestParameters.conversationId, + requestParameters.systemPrompt, + requestParameters.userInstructions, + requestParameters.contextSize, + requestParameters.interactionSize, + requestParameters.timeout, + requestParameters.llmQuestion, + requestParameters.imageFormat, + requestParameters.imageType, + requestParameters.imageData + ) + : (requestParameters.imageType != null) + ? String + .format( + Locale.ROOT, + BM25_SEARCH_REQUEST_WITH_IMAGE_TEMPLATE, + requestParameters.source, + requestParameters.source, + requestParameters.match, + requestParameters.llmModel, + requestParameters.llmQuestion, + requestParameters.systemPrompt, + requestParameters.userInstructions, + requestParameters.contextSize, + requestParameters.interactionSize, + requestParameters.timeout, + requestParameters.llmQuestion, + requestParameters.imageFormat, + requestParameters.imageType, + requestParameters.imageData + ) : (requestParameters.conversationId == null) ? String .format( @@ -741,5 +1402,11 @@ static class SearchRequestParameters { String conversationId; String llmResponseField; + String imageFormat; + String imageType; + String imageData; + String documentFormat; + String documentName; + String documentData; } } diff --git a/plugin/src/test/resources/org/opensearch/ml/rest/test_data/lincoln.pdf b/plugin/src/test/resources/org/opensearch/ml/rest/test_data/lincoln.pdf new file mode 100644 index 0000000000000000000000000000000000000000..16eddb91fdd63f2c530868884c69dfd3fe68dc33 GIT binary patch literal 65220 zcmb?^2RzmL|9@6S3Xx5+M~<_c<5-!Y6haxv-ZNWviO3!iNr(tVLS%1QNk&=O$(||6 z?{m&^O84IH&A*eHf)EgljH`ZfWpeh>>Vwzyjbt!3 zFq^%R6`8m=o6u=X8)s8T;HQnDv#Fe^vAu~Yn~<`pow>6Gn=lvwXOom9b8>bxHMAvj z$16&H7d_Y*7r3}g6DdKBcREtlFo(;lp`hu64*5;fVp+QJ(g$Y_4HzNw_+KHmV|}iB z+(8{!**Q;pbg0O=gyl$@zuc7e_d>I@Z%*G`zB^SuUe(ln(%WhpSJLWIY4h^glSIUb zcZzS!$W+RXtN!GG_m#stL-s**pX7aF+~#Y)YfUT`e*Yvtkk{&>nYZ$JJ94wr?F$8u zP#nw2qLY0|PgpJ%tS3InrJZ-Y{B~y$Zdy33;Zo(`E?x5M^Ou(sW-4Eew0m8OI)q;j zPqL0JeQ~xloA3xiM{4q@9b(*VYt%MGp4Gta>&Qsck5); zSL^R1mu{$fPrbYOZ2p|xAqG5{fF^g>q*dVxQBwW~^SYy_rCIF0Ww_t5?2>;68-Bmj z`v^MFOm)7h3@PKefeez?qc`%{ikXVB$`3uW4OS|95@ zQm=}{Cts6zbFRH{jBV;l{arjM4Y{mSASd;$iVeCp_RNBT*c>zJu8)a3?IbPi6g%G~ zzioL}pIOd(P_QL`$=;=4>tP@N&|{ZnFX+|vlT0UydvDzYAx7DWN=Z*5=%bR4?exB6 z|1de?Z0?qqV<7I)c`yBI7QtDf{z+PRW+QoPa<5p&$^}`EX`#;gnp11!%M0tggDqrp z8Y)+Y3FRkmu2z@^nv{Q2s-e8IJpnt%AYFW1jw7w&j?Srr-W=P4mvV_v-9#4-R&}2l zbAC#ca1Qyv)3>frH!7pga5b4bkdA3Z%Gvqc^>7J|`R^T*0oiRuevcbyOje$W;JNwD zjKfyzrpnhl>cd&bwAFR9-ao#a`!bA+kF@)P32~37>Qb^sE$jnZW&``J#r=E{ zi$_*v9N(WPWg_LQ<`P(X`QaevyZ%){1H}BNuY_N!)oH5+eaaDTh75NeiAzDZt~o4G zoGndsDSdyk&stYZsnmp;2weSEb;akn9`p!Zt*pQQ3B`fxDSR(74UJa$QX4vNe+NL}^e%biiUAO3&2u8b$4_w<1 z^4Y-I(j5VV|hDv(y&LtyftP;J|zRlg-KNl9jOGDZzUF)Zx+;;}hB4 zc8?uPuQVKri5X6kZZl!7inEaF8!2TMh`dlmN`Fnjw?fQ_w{aS^0GoWS^W;@e?JJ>p zNbdQ-jCbm^5+jrSeE=|9B|99ka#7|{jiQKf9 z@V;vjVdd6FOchFI#Y23Ub)irzbGo$fdGKouYwLL1DXv>L&MarbNgo${a^MkgHE`UT z$@bQKnO|&FQGBs0wCkL|l->4~N}4mLhMqcfDu*}by-i6}$U+7y7+9jFNQVaV0fa1Z5>QHkEWpsx81B7Vq*1g8y?Z@x(^@MYt>E`C%hLSvVQ%^mv zR=&Maw>29vC9*;3LZzGDj8oF8Mx8C|~2G-xR@H z1M1^)abDdXC%%EZm~~Q!hg_|dSuYy%m?ypquK2Fxpq}}zM2^DovbArB{6oLN((whl z&=${t1Zotmf5$cbtT$VRgV3hYN0+`~WkbQ(IXC!)XBrap3?7_Qk8e*aI%l$G&EDSFUM_T1rT>BQO&QCrZit9vzu3FV zE0zk>p--}3G#+lcQUcX(BIv+-Pb!?w(;VWVFa~}hbzkp_(Az3!9r39f$PtOVk#U~d z(!Lj;oPAUEKng5)xqPH5->PzC5!rikJowdh-tpyE#2&YG+z*~lRk)MDZr3FGbJPT7Z?7tK_sy2U!5`H7i2~`a@_k@o%-`5N_sKlvdBSIgqMH_CtIy6a zYnGR7FMOTP8`bl_#!cY-+2u`Y#9c?r3f@b-Q3-LDsHhn_vZWD$&ve(P&)F??*iqfm zn_0Wu1895zu?N9P*@fU|4tGimOZQW&EeyHY7Y)` z)e>7UOKg9r#LKNQ;%5pjkw>2RO)h-lz24MUQ;xLAt+3WT;PNy-xp1zA>nLkqB9^2Ff+D2HXy2$VJ4-xe(rlSY2@AF zNP4ze@!}mPZBaPIjbiJ(vo_3$nyhyw8=MM7@IJg156m>6C2jXgVWE_;DJZWHrWS#8 zRq+bw-HEd$Vk#TUkiB?KU(>d-cE#&dl63kjd%m`|x4xwO%f{Zvrl?G~E3ahsD7N)H z=A&b>R$y!mI_RQb(zhv5XsDK#+cePMJL+#yHzU!Yt1=~Gvdq7h1)&B}9yO%Y{m}L8 zZnVOhLQ%55-Mp$5r=&{NqnVZ*%BO5vUBTn*qnt_?pT9Cel#$%OPc2L+F&1Fv)RyhY zW4Zi<(4IQ%x zh3GQ$DC#`jtq?iZ_5rOFi7?1c4e0Sm4D)*-CoK=ZFc+v`s4YJ|8++bMFRG{7x*|Te zy>1GZNq9531ttf;-6b~?TePSb!Jy&7^(B#}>e>@@x+qAmkopPvpvv!p489qM2o=J{ z^g^~(l(`uy=kmHuk<|Fd?>(aEMNGtkv{u&mI30L5^~nue1*wkXUzlRLmdH9q+YiCw( z;axI(f$REa;l&C!l#g93YCo~uer7~)IqS|1I*!taQaOR*@q73J4(Qo}l2OOlQM{B5JV0lit_rvu{k`C}1!{v$Sshb|C zAR|dtViSyvqplKaL1`_mMeh@~%MGEX2o`hm?%8dcnez70x6#A=VF!HNW!S#3KDFdd z$v*ZuVCGQxXhu6UnKD0)!nXBUqh)Xu-NRGXT;QWiNgCIhFAw{lxTXXW5U>NY49mQo zcofyvG{!&=fsx{ObKXAQDslWC)7v2{ah3(BL~0OmO@1p2Cl#TAw1|T++hXi?@p#a~ z=?8|M+C`;93>FUcOfBivYu1r+_Cl?y5+o#5F#-*l1?u(w=Q!fKg)Ny#c|N;NsekMZ z|3)G2X_c07^{U<0rw8|K%MhfXr(99lE4J43Cr(5w*3y!EEj}s}!Wwouy}$kBC;anQ z9|}GVx=&HG8aw`F7$xZ+MjXD$k@2BlnqiQakwiK>VU2U$$jh;izLgO2RdGZ|p)4;c zbMEW;y9M`O^i5N9^sDm_x~8j$P^!P;ImjM3xtx|Sb}>n1R&b7aJ%0w-^y!jJ;f;>$ z(6Q-?jQ6OQ{;Q`4`x`$vguk&AA3PLx{+8Fb$eqpJ@Zin9j|zo-&a*Z5rujut)Md5b zgAdwr#%soL>dfHd zjF~P)GdJ&wyh7dze<+srM!k^1LPEscZf?~>?Cbl}0Yu{38*A^f6qFiu%b)l>kJgdQ z4wH)zPG?zbfM=H-HZYd@AXtC%dlCML^pt)gH<}I0DTDkGWl3?kKe&oEbj@T#)Gm-v z19s6kbFn&*wLTN#2fN*wF0U5^A$xYfNRgA3Lm-mNO7&pcXAp|e5_~oc$^JC_ARqNz zil-d|>>7U1hwI-bPUwQN>#XLqwx8`tlc*aQS-lT`MnfnX#dv@^dHAzNywu$wF?g{J zXO2cbcgJdhy8=Otf;8C{9}76p;GuP2#M*OM*g>hR6}m0;!8;F2mb-$&O!T<+v; z{5Y}Rku3U&l5CM?E~Yk1=N;`O+xb$|o#V4HYy}5t8(IUoE$8;#`D6$=mCw#4Q2OwmQnl%ACVf+os47mQR|zw+_f{7n(g;4 z)-h5A#Ydj|a(1*a)vY-s{@NG2i*RC&$f)7Nk*97&#F(p@i~9_v`l?-cT|CqAu=2^R zi!{V|Z!1$nxCu$Z0#1weJk@ki7B-nzRA!~(dT*Rju_(ZEzL%7mCE*02(v!FLtEP6< zpr_KW5lWZCuXXZPB^1Sw_G)!2dT05}Um7~{b@k4gk|v4wbgAse5%C3{Mz!^!caXTz zzPzs~Nl8ZoLo&p-nONynePj?M7ruV;Ch6|DbEWSVGL{4Rs^;!_mUsnG^?TeOToYa} zAK_}h`rRwqk&U#W{#2{`gio5i|H7ve=iW^-Gpe2jHen;w^#?iXb`tsVdSEBQ<{= zH$4kkYvr0HoideU?jhsOZbyRsM=d(3sJTUeial+ah3KM)S;>Zi zCQok|hcD>J*l|AdgV}^gKEi`QxDCwDUs(!f6h47yi&0E7&YZQj`dnQ#ykwrZ1Y#Bc zXjpMF_=>vdS@NYfDYYNxbRTC^L{u}TF+L~BjiUEJG?WJ*U)-{!^J+`VlQ6s>zO8Xh zM|(iKF_i=^8O{DW?_eGt@#4f;F7apvaSM+3r-Yf#-@QgA<Td^RDZGr%<;{sD_NJmtLr=no&X_4jQG&4_R4{%S(${76EsfJvXE1VP4G_4uZ_7MEi!suMWs7Ur^2*htW!rT3J4AyV zR+K`&2(}e@6AWpqF?D%Q(`I^qjs83xbbm$c^D9X$-S;P=P!J>N$3sEqTv9ot47xjY zmdpI66^w#YG$f>$VM}+HHy;Unl8M(IxTI^lV{8;9vUFFSK$d=KdgQarinFS-!@2JS z1oCY!B|fv-v7Oz@*v|HEM!FAGzrSW8Y_1sjK|VrGBud@zA_bfhlyH{gyyboN-kIUc z7nyC-Cco=ysUNNnvZh(l{$%&Ljj|w;SB!t?-S)U(*6Udql>Lbu5C8ddV(fH-ldXYi zv}J{%Aj#8dCIP@*s*WO%z0bCG(sYHFNNmqxc~& znK)WM$Nn|JgLv$g17%3Zq$ox0n2cKai)^4Z3Tlnz`|>d{(DYVYtI&eM4wBS685=vk zM}r(@-V~(Z1bT|Rgk};QDWAA-S6}sZySj{Qw>2qn7+H|G0UxzU}jeIJI zAEd<_8vKu8# zyDEAXTo_W-(cW0y)R|2eSX%Bhn~;X7yEB^}n~jYsF-kk=B zSpw^0hBY)GYzXu)nUIDCuqx&gz)KjL2<|1E4T7c;*rkSsFdGEBV@XMLcyo6<={ik2FK0<$Ib%B&H~5I0>{oGjGaXf$0K3%bU2xv2jng!jSe5%0kLB?Avsf5OJmau z3Nk+e$WXKn{9}lDZ&W$?j+=)1JKCVm3S`CWqwH*4YmC=O+1`jL9wSr>R`OHD4-DXT zl_NS$L&bN|M57u{A&A{rp61Yv>qdQ~E0n5@HEMG5lS84_q>FjNtHX)gP1+;xrc3(X znJ;?xl^sz1cAc=+Sqsk(o~lK&Fk5r^D0OT7XMB1}ydzq8)uU!+gbJH-_;0^eNZh@f zrPx@wu>kTjc{y;hN|oEGcgHu19VyFs@Bo>74?^6Bn{E<6R=u!+0lGu@YAucr8VG^X zrjn-RvX9q4wb^(yCGfH(A)b38l~KEcp&Wic^rh->BG+hoC}Ha51g;O}>0ZR96&gy3 z2PQxBc6V>yQ$7HuxvSqc&7Eo!43(#0SO{LhKR^;&k1BUOa{CL);{E~mO!J8l!eAw% zOsc*F%gK+;W?z>J)xQ`)1Y+yXjoxxs?p%9L9uB7Sduo429*;}g_O3-@syms%D z#h0pgBdk8^+Y-&!wltTy=Q{68hu+s7v6;&is-Qiv!KstPQdk+MdEmM2L8-~nT$M$( z11m%ao)6z!Wev2s5?wz=&U5JY+?O^%weQnkj~}`mVmGQR&8|MRK^|e^$dz`IA$=skrUw6LB(d`YGPpkojDOPLFtZ=b zfnb>JK!BejCBYp6kRraN6yY;z$^HXm(vW^U*Be$22j2Q$drf%$I>O=LWx`9;l3ZV+JeDT?IFj}u#a)?RN+ul=4_Rqd5gLL9+2BaZXsIu+xwGi` zZ#+(wRh3xdaAqtc(Yx{W0r6`#_`z*F?*OU<{BFa;{RfKu-sOuilPJ`-evorNc8{2@ zdSLX-3tGt|CAD;;OiKq&-qf#+A(3h(A?P}-MZ&Z!Cljf~iAQz9I*R=veztsK@_+A?ny9CxTs7wg)pOp`G2{dVboKQsl}6f z%9RvBl@umef6*YqKsP_WfS})SPK^0TQ#HdFvxCnI`pZx|;U|P86n&vD{buV9k8&@& zm-7$PEMBLmaauC+I#EgH%k0ZK5^=n-iYJm(htT>u#fdv?oHm>ql#C?PB-;U0wQx26 zEEQ+=KuVAL%SLis)0SvAoA5E|u86c(-_cPzsO!3Ewl{GrzsSSF4e#QTI>JznFS@^m(ns zO(prnv^b+%Mu|qjMmNTU8`QYUqy3{xj;dr!&E&t~$~eI&)1Be*=Ed`K&qd#xz6Y}? zARlgZ$ILXW9sgLdMrz7ssyoR$$vnw8S@`P0!CJ7vChP)3Mrcgv%g~5b_*7}eNkCip;duV!+AKN}wc?-7UwlcJOXSLM>$_eXy)aiD= zGg%+LGx06xQ+`?SwHIHl*6Th=e2QUAWUORlVAN!chc^l{B-$mmBu*wy!<(L4=o~5{ z)S1+oF1iAaySvb)Va8Q>8(Wd4z0m*m5t9Em*RtU@jG^z(%s)Oyh%U8Ho$H5B^e$DpY;I?#9O;&11YL(%A!}L*Oj>l0sUHs`f z>0*r{#>H1dmLvDC>OQNp3x5!41z5e%X|kKd7ypd%E`l zs!qJwE8QdMi@+g?+gi7iX@_X_PtKoI7o0tLT$@juNXz#5r~I#V@9Pt-^^Con0kU!H&2xp8EJeT#bwLM##F9#nDUbHB4I@5=!19;(@o zqc1v^L&u4EJb4Pbl^%^GYj^N<9PUUzL{7Iv_g0YJvFqyS3uebCkO?SM-`yegt>!@7 zxX$d?wsU=9)2{Wl^+Wv<@3voT2~eD+h*G%S6#FP(`;xzNP7}eOoGoB80i5AITmPo+i9VEO`h@dX&h8 zu&xHL=H>O{LT~grFTbV5m-!~^dQ+TKm|XAJS0{SVNq8>zoW~1IcOQ2*LO~;WIobM` ziV@9kniXz`(VnGsJKdttAm5{KPTo^KUjA&n)=BNciPNDvp=_BFw8xo_d2+fsW`BM& zbT(*wZj5Yv0R+2zr*Q7QW6@-GYv+?l{Tl77{H#pJSw<*#f~jk>CzYS9>6f}q686Qs z4BCpGZWNdfTh`k&{T4+SAMh-oOD?`%FW6Y=iPABpaeNi&q7N?5^jG{RvSTA-)A?Z8 zUD@rH0nay|cgt`YhU5$A!6bMpgzDhwVA--)mOTra)M-*(QOdkd+GZ~k!oMr8D~56= zaox~9RA}{V*j&Lh<&|0+)zsQV;)Go*OUr7@mXENRu$zl{rR(>=Z#BXT)<^Pg_CC2h zPd3fmd#f2-BRo6)-Y415xbD#w`vd6@+%E23KypuWlCk#Mk@q6bdT6H z*c`K&A2KVveGb@`!;`8wa$~aZ7>`h&8_R4P5V?tvW z9M!3CQz4_NHo~Px>}?-13mJ>34U29p7{A(Nzx`od){{DCMirJ`x=AQ<0<}EWw0V5- zKDqcij}Y(2Gd$ZV8^y16LUdAl2YZ8yA0jM0l|4sh{JN7_I#>ixCGB{vg>KwUEPQ2t zwlk{Z*u5lFikr`hOykYv+re}}qL-W{%briXNq&<5WVqEo{^^zFQpfCpimh*&@j2#C zPdfvGIQ{(eCxtwP&d@KRL)Qw{8kSl^i4-PWMyRxolH&GFwW`)wx7-`+WW;h zum4u}07D?bztlZc+#9$v_?9Ygah*o2Qo68L>-j@T#dt*IfmTjK53S1qmm?b+4@T4s z86=3?tnl+GZS7dO*|&q6svDfoZmF*K=?8gRR2|X}bTjM@DAT>vDqlg`>OyzXRyXhK zlfy6RMrW50UYlK&M+UsJnsz=96v)23qIqs8a5Y~CB*RigYoGa)aiP8DgMDqz(#m^v zT4MDlIZyNjZC^BiORAQFM0X>WO$V ze5ZQFW3LiOm8kmN(THGRr0?3&X331Ynoo|xOG+m1Tpppd(QiE7?-wkdLO!ckrG3jB zTv-VdUGlG~>=tPM{E_n;?4CEPBnmd0kkF8(^L}LEt6u}Aso`wK!owQfqRc{hMb9`x$V>bLYq7p@QkK`uVRtgttV`MdY>eG($ z*EjTxwj2iX!P*{fjPH!=UI;H%Pgk~Yk}oJHl2+{~J-(lwN+X*x6z=whH#W31V~8oN zp?>Em;$x>`BGJKlgT8Ua7ikkzg%zRsKoW16(+!o7rDU1ZIw=A+pDKmW2Si_eR-D*! zARHpE@=%&En-@sNBLZ$EMb~w7=M2ZhfVs~%!y{p@+k zd`GA40M8XGn?M=x`-=LS+lW;-X+^Y zi#q09U({5_+SkeYvBHe#+Y%j1ITIt^PgP_#NS&u{h>iI8UJwyvJ&7vL$$bA-Fa9Hg zbP=;<{*h0iM$DWlt)`H{m&8f}hW?i8_=*mkp0H58(3bgYqu@@noP0~cY?YY`)oY*W zJ8V>y@ukm_gtggc9IxmJ=61BLh#cNL`^LqHh|ocRFo9B1L9E^*a&)nkyy*_fd zLR?lcyO}^s=-A9*f#_n$F_T*j{kpNNj3iB=<9Y(#lR~#m+ajRDbuT!n}%v zdzj0C;&icoGAa0}rkXMLo0smf=F=-SpHM+O<36rFM?iNr$ErJt$u2ii`Gl7pY#cA>_oP^G{mKWP4W$F}pCug~ZG4hdJiCsBA!`s?^us#aW zsm7RcPxKg9vLwCuqa$23r|U$8)DtorJmf_bty%|o`s@lIPISZ>qmRz0P^w?^xcj1% zdPeN%m)U%#->_N;Yal9~zP`X?x5MKiBIi zN$NS^P8(2lpnZsF)XzNNSUNGpz#-U@dKpjJhOX7B&?Yf}EB^ZCye{=(s+vvwA>AC; z^^+m)^pw|RG8_E{o)>eSIm+(mLb%yG7}}#dVJ0#$^^6sY&EG#MPM7p5avjPTSDvu^GKOyrSAMb^RMzhiA#J)w!;- zZ>W~Dv*PEduDHiUC}gDja}hXNz1Wdu=yK^XGraxf2(`zi#O>_Xz77MUh9?Ss14q+s zd*hOI*Xkx9Gz#^b=@O5$D?)9}zo0bgjrPa}~C7tX@;SI9gaz7kt5!-@a^i z_-x5#%Oe|C4Bic^S87ds*(R$!=XsVQW$|UE)k}y&8STWDN=E^ga`%RtT6CKwXOpyN zh)CJl$*Qf562Yo)it}E>0zuBNsFN4DX1I9F+Z&IB+!=x<#1*c2oe{LWTr8m?!x}}v zSuCDNnUJ&M#X^?0@Tnegg4g5+5MxQD~>SFonVYbT8DW>a;iiEmpZ-`!@E z$svW#;(b-qmJFfh%5j=msE*?PNO96j${;)_@>|sDSb7H?p%v0hWaZpd!^l+8v<~y- zmu6ibsBTWk37#_aXUrOuQjTb!f%+quyGA`DV6WColerd>#Flu>)51c z`;&+O^RLEYXDj7GIX-!Fjo=$eT`-RnIr@}0C5QCYM?CKh%~0FVjWL0Bpfhpi7x3S8 zovuu&R~;sm>?JN=F0-ArSo};~-cmz^3ZNk59&8Fuh`n*`%goD;{;M9=o(wGSraGvz^q~sM@*xBzY6$ag7FJRLO3Mv-F_+-fF8`Zepx=C zVeBpas%`EN5wm_6x!J=zA~#Y(^tBTRtJF>`OxBdVP}huNN-N@syBbkH7bSl<4wdRf zuEv#^vhYlqDB6n)6#6YOB!!OP)OJA~e=s@c6wIrCE!~?UO2Y z=jT~-=+qY-poiNCrY`b;uDnSq{*;@kI4Kq-S55L-`Hh>X=K}+>`7rJRjul9k>T2EF z%Wq%Be4&5xowlkecwv56p(3lOE9D!%U%-bbILubCOmKa4bC_6Av`A~V?#%J?5o_P1 zgkmX+zTlPGoMJ3q^Rd!@xh7xr%pzHGLviV0RrTs4 zkP*wnZh?47y848-Oy7RviYvCyxmLl9kAL4`hUj*meY*+&%oBR$M#zERn$GLGtyG2_eWdeP=n0F;rBn&!ET?JNff(&+-1hDF@I4!YC#fI z%CZg5(O&DZdZ7(H)Hh8}EKR+j{FRS#^eO$8g28o|3PU6XHI1?Kq8U%@NC`zzYvwbD zvb$X+&#O=8S7zL1oCivr+c)nr<3;+Z-HyxYkorgzUfI8#7rUys@SL?CU-eRWSPV4t zKIhe<{1lE6;+Z*x&Y6QAg<6{4=0Pll2H7U$TuIe*D>p}>pAv=^N1OWZOh+UenpWi$ z+TPN5Vm(+GQq2`L?)aSO)CS$Brh)vH2-el&7YTuN5|9^9lZ$fl90I2cbRK-h>l$QQ zqzstl9-%^Y@CB!)ADFSr)Q~gRAQb&D)pNc_b+qcoz4n-H$F!i(=%h0yAZ^` zK>>THxuE(MEEvq>$A0`MLF-gYrzhKC0#XRUVp>SX{6iDS)@+fM9G@}|lVUYbs9#sE z=;Y~uV+1BFp=r|555&GPQ7XM=Q7BuE2wbiBkXht=w7~sgahpg=!_i#OCJI= z3mF1C3j#X}0y_(E!AE}*iJb+Bodt=V1&N&niJj#~_~4(V*Am9ArMD;Zh(vD`$9}BfWeuGTZS2jliCCxI4D3&VVfKdmhBO-zOQMjBy`u@x z%%KZRhyDZPPqDLM<#{;KCpj z0^lfA7zq+V!iDw7_AHEE3Aga>2OlnNiIFI7*8sb) zI`C)gLc%}@BoJDHA`u`sz<3x0;GZy%FNLCnL0}~6S9YOS!Y%y2*(HM2#-HrM4D5ql z=L{X4Jsf}1xi`Wk%_f4?xId&534#Dh0E2)*NH~xchJz6x5tIl(FcbxX!I7}NYJpj4 zA9P}Me-E8Fqxy|noUymC)4ybCV{PiBFJo`-Z1Iz1+zjaQ0D$CO1pE_AfPo<(K(ILa z*kCXO2nGeC0E<9^P;ivUFBI=(^}k3KizRy~Mi1;yvL5;x_9iC!fUz3e+t~f27%f#C zuK|j&F!e`5h6xLUgb@I+;wWT;3nM@%mVi+E-2DTU&F%G5a>DkNiy0#a0>$@{|C4OtX2dCoOeGch9JNoI0}Hhe(}{?#xsOW{`j-%5(YV&hLG#SH92W)Kks{MXax=jmh#99yO~z_AJ( zh(DAU8rt*tBUa#)7e_Z6L>L7ETp&0c3>?|N8Y^ZU+_L||2P|Oj;R6oX zeq*c}7M70ArofSGp)YL@n5F)J9~fs6_dP&-{{?O#5D)|kxC*%a z0V)mXwlGkn1xJbi$1M{6%Z~R4x2V6uEoK0{i0d@?AvjHa#NErDSHtY|C z0XRV0)gDFOr^_jCYiMU|>0pS~Q+@R1$Hovq;$JKnXA--#{}=Q{K|p}wiu|CI4Gx2X zAb@klCX58M1Bv=oY4-=e5b$3eNbJZylt2Sr;i)feW@c#l(~rd5s($J`1pFTyNtiGk z1l%S7Y46IG4fcZv|3EQzE!^7w8@C|fzxtBckv}H+SFZ{J#^T~1Arl-d41$6IR}?2w zz`PM4pcWDs5kY~FBG6y#9J|(jNdBuc2?0!IU*4pmp^>Sf69CDM4lYjm>Ml+Wrglzf zeU~&mK)!eZo z`w+jpyS@W(Z8FtAXX@za0T}-ejKf42aB`3a%7Fh1$N^ppa77gUNjDU5ufTAie)ym1 z-XF?Az<-4~?8rXo-qitPj1K&y8HbWsY=Z#Toj-O@pdv63RAi5q11=>JghWB$YzQ#g z$^7+1#;%3q_rI9>Zawx-Xop}%_CYfcB>lQzrh=(9DN@&E0{VvdpH@n zIGXEAo0tHB6Q`fT{?+?}?3Tg*5y(N2aF7TbxJe;_YcoKtFmR&)ZWds`h>%e5UOxuA z%zkM88@>UWvoCy;wRdnZbp&pWCIFf_*xOh-TN*plBjiD1$(s|C$8T(au*CuFcGs0rVcJfHfR^_ zr^qpd{Cg??5159#e*7RD-awtzE4QoZKBvqI1ZG4Lpb1~>dTqh+8a9>I{#2| zOb5eWivI(m!G&P}W&@FF9D!_b5d;VVIPq+Nw+I(gCRBZaDiFZN@CMABJGjbgyFIHvJpL_!$5LVt>wnV`>X@ z6aBPg9QyD2Cc9lze?&T94ySb%l^{)h!Q_{MPn0s*2VNFW9F z&m6#E@b5T)h1b0tz=7azIG}C{^b%rjA(nQ3&*}VR!LJd1;PCl#4;2${K>_B9BN>Q6 zKtON=&_V~qAAsP|uSYj#sr?whUwMFOyTg4EFxuT_zCRKh42W}qAwWuWcZYzV1zf3s z*a`3g1%d<7>|Zn&vlMRWf6?6C#=<>v!;GN6`8&DUy8szDN4sC6pg-mHm)j$-(LcUX zpp#Bu5#TP4BNYht!GVhe+Kxqlyc-G%`IY#+=JIdE|IO3HbXM+z`11fx{R{D!?n@l^ z(SpZ9;~xngA_DlA5OhX%SI%rOFp!J`1Id7YCLRa6zp>cew$Z(k$D!SCBo7><;9sTw zo7gv|Cv`77{)6`k6eI#cM}Pq}L7?D>Uv&Wo#J^()7E<@J0|$M- zVTZc2se^^7oxYsCi@A;APoEppkh_;^EIj;~65yP@V|i^5D;ubA%DezU-0)|aA3DDdM_7nr_FCOV)QrFQb-Rt|f20{`PZlKtKq8J_03HDQ05W92RSgKJga3jGn6>u9 z0W>7;O(#Hp+)RF>8R`y>K-r3!rKyd{Nqr?78%sNT%RN@H_k;oth~2jPKcgL})B$42 z0PVY!0<{Z(uK?t6+2AN3&J6ymRqPWHcb@^U*DCgjh{{0u4%#Gsx!vhInxgA>fII_` zR&fO?ILsXFZU2EYILkvL7n(Ew2{nNb9N^RccdYy{Ka#`l{5ZHSQhBv10gZK0gexFAo~i11AZQ+ z;Nfq4*lU;nf{eR0mwWkuL$iJI;Rm^x=Y8N<4=DU@S>_*!9SVa3Xb&7`Kg140Cz}B( z{|CAI^c8n2G53a6mdu1PUZUe^3jB z0@np3U^D+r?LHB1x4d?bz%e8Hp!P@k008fH&X#-PIGDK29x3nE?*5UY1DOV(Q~-Ez z4vtFT;D-PyRS|UY08q^a9Miwt0x)ag*8UeM?-mm8rF)EdPp6U8g^cgd<5BZ+P1YvFT&CznU!&I^T zQ;GRc`TxaP+ifBEBW^%}!~_y;%Dd9WmK+1+*8iL@$1H^dmVc4|Zac$XX5d8l8~Ll- z7y?}I(Erzf>@WEuzyfyLCjN{A=*SJ=s{CXDP(T1AYXE@%$C3feTKi$a-!8B}dH{aI z0(m;w`WQb9~y&eHv&xL;}=YwGSVsI<}i_msE zRQ56e2ZX;--ydGsk0Q<=$b~5|z%34F>u%S{ACkQrki!!BGaQGjg84;Pf3&mwM((=} zD|^WP(OB|3vYmjalanRzAVPG(4`9F2K-|f~(DrZL6PU`6y-YwO;~#226zGkC0JW=o zOMif_2`Esh^G~HeK+^GN7sI~@f47HaFBfn=$8WfxiffZXbKq}XQa=>|=j!bu0N^|S zF%Musd>W8Ba!(l$VEHhhJnyenK$ul?SJOwlnobA4;mpDxXhS}Gg|u+hFhG6kS#4rmhsX8c(Q^@AsWDTET)C;sg=xa?&C z?&|pej|G1Tw(K^s{22=%aKLK^+}0l~fCAwcAlv@;TCg9jF~5TX?A>@T3-*bC7%fmz zl@^k>GqcjjXrAp5f|Qp3^36swr%2S{OmDUJS5MgW-9)f6ZyGZd7uw=uy! zdRETV$=K1-0q9pGgWwumH*=sj7I=+b5&cFG3Z(Rf zWDOk@O)bqWoXG%;0vcDHO>Hk?;>|)@*m28zEDeC#K@}V`{NH$VJ9wZChFe?R*#a zfykln_~VQH2={pJs60dsjt6_fAmKx~eao}-XMGexi=Wd802eyw^ zJ$1;kkJ$Da>s+1x$P}VAp6tQf8ss(%F)At=Nw;CClrV}gKk@uHTc9+$pp=PK(~G)sD39GvK-;&z-(2yAZ6M#sQ^zvLT->2KSpSi)6Ju>}j48<={>_ z^{!*OMvz{Ts*!wPoCb?c>moIyNg zJ!1U!!`ztyqe^@0c3G`e1EQ~PT7}L*X?erEH+i@>?+v9INr;L0=UhY2Kcc74 zVAQdd4c%m3lJg7=b2Wt0G?c|Ng{&K~%#xRkz7wN!aSnQ4a6#g-{7TN+i`UBKYiEzY zrkXioC0$B4c_!BQBNOov-c^g(68{Gx5evj_knA7dZyDf<%v>JZ;%SV$OE>plH*g9Fq&) z4YPxyW5PwCq3n1Om+A30G2Inor-YK@I=xCya_1%c9O-6PnRrxigvgVG`>E_$L|mmI z%hVYbuYemuHm{uPrA*HWcFQM^Ch^_s4*p!i$aB-CSh^X^?aLt6@;D6nCH*FUs^Id` zesLL2i5upvG>SeenymTpM%Scn$?CdWEInFIMy+$S36;aP?|PYC%ed;lRXn-!@F&67ZNafmo2jjFl3NPCqk`#vOEgRrfEJ{xr& zvN)o81df;ENBNp!4v&?PmFWIKqiTD+BDWB$d5;1rB7Ee5?F&nO&Z^Bbxza*@okNY6JCBR7-6A4toV5zD>7Gx(ipI#J&D3 z3d!=Jne&}*#7(U)-MCQ^e=0J(`4L|{@Xs~0`Ol9EF!wtr%_TB7-y@i~+j114M;}yp zjGvQ2^&lKyz~7!=z;CFU-hpJ3VM*YFrqn!(2LOvd)%MxE{8pqwsTc2;C+C6R@$wl~&6#oa0OV7^iMwd}IneYUTy%J@Sb_asO23W|!_8=G?3^1lTR z+3!rp+^L;vIwHhZKK)W`;Xd7xOIYh8>Ye$MPhnSf2Ep7n8#t09#NQTd8Q2^ZlS|7j z{}w)>0BSS#XgTR#8d2eV^X7!|l2OyMV++t)z5AqY45J{9LDPVjp$_p>=%LduT1oZ} zwhxs-lC0FSuFD*wdHg_?=q5j}hIA?!nWpN`p-<%K(E{>^%A&Lo)!H|yJB%4?-p;yOeIjJa=z%XN9n+&I1N5J9r+)}t=a*poZV;P_{o6>j;WX-Bl zz+0NgR6lYe6J9pi-J$F4Dr!eqbG5FXA})}iGq`2zW71Z0<5>q@zG+&rcQ{2Sd*G-8 z4g1^Bvq4GLPn;NXp0)Yu;d|F*!W=grm&L_{5=v$+b^S&T# zHKD94yxQj&#w9YD7L{sdVlG66G11+vEk*eqI`vuZ@m8Xy*U(Io;{vn6YAD6#RkN!S zXC_40*FOkvxUYDj?zPIrxrjDEejT0)BO($2? zTHJ0URz;-2^+I0d1|Js4Cs>1Jp<|p_2lO{{CwmY_+obpK-Bq>LoKl4upx^Y8BYrD4Y{az zez}w@`nR7X)9Pfc5n`Hghcqbq|LSb9GO0?1r^FEC(hujvR9gg}cN2EgSJMBFYN6gg zt8P!k<^-7xr3`|{NG3SFVPUC2fwp(^O7pp$ICGjo&o)Yi@WgzXUfQ%fz2!a$?M!nmng)CsHn5mCjTz1m;NWk$eg;noJ>pp90EZvSLf@) z>0Ld-OI)#33BJ}+!L*+89p&Ba68)@6!$Ia49$ujmC8tz^Ea+e5L6f_BW5K4AjfVp- zamu{P~da;3&VIx(RW8vQ)l}$f>w%`%Je1H&yOXt|miMRV{>}pE$ zHXr9I=!ojOl2#Vo?Q?|&0n*qZKYBVc{R5NANrKu)NI^4RegyJDUSWo?2O@p}=K;V2 zeK774Y!L-`czI!XLVJRSgB}26o4HSF--I(PP9lA%&-SR#CKBBZWnLB%@Anhk{bjs~ ziuhA!I6iizbvC0DKcxiER=>XZhshDZ{!c}uzd`T6u;Raq*hK$@*!~Y-?LYQ>{sqnc zZLr4j89V&n!P=*a#yfhXV7bkLjOx{$cko zSjO>r^7E(s0d{}NU+que#rf&+Px~*=pHSqp|1VEmpCkVD?C1SI&ocd2t4|9iBJRH; zy8boBKWhJ+8xzxCp8mA|^SoL9jP_an0G~hqe@6PJ&0lNyv)aGz{=M2yPk-j}r|<9b z-$(to+y8{3|F*?H-^_or_%}WM(-s z5R>dc+S8PBu0yA4lc*fa&BQL@P5b7J+e4YI`ol)6dWk|7&w-|!_k9I7YAnP&DFi$o zv&DcInLh*?@h^D-PT#Rev6X955zpP`*5yvIbl&esbv8SZ(zB@6{rXU4*_o~vuM00{ z48hKj3fW8+K0AD^;DhbMv2jwn_KtFCe7^VXVk;&Qn!a z#A@Jo7tthkP3(EohYH-3WevG~r65184R7NV#0y9txlE=X>nVJuG#ecwI_ja}rN4;y z21Edy0W&k_)dgPd`25iI=;6Yum!?(%Q!u%f0ZWZb3{v<5F=v+jOUQhL+)kUR5qBHr z?w)5dNe@93W1Ip9Z*%YC%BMiv&p5pIUt)I<$TuQ#BzNO5yCMTUfq#f0w;u`J8eCX$ zHNcJo{eU$2jkK%(!oR$++YAd+i**i7jQAbV>mxSl{Eegh+t%Ag!Zrm4Lf4fLP4P{( zqu{|?gLnJ`*HreevUTO=om`KU7lR8OMbwS@&9;H{;O*=ZlrV1l=DS=EFV_wapJboT zk7tTn%dRG%3B>PZ?O&h#9Dcq;aI&tEnMjqiy?CqE`JKoEaJN|q(QqQM+Zi5o91ywo zoKix(f$;m=t}GHGI4=jlyP}_Lv|HmPR?!qC@-9#Az_2)EBVKH&~Fu*rUuCaT2H~* zQ(Rz3B!S|A&-oCqo~SX&S|<@_{R0M}f!8x1)L8U6E%?Q6lZ3qPoD~r5Vg_Cd%ZEbCV7b%qG4t|`oO3aEPxta! z!itYLsZ)nFyA-p#snGDOAgD69#Sk*SFUP-Mc(Ssk6c@o|9`e4Kd#GPzU}Y$^b}?vZ zVqmH2xER?ESF9jR>P+8PXp?{JYm?VkYu{>{?!EBvmnUZv$mXd3b+L1BX!v2+`79?A zlOv8n9Ni|#A02!NhqFFpwGwj+a*&g^M%_S2D{>Li%@OiC$BJ;g8ydDCVHBay0nsm( zw6+>mKZl!AT+A#nxakLD=o2$$p~CJ-i_mT*N-4C(Pjv=K|GhBWV9 z>&~Z`BkcIDtHY4{?U20LkAvsiLY}V_{DYh%23H#nq%MXK55!fCmD4CJ75H93(IFO1 zfQn|pezHD2&P0?SJKHVYs8coFeOE1p+k zX>w6%_B7f3xbiWeeF6)C8{!wBwjAAlyWi?F5A>3{6ZLvx9H|)R5_MA~R_oHP1p1Pk zOrYj1j!2k0D10cLcb=@=kxkouW<&yj$%OSsk4M~B{7z}EdyTRmS>w?~dcnYG^f*GVVla52f+;MkGq_YQKk!OV z^iJ=*MTNZC!_3z=h&CnGXXNI9YGhs~j-P=SKt7b&;U!xKm&lI+{FvVc; zZvv^PdeRDE>lO{E@)AJ?1sc@o`Q0vbqD;ps7*vF3U!bDwjYI(>Bd+NPk$_0%GZu+H ztq1l*kM`mo$qQW25*|J5AX-ZIR+i*umIZh6g_j5=WiL!a?e4|P)tP`Q=olI z&A@jAbx3SX$MRKFUmPK1Mse_f!~3`?v^Xq$`nrYZ&5NY{{NaBKFKpR(N_eGE!z(G? zrh54py_(81v$?ZHMYWW)0(}v$0#C7e2#Z3D=lO%e#RZ%;-St43;UW>a71NM8@NvU(AI{1f!_R3GW790i=}Of&^~yu5`=DM$vCG+xpZ>{IWoCvXWZz>by94Sqb^1_&5q>VaTbvXm2j zi(Q>h)k`m3J1i@DK}RUzlTDiIzAMl?_7+m?xqA+9*n42gia}ueC^DT$UBDA4{UDf5 zs4rMGD*}#b!LKT)vqsY4UnWSEdudWQSCxW1H9un^lGheJSu+a7_A_ee>d^$nHnMj= zS`0?}T!?XdYcX^aowZ2Od7qIfYk@@X5rf)XaboWcaoh$ti^?0)=;01+7-fp8_EqI= z^5M@K34OXziQ#mITG264){x!V9|7He)tSiR0#2$|Fs z9T2>|iZL#|xdzy=MmZ~^yWHc@=b6G#Eo=B&knoFdfO&XIA?GhPTR$oq(?z1ntg}j8 zIOe!%(I*bbWPo>z2_j0hTjGv`fc`SeFu%;t*1lP7hFCqqe zZzEk&4*PB)KIZALCzh-W6*|bGVcmVC^?kLw3*~JSwYxDxIQU56(z}%9THD2RYOby< z#^=%lb7LsPJAdjL+&1`Zz2yI2Zx?-~e=-X2| zU>Vtt_~1v-bl-B!`54>o{B$pKAGC7+srUTR^=tcwVed9@S9qUC<9t167^FznMIj!+ z?Wg}9d@jDB{p~`|!UL2)bga4>^@62gndRaN>paIx+H33Iv!4mFUE1JDx(Q zfvC-Uoq{g<_Q8u1%#g-{L#@m>qee5x7*z-HLJ|k0h7ZziAA&kW>?5U9`(yng zQHZ{h0*X-vByPR@0G^p8!8a;{!Zy>JNo>|b7!QEZ>M3X-g>=p*^Nh#++70^v=t_$O zag>LF$wz__rJFhLT%D%!)VAJwD*?TZg*MQ{@O0tc%Gl566XL(wt_2B@hE+y3%I*L< zjmYD*-+cb)srfB9SHnq@G`TM#I_U13q!>&qlS5t2dEeRX1wmpnZwe@j_0o{z;4w9U?PhMTFLO9 zZ%YR5*8c*`iq3+A7(;4~lh~CYyqI{x2f>L8#?fr6^brpq(1wJQ><%Wgv$e!dJGjLe z43Jrxmv8c$#BZB!`&QWsO+=>0oSG&r(I`NR#fti_oh(c>Qmx!WOhfiDEz=DKsaIvw ziT0Uc>?*(FXDalNAbI_^9^$kDgr z%Ie2}TIeE8WhZr)d@jBv!POk9(CFCt87Y>~w(zl1O*vx@Dme`*Q71G9z_ihTuE`1( zT*JFpSpT!)M@P_^yvTjL7=D%^SB9XCJ$*aY2nC44)M`bs5ClS4Z=Rm-10O2(ldVO6 z?rFo#Vxr_t<&Tx#f;I7@H%h^T&g9P5)!CVxpLDGcM_GLrzv2Wd`h~ZUC5n^?apwku zuT5;G1LQDD5myZ@^VI{%VM^mgTvO>kZZQRyMk-^{yR0-36KcG(g$WUuxe07&{lNQOrQ^e zA4^b|AE3c)pmv;xBf`1oNyg*4OR+Qyaag5^-tQpT1fvxOkq0{78}`VC~67CUMw^GAws9-EkQB16dp4 z=V4X8+{zb1q3Bg(`!n-}XYbbt;_ z20e)C#0^A37WL$e`c(ODj&|T~E)hM6P~l)@6{AcivlT%QVmgMMWb;D$xt!hxE=b5r zNZg{xD77DDe1kdy=K58lN8}J=N zWa%=^HRW5YW8$@qfgds*UEW?}d+^N$&+hlcMpo@dj<${t!$WK_@NlHUKG8~9*v8aD z&_dg3=h3f~PW!Usv%&pt*6^Vz(s}o8W~cQ~%_+!TL=tOserNIaFs^BqrMza#gkgqb z2aXjz5C{SaR;ppq_T8;CC^UAYG>Rxh8)&~4r-LO>92TtGu)|=Vv6^F2D-yL9PYT16 z7|6-LaoPmK6Mvr=6s-0lelydvd3Ea!7Ev>JNmOna5ASI^<7uy|3M)f!W7x_;9MN<) z^5ta@A+6Ez?1!Ih^g>pOpXXH)I6CU{fjN9mtM0xm#!yVcDj4flS!z>^m(+9cVe>8b zH22Z#GtxO_G4+|X1^kU_GPfoukO7ee$M0kRqG>boy<$fZ`{aR60X-Y5z#{G}7QtP= zov3F5i(ax4k~VzJ^eRd)#MIAZW9pj~C9iqyaoX`I%k#)0FbuyDD8f8{i6M=K>jUtA z25<&u3VVaqX#u&Li)>v{0#Aw{)k7C_hRf{sA%adi(#tx#R1kSDt3iXse ziZwFj9FY)`1_b4%j!^|vZg%EHClczsjhw7T3MbH-WjPL3M4{r0imbga`7T{AZ_>0T zPQE7H#t`-`x!>39q~RX$>xW(4z+LSLa@*av$AjxyX8PV-?(}fmUvxfi%Ah3agEWGT z8LnZ3TTk)Vn1L%t;9zYIM)5Hs5x4|GEw$y6xu~sqMCur&QP_IpS1pEDQDr#Z0V`E#-Q{yhsWan6z=5+Y1~`G|A}xhH&+<$?L4b2Sx;RAxqeTMR>GpOWcFKnma0OqH zj9$>MT2R6yIhe{FqP#S`01NO1@yV0ImUBT3u58H~)_UjU;K$ChjsPoX5{66XpCT!M(`;pQ8Pw>w_vc<^%H43i54AWJJq9Y7p`IpQlsg$)dUFo1E;1AW z$<_?fq7H^Fer+X3OtDEz)`^E+hu*xUdEx8d(aC;!cFP6bX%Zd;GUru`26WhhT}dN_ zNB>&N{UK>6)|PSaMn4vR_RKBE&ITGpMw!u&`c06V9FLPBa=pQAHt%_bJWaPa;mg&5 zw?zqJrZWz%gUq;h(MQ4HEGN3xhOgC|Y!WkgUpvJ(Oxo*A#)Xtaj-Tdv-;dSiaAY*} z7>hxNU|`UKBUnom(iAiC2}~vykSiEp-X)u^XXg3Pp&To1rXxkm{|`wF}4|mw(7mO-7#WK%JJ`6WK$XgY-g~ zU`i;(kN9`vhnq7$J?I<9=>82n_Rz*=wy%uEcmRtm&kSFxa1sb*n9Ha+R4o?H zN|>sH0;(D{)J6$DUV7fCJSjF-Iuommf*vAo?95{9Dy_l@wU-DU(05lSX41h`>Al~yl zhQNlMOt@!*gS;cr6`M!?t{Oy#mKG%L*x!5 z__h}_70Dz=uF9?o4?CpSH6^KPO=Le^ZrVXnYN>U*6h4BAv0PC|-C7aAs%{J|Q4wur zgG%7mZOEz2VfOvUjyUM8pxST*t%004+4~KceLXMtH1coJz%2Q`OjWrcV=?f6;{qMJ z$FlB_k&>=l$Z$zI5okV`+`{Y)?U z$?t7@<8VdWeENE^*8&d#&57_^3>0gmFx4Z?xFgQ&J53D2uSJRoZHf|A)Hxo!3EB5n z^rjAM8jw;h1H2T=3ZzDyz+@ywM-W;j5M(ZcGR(XN4Jb4TbqT#%(s0|kdW{S;S2Uh$G{iF3#S$s}~8vFM4g1-7*F2#cS$yQ{n<$Op}iI$0AQ?22bI5b|7 zi*&le7(JB8Stn5#I5(Ao?=Zk$J#3<&QvizNF{=R!jb5&ipV^4Sh_w%j?KjhGn_{Bh z$8}D$LdKI%@&TKGr`ZfHWM>wO`Nr!}M+_68N()3ar2Ej+l^a=Oz(Js%Rt4IRkd`%P z3({FIUE6n}(ni)*?OrEj<{wLCkKY_~;lvbre8iXrud3segr$Ow10}G~_y)nFV*6{V z*+b6dkUc|&kTr%uuJh15op3&g{G%)%InKWbJw+yh$Fr<63;8!JAXJ1I>yUXbHi9^5 zt4rWOW>3|^dE=CqxQ|(PXpxC@lmh{Mug7`M!n`e>t1z`-x;4D%Acg2dJX(*hEzo1Aha)`LAb zQ4XP%)Y~3Vd+Bz6uweXJ4k<}?`bK(L@4g_%SMFZ?%^6LsQ`dk2NQFoZL(~Y=efR89 z{FHA%rlF(=dClp3X&lM>Ny5QG@A;{Pv*2{_=&PY?+a29y+E8s0j8C4Gzy8=7{=rRf zuaU7HpjRO;yZ!mWybFcPle$J40?W7ITb|VZ4!o*k<@-J{(HKlzO$#ogr_MB zz9AV_TyAQR#fUC*)mM64UU6@Z{>@tI_hk92Cfn}K_*!89AfZM~DQtzh^E3EsHH;ITGbo0m1n1jHsoIJvX@UHo zDfP6Adk>FNm8WidlJwwz?iV}mj^A)id!cbC@F=0U_sf9z8Cuml6+1#7=`smzQ9M~feKNdD0p9eU+BfL?khI6#oh-g*1kY=qY$ja_?vqry_8LT zLPJ{uc2T3GvrKKgvHpG0{@56sCL(L!y_BjexlE;%l=QvCXTI5`phf``rk$~S2zXy5 zrrEfpj(Nd``5z#Z+m@!wP^sue?h>qLPm!JqXL4WB)Wlif49Mbo(=E4xv0dgm+`uvoH68du#iIUYSuyEus; z^z=IISnm9pHXM?mp#NbsZ|ZNZ?wWno*BsbFfY)x%Ou5f5MO0A=G>NuOCKqe!aE;)E zF30K`6=dSS=umY%)dFD#EvH(|<-F_?zil_%jlb%>OPGlPL5mOB)}v9_2(M2{9|3Nw zmpZ!0DkIiR;1!Zoqca|nyJ9^kT-Rr@d2G1YW7}L=7-7U`f<36X1Ph&+1I=wjlcleA zWBhtv(ey%>q#<3Nd`sNXV%lWkkkxuOA2-QbZl4wR-YM}6OuJ}=uL@)b-@D4hnkQhx zN7tyJw)M5wr=M@0r9E+v#!*mZM6FicTkw@{?QN@VeM2SWex@mG!DLiQCoArYayfg6y2u4bT~>96=fg@K{EBwgR+gC~)zN zY}pt`twN7#m4RQr03&_kP3$u7L1sEeDzOlh{Q`Mr@>E_pGtvf|Wswf4C{zadLX8M2 zU7BxVUBWkrMpJ2ygThqNAtJv)&|#RB2`aKssf6cpVx=eP!&m!N@6Fk2mH5ElSj!vT zDaM8=z$XBN2vkNK(WbEHAJFG9^eIK+n1z}}{FsGasq>+=-c&tf?$;KKhwu>a5O}HQ zkqg7L>xRFsRohI82u_hTxk^+7#hKL^(ZVGf0;NYVRM6}@R`gRPSn@-{V5E)RZKg^w zxu($-8e7sYWsq2#G@;s9)qZ7_)1X8|=vfe_x3cyx(^4NFm%|?Ldfj`0Iso+uMlKd; zFp}oc{QRZ3p3HMu(F0mQ6G$^2xgTk9Y#G3ilq618)kcnOEQxe*^g477+|+l>+J|Do za;)$=bkz7{Y$P;wY&tY+W@1bv-stUZLZ6(XNQb|m;njl!Txtr+9=(l>RuZ_U>XnOR z=z+R0fy4B}qQl?${GO!26+Oh%% z;FGIk6U}DBqOg)<;S`!!Y+ahS@Ts>0{188iMEd~&zDcC9 zvYSiy$CmV7MGlL5Uo7~JmrkBmrxERqb|o3yh580}3D&uNPZ}pR$2vT54)j7avAqId zM4sqV(ku=&(VYugYEI8rV9p%R5ZpCUFww@v4j|}e&M@R;b{fl?;Z#EfY?ydDepJtH zUoqTF(pYc4KB5aD@Z49eWR zz1b$yZ&%Ti>o8VkiT*bj{!TuxeKik!O0AVYa;tk9`rM!f zfN<-N0~sdr)L|;NVc1RUw3^LLILtA`$D6_csBr<%L4(&T5Zz%Eeor3%Zk*!(rSz zz`kqh+|6FQeAL$;e+lNTgP80xz0^KZTzdCDRTqEme4bK3fLw~O7KL12eF(u+T-;Yj z6G&p7DuT3|G!2w3oXWhGM5y?63$BssV7!Tby7>sCVe-SoIxs`C6}y5JQ;WS_`6xhJ zshnBNPNJuD)~i9xd-SxL*Zzr;UM5_ty>SbT-RSNNKH3Buo^xiNok^Wl$jlQ~-#*yE zUg7)%up=QFF}|k+e}Cq%)@6^cIdeuJVnE_~_~mog*Rru<1NzQHNwF-p1)_E*=|LYq zLVYjoCEOQn-XzMT#Y=NW;o;}yeSM!aO=@1{u<{wTIg1=M9ibaCFf#<)G5Gcik6-#^aY_g>M3``QT76#3O*` zKm&`KG%A4`DJT%5#)j7o*hG0Ajt%R>*tWA{ItgK!zv5zcQmp^#Mb>+ssftd4ToiD< zo?w^(Z!FZzeW=x4zF5%Tcy%^!wi>Tygg|_y2?X?E@uXdps_?#{AW;aj%vQ>0s0x_i&?{IG|K{nh=5!+q6~i)W;+z zb%&40R;Rf;_A3}!>^W-C9vYI)8Z0Q3gf}w96Vrr)ZK6SH&O8>4{Zw;@?afn8YYo7A zDTuRe@`yQk3ODc4Ym4(PMBPf#)(5_aU+k2KsnqV((H?~}Cx?=57lHL4b+j6n3fPa? zjQJ}L1c)e1B+#nUr3i-LIut)SsF5DX2goR1Bj05}05$M)U>1mSp>rww%bO{p>6c!W z{4BCv!^%rd=7v}%q>!%e>Q{Im-1~YRpk9Ugd_$(7aj-t5ZpLajP_)+{{^(sE>gnMR z`bqZ=p6GD*{Ev~Y{yub0m5JvGt4$y{(uXw$tPZEu$!#1LV#eA#f!IH-Nv{ zXXt`j@aFu6VWBVw3+_V-*^_DC#4ch~8Znl*5&I;`R}?CKbXsc;3_ zNjDaLyC291M$^?PT5?|<0X4Qy+H+Uw5L~FVuy_k#pcZG8do&X|p0-w3&EftXIg#32 z5I4T&X(=*KiRDq+ZA^`hKD8DEnKh< zHJS6q*acrucWcbXr17N3OSE0T$Q4F`QbiJ70dt~S5`QQTjKVdHI4yMm1qW_rUarZH zER?*zlI|>BWRQG}RtL@$m2qR%|KT?+Xd9lhhd$c(`*B4@n_s-S@1&4|g0J8_0)rS) zi(0bImI~jx5R*c(8G_#g{i?_3t_@ZwO}TOV@189y80&cZ&z|)cD6T3VnO`dJV=UYd z<2+Wu8&*r5p%B7Cg54&_#Fus)dt2K<=;{}+xwc=Mjy8fvLe6@Llca&`iU7gYMnPC5 zfYg}^gdWcjq544K0W0Fz163TQVA_5vP>S5IhsV6OCIRB6iA=qnIx`I)aVd*c{>TPI zo#byfutyji?|_^rx7EcMpXY^le$CX;8N+U}DA}s*dR?Q(!BTt;N_g|2DOdW6!m((# zW%BdtxbQ<4?E3aql0(nR)gqaN`0Dkve6W)hLiq)55bBSG)?|66)+Sl|g3P0!fWzg1aNu?$Y1Ir~&#+CLFHjz(Q~~mwxg1!Hqgl-^Djp?x7>o>| zTw>28AGCKPmr$P~qSm~2S@)xoqsn~9$gw7FQeJ3PXnOlpLc=JF>f@}O-#C9cV?F1P%GTYmi63aTe(T|Rpo)%7#O7jhjGX=ENZ*Pmr5Y3-ovNDw#3k&2K$3 zS-YI~aDtb=B6uzQ-5&{Oy%N^-N*7sCOA^H)#`v{a_GTHocuBWJJ-5Pt_JwWgCxa}w zJKN-hady^t{ra7~j1FqDSB8+lTPM3btC;!WO8W$DHoS0+uk1)F#0%%ApOD9w7vh`R zUqkeTzM8pR!`d;hH8jgs*=$GjGndvCy0OfxW=uPlC>;8Gz;VT5&dgVuuhyS z-r9YeXvTP>{xQyU1o3ZFw7{=V`}?gS0}r6y3N8~vj!0#PZaJBjD5PmW8F^U z&fqJ#O>ivilW8-7FBC8dR_>{I`%r=@PgoSXzpRtO7pb_X^xb>xAvu{uvDM2n74V9Z z-nJUSNf>WbuN>;|zXK0-RC@5{8UxO2=@GpH;Dj?TPM;5Pi+Tpu$)@aPRvf|#XUTUx zd$#NgJfc05tsD$Ie_9^vPr?}p3pmYOK?4!pP;#IG;L%2@dP;z)(t{}yhR4YKu-h_M z0q!Ws@Av_UHNGuHr!ePm-pQyhVg%t+^9V`}wPgrIYO14=5{?Ewwy1u$NPTjgy~pr$ z;QTl@mqzB@30&E|N_fVe5cf$nGI8z;2F_rrqqbMT?I|8y5K3z&o5&{C*2 zDN0hCf?alWM_g#;#f*P&33H39Q;uPtB;KgY1}D*wcOHi<|b|gusf<$;ia- z&;bo+m(*89{3DRl;-0#H2SUJ-aOXR2D&AV|gzc1G;Kzm%PUK6tmXgq7NkTQet+b7T z*ZX6iVvaQKofv@br*^)0U{pcC* zDO^L!O;pl=%`TZS1FNQ%`395&gI&FCwSD!O7;ojX%-p1kF|VbuY)cbYZRBcn<1YT1 z0&WV*_9;dNx)0QMog$a-IyT{22|}f1W6N~+9mkXvMln8#Sj0pt>YAtDI&IEMgSA1p z95_xZ!{%{XDZ=x@6HgWQ@&pj@Y12HEX)Q$Np3l7ET*{zkxy(JpQR7U zXl@u}F553>TTKo{;Rh_(>k*V7Z5hvhqiN8&}O<64b7S;Y3 zANsJ`U!uAMOmrNhWYFVhT#S5+rH|ObsWFpPCDy0RPB0N<(Vg(vk_jTgN?25o{u2P= z2#^N$gqPZa1#;Jb(PQi>vv{5fnR>GuNVSz1S(P&qkFa-)8)P4IXj&9@vh#rRrl(6i zcR7+Eh~#AH*r6R7(YLhPmIbYj(&Jnk`Z9o*6nTD_3byN&UR+<~xT zlwndr8Wiw!fi8pmJb9%HOrtbKmfm`Z`L;*ydmH9D? zE_;n9y2^yRBBZN|KT5KtdQxc*?|uc203O3t1o zY8M9n!>?^$$w~0Bg$qzJ)Fy+<^u4>SM+AiZmeX<(%iHk$e$|lZH|JzxUF7sM6*P|_#@$_z3BXb)5!L>*KfA|0|G&x&+PA(tYzPACURAM_a+ zivnaBKqB-clq57SlmOTVriWaCsz)t_KBSwX6j}`M1N$%ls|6JgxCYPwegG&y$3b-v z>XEG|l~AX&a}@)A0>44Ir_3S%!?Ij_>45(%*pe_iK9e_^`-6SHHD*g1?w%vu#@e|RRrtj3$?Qof5{S8 z=@PbftNHEc83k*e{FM~V4Ql;qYIxtpZ7;`PyoE3BG7l$-=bO~}^VIygs?QS&M}VxE zAl{-dSDDSvu6k83&k{ELhO;N;MpZAplG{fGYi);ygoAHc6N<-)*=yANMXFvlCAako zN8Y?edmkCL4?Ypm4GxoUSu?CWMV4+df>YG|ZK_@hC9e7KJ#q+lZ*QoOZ5-?_ESIdpUl<&%FFy)MeJWnJb$t!{O?pm{n$~P zJ|-mbhvx|G;_pVIPfAd1veEKMp@FDG|a3=-83v7tMQIDZ%jdnaJkP!qwzZsZ3S80 z?V~kKR?Iwtz2_%$Nm5Ormrl=0W$2}UmP6Cc9CQlPj;^`14baVagl}rH2|65Ff-i&ce_jQ zT@jV$Y(1;*M2U%^`_p_xb~uhy-^XGbtiuzei-+s2rFr)@>a8=_q*q&1DSW4;Z>H`& zG$eZD`W_J})>$|Gs%{=9E_}Q-yxFRKEn`uv!o_S12oEpFc59z|I@dNZ4`Yt034QYnnJ1GhmtqeQO{Im!-KDW8s(B6Q*LzT!-keloCD>o&U04N z{wQgOwLe>55ZG315bdgDto~x+vOuD%S~PQ-@F6#gnlWCIEvv_${QW(3&?+oLx*!VNHqwkkCIfcq<`jfJkC!T@mJ0ja zsORPsQq3i3k6VXrSY)!!(i(hS8kUN4L5@_gl7%h}6+ddmd9J1Nqz1ROnAts!?3ik5 z>1&&%V$*!3@)v-_U2akCQ*L0O4J~|4$syDsbgb0+DbX1OJ`g`xyvdBTRl%u|R&DdZ zlKqGg{7=u}MhfzEi!2O>m$U)71A@8^W8VW1SxM)P@tHnoG)?{}{vLu7gxZ>m_=I(N zx5QBbg5i^fr)?hTw_M^_t2Jz1>}-9Dwjr2dYUcXk;$;+GhZj-nxU&uwc_ZW1@7(EN z_r11OKUeK`wPTjECs$0`pm3sF754H5zOD{e7_3e&<9Kt~zDOhwOzpo@t)_((AK3YR z9kj(S3hg>drGN{wgWWbfh-gw*-pXOK?4z)YK%vUZ3PyPnw-G*5}Lt|?^CBQztIJyNxk1@Og6ZX5?GD=a$x2ap5 zyuWn`(?_p+vpdtzgbcLlLUAqUk686{pR)1@YD;K?M1RmezOA^*K63?M-xglXm4t5g zCADK;i0r4lII^!?W)3*5 z_in{Kv#~GN>GJB(Y=HE!kKzX}@TQF8Q}}l6l1PiF9I7VALGlJa#mF5~G4Id#5sO0V z`)v4&<`kzOGk9pGpH1Omn$#7d?e}b~i#000R8=hj-x}^Ap1wt$HNgfviuIhBphfFD zkK2rgl7E0fA><*Qykv%eOvnS1%dv;3hp>Ht?=y6|Y|<^dS2RZcVxOzHU|CG}s>qik zH28wIm{gL?B#Xn%#Z7Y9N{O78t*`nHGm}P5w7&=z;7$Zuhcb5$M{C&a3aWqs%3k08pXS~=uF9r)99B?DNeSta zt`nS~J0(<7B&0!Fx)nh>L}>&;x}`x7NoggeySqDnm-qd>-{+Cz^ZdT=``62d&vg#N z?6tGAvpc)9voja+mqwen)WHv_2=oRQSG_;FJZs`SR`B5s*in1e7~6N##DK|?+``0t z2r-PlwkZCkNPP=OkH`tTj!YwnvT1zgdph+aP2qbYWmvSY zdAhxFY|}Zz+fJT0a(@p}`|O?b5NEYG?X{Z*ZJe*7K9R6;46_fmRSz3l-kwh->=3!Z z=}W+J58J$fGeynp&WSJmG+9GLEZagb?u^^^?RJhC<88)mM!9Gc+3eou*-3bEp1qB% z%5uv0^_Meu#9aex9_g~+w_(mn7<@jf_82bx=`chLueCXW9yBxW!guNOx^r8H$s#(E zb%_#!mpaTi%Ag2~kA|9|vz(&zMjDu>6k98q+MXXPHMF_k(NS{u3bhXUo8waxAb!}L zUpA@i)wRCUb>NeneEW>`{spP}qAt4_si#($!9el{(=F}W=q#x|ty;5H_N`=W-=boY zV-nu3C+2qA4Q(sMM#a6cR!oSChcNQ7f%VbXg|r%Tn5*ltl#RjY{o( zdoEinEyG;uahy|6-Kh0sET3;fuVh^-dtzSqO&V&+~wks?LhvW%qZqd3QPOoqCWQYHvM$+WTZGlnZ?Au8e55Okz&$ z_+3e{qNp>{ytD`IOtua4hOs7yna3G8t>it~?cFCY`1{?sLb#oIk>23~R53~VHR;bh zvNrLVxEthCyiyJDb(&Yl3bl{>jP5^eFnG?a%d|0fi@Jj(>SjUPt23*Wro7L2@??6G zVup$-9O~a4F_x$(XiDOJ5FRBzQ${>NQ;R10AEoWOCQ3r656l%N3*QJ>5Z8Tb?n;>A zp35FwZ#jApgrnc;EE=z2R@mHHCaAGS@1FIA(WtP@nLh4bX8knQYma8pqJdk9X1LN0 z#}GqzN(BWRE3cY$8V6MI(q8PC@f_JfeYrhS2sTRwa*_Sp4_Kj1$rV`z;Od^K zc#v6Bu5Bv}-XBu_G>EZJ6WxDCR8u%?nOb()g4~aGA0bo8K#I`f7A+;gXo#x7n)Xu> zbLn|UP!J@=k3HQSP!+`JvAF2J6)A_sQ_I&-pgD$BOwm()+tAtK_t%1p2AZSPBIV8_@hEjjv>ZpRI$ z!`ln1LP5N8e*vp7usM|OC&LH~qhS?zuU;%p!vq`sTJSF#_5+5$5q{x#UBsc1zIr&F zLRqd;Jx{s&S}ww-hHoyx;Eq^-0^tQS`twUJA3+dF>uAOeS9i`{xd>;v1t&b$EfFl& zMaPJaHdRK>robt@9Q1WVy>uOM7~4@#kTS)XU4%bR9kDHDTaj#KaLX~~q1p$JR~OwG zP!ZnilM45A8Y;?VO|(dbWJ`!G?!Dv;r08KH_(>mZBaVT16+C5U(|cEoFrVYDgfhfM z&fGPUkquLok$O=0;{Dl}e+RF5vkWp;ER_EomrWw4xnf;AXU5;b@JREWISt;L(H`QE z2+J^Ctb4&zfEB$P+}Uhki#^G`3u^IMOI$k+b5)2lgvUiOS&B9L1nofNvQfCU41B{_u2SChwF9~Zf4zC>DiB!iPv|O zztns(a2+_`jW=bI_M$!p5d~_WF^ra%98A0+Xu$9}W$2nAG3VnmhlNx0(6>|lu%x56 z=Ow7Z5~6(b{e>X$VDufN$H8fq z$a{jr<&}*3CS7NjD;IN_kG5YWXwh6Y1co;XkdS0F z5bEisRHivdyG9o0mZ;##==!PjIMVC1XO5bimU^LmFpD>fc~NE7^dn*sJ=qh@g$jwD z92RFU6j-Q_Hg3WT)jv;_Q#`bt=%TC-h++6or0VLdbBT+;0f@E%?Kp_22V${A=zqeB z{ilFbzrlYCarG;R*Ixm`{tDOicR;XzVgD1s`CpKKeFH*Z{cTVI0(ku$qU_&4|Hk|8 zA;5k&!~8j@+kY_~{F(iKnGk-3QU1q<0EC6+17eT;9VQD%F9U?hf+DYkh2{eA|5tIL z{}?9=NI3iFI9Xi(HBc5H8^3t}TQbSNApaOp{;?u}K_DO*$3JES82ne{X&|sPkP;pE zv;>l?N&=~Q0J|7qjR3O68|z!)xLmK*y5;g#E&ih0Tdwz>Xs?A?A2WU0;iD1p4n?eV zHm;kThvmqa9X>phlh=}=b)ro^zFb_bI$uoObpL+pxp;OwT4Q2GjhmuQlwF#dW6sAGGmA#( zUYzkMr%f-cGGNbA@qWpGUNjG3;AjjY-)t|Ykv*_9 zVen&vo{3>ZSe}ig?jny&DFc^|qq5=$p&hw*DwMf{Hy1JwACwmqJj;^^>e#-0k%kzz z*DHB+Pll~GPSIio?RyD?VVT|_D>O~+9;-68(brmE*>er6nn#VJc0v>T3th&I%$`!x zp^tRot9DH1Qy(90p5?EbM0%NOQng^Iw9IlXWy)`75G<3Ad|PpT7C^05edAKn%}keK zl|Ig~&xhBEV%VcS<6e+Ij-+$?dlMG3oe%Lp%^E($zkQe6FNx{aW9}4U(nH+-(Fk%A zgI_7iVetjU_X_6K3}OngPyHef@CBrZ)Y4psa@Bvn#!z|bAJiwvdlNz$84Vwh%wMBB zC*GlwOv5-}&G)x_C5Zdx#GWUmzl&!59d9>Hd6L2K?M$WO&TuMgu(@M2LP%j?KX1Nx zc?BCzrTV$ywKHByQMT3r)hYQvq+hKp-oZ7JhmcWcUgM~AB1N8uchX20*uBZ!v!V2( z&x6w6N*FA?mBl6!Fgd=j{(Al`XZTlsF4IUsqDVca4|7x|%c0(b&-uOl=84nBCmoPC zm8nWYc>3eCK$$2+~h>S({+imYN~%LnqD9*a_nQ0aAj9%rRbSxQAtMGXxj;m=elh8w*C<(F@`OJ#e^ zS4#I{QS?f>dC)rAok+WBx|0R8V#OJ5ve|C=P$`!w|)7LA%P#`kWTl%~zw|gU}^z#Z`)P~HH6~&FgXSw^W?aUKT9fJd-24MSw z`8oE8;Z>Y5j`f>^j9l_{VqG1s_QADNSX|CsJOagIk2QV_&sf?G=i;=7t1lKlhCD5> zI=9bnyA-gW->o0qs>+G-dbOa{kdS+t7Gnr_1PJ(vdQ$NRs%mz!lPnEi7A6=6B*{os? z+xZ7HChmOM{Ziop9-`k!g_RNxzE25uc>jg|O?GS?-;iYV_vjgGHpix_k8glyLge4lMkxca=i$c+2Vy-v^G9ZU#!xq(c5M3d5X)krPm zZEGQ{@(pwgAOOPlwevP-O%F_k$9`f@mdW1nH22Z)CBb{lidwxHhyA3yB_g1MU@WFli2Z8Wv1#9TXTA6oQ4Ic)jE}0t(e)0*nf* z6SUj}H7%Wu zhUgQxhOwA5|Ed!jnw2zvIHywY_RDeJ{AcJhJc_f*m1Ca{&|=&xsa;D0@if%(KRYPJ z<SEJW*vh3sJ<_R?hb%MM0lgp_g$=BjdhKyjTZzmJ3k_7RmT| z**N`KRIdvI5IcSwi$RoL#8Zm5Cht=hRZkcdF0PKI=CQRqz2!(lGseUHIZPdRUR&C^ zP#*me&A)4p2;VO%np)pdrpo`g+$D?d3+1Af-S8vcO#2)d@l!8iNbkcp&m^o>tIG(W zWX}sUNAs+j%d7^)EreR{@x3F)b+np~6SdVPw@p2c?RiY|GdZ7wC;5dP$mI;kF<)s(o|XPK2Og)&_GLyTWg)$x9KFP7s@4 zc@ArGJ6ytL=bCS(he$)WS8d9is@bY4LHFfLf+QyaEV41uhJs(n+F0*(sHT@Mk-otg zblPX&XN^EXT3!K8JU)Zuae0j?#xJc{2HFh2A`>B`j z;^8VT6Pf7fF;2rR0GkZJHm>lqb*;-lCn$K)Jx?|L`GnA^rHYLguL2L}^WH1Wg=YGQ z*|dB9Dqlb}n*;vYn9Hj>;%?H_;Yn}4GuF1Z&`sVjuRlG~N~5GpKrhBHr#i8c7=5-U z6#87?wNd49`nTLMaGJW--K>W>nCJB!=a;KQJ@*;Nz18T?2k4JqKE^$;=gM)AFB49) z!@^D1XQgbFohD8xC7v#`U<(_mOD~Lj3%ixx@N>q5?TMQ@4ZBF8R@TbH!iSeyD!!&S zYbqlg9tsEQ1@-rT^>*HkVD6J3k~G{R(DB_WPNB4U#8U9lH&LrC&H$QwLbnH__@^(SoAFw>4V&_zMjdx3KHf9h4s?Q6smx6mJv+ zeua_0Z+;%D^R&>Nr>5I<-WFWL8TpK>rKiVdlUinOHh`Wv$HgxQpDI?XgQK)26_el4 zDs+1GIFX^m4@~)1O5n4DNug#3!wF~Jx6#G%#dD(p@?)dPyVDh(8wsC%IW%g{`Q;RU z$SnYulQ)YMXQ1+4?{40FqKTC_l)gJ*(V$0-Sp3?}BXF{w*AlB=y5|1vEc!hpSrYG) zN+Soam_WVZ&@ej2u{};Z;k9@>_sdYvLFZogX^Fy(1-9e4P@Lr{L|+=2%~<+Sn|Fsg zcquI-u;Sw#f2|Zc&Fj)_A2L@M7GBjmnRKXt{eLdi9$1i0!}#akQcrrY;m?=f>YsDR z=V@u)iVECQTv8yl4)S}ytk?Lk#y~aTlc8&~rbts$m`*YUrd6N;om_-qt3xY8D^BZq zgbbZKUEK>R|3v@Wk0EL;JYtf`;=W8rt_dxXH@~{JqJ0aPX6$evpY_5ZjDAXi3t^Up z7*VBRJ;KQL&wFL*x8_IX<3dM9ClPJ?Lh#)Uvqx1%2U9{5B7IAoGETZ1#PD~xhU~Bn zv*-kWfgWKrCijij#I;znvaF!)w(c$!eB%Q`xjS750?* zLGUK}a-;`F57$fK+d37ZE>-jM*CQviWYP5y60PDZE`CbcbmS2-FC|{oP}O~giC(M1 zi;%g#!zxO1v-RUOSK>BJ&g&f{=|vuQ!yDR*mqD-jWU-({h*tzA1P!;vW5uwbSo#?B zopiqBzMy}Aum1X~P|y_E#>pa}tx6~(EN5lvVP2#J_K?z)`BuwRsIH3?68`DFFpim# z$@Gnuc$4Pw`OqsMq)MZesS_|4?46J1@l9Avtm+oJr+N;;>o0mSzfBC4?6vB-CO%pH z%wQE^GJC(J?ZG`oDpHjIOyAugTM;GZA=y02FD>?hVH^Q+q6#<+%-6x#zrLXv3=NAm z*a{ZjO8tn=&;Q{+KmJBgn4`5}<-M+SaLw>svg@JT0in1qa^6VhJ_rAtAElnIkf(30 zfoY%pFg4SJvU1JvdsrY><=%?iv`1H?KSi>@)oS2O88}DUx3EKb%;_ai;JR9L-I>z1;;#oia)^M+5(^K01C!F{6>fd~>%Xw{DL#{S1DM2^xeYk zCoZ+{N|e8s4H<1*$+MPSp2R1S0-iH>KFW7t`0A6~(YzR+U>PK}<{xQstsplIS)6L@ zDPNB}{j|N9Y}eMIh) zBr-V6h(>Cd%RgM^wBZ}Cm>e2bw!Y@&;ZiyOCZj?of7M5$7V!;!;kRRYx`4$~Iv2I> z{yE%xMs1D%TKDbC&raiz*(Yz72f_43`Iy?Er!mn+#`CTrkQ?ieuiO$pxY5^u&%YiL zU%6r7w1U2voed)fFZD-)oGj)8Pr_xI!-ibXwpk^{q)kuW`^`R2-Q0|X9CE)uEs^?e zKbpfbfEUuw`4Nn5F$x2+S+&7ivk9$ii0AnS^u=Wk*W7w3|au<&5pvoRq$%4x#x z{%P|q5Yj&&F(_6pejtt~CKz#x;7c4iC<(swSj#1vG%4NB)O`9sd>#4l*Z;oy58!hm z4_|BvzO;NfM9BTc0%p2bbCl5Z#*v3g==@>QA#BsT&f972M)?+xcO`q;n(oP5SNB2q zaFJ8us8?IBwE?B0n+T?o-Nen_1mu(3%^EY__|0shKc6TR*-csIRa_S7FOZmcXN8`; zdI@^xvqPcMr72oJ zd?#WSpCAEA3ipf7!xO(vpl}k)0(@+&Wn8WoQ>fhOkB7`B3VC&ke4Uu zfByRc{Gn^{`e*!fnTbpOKVQ$Dq8l7xzri%^FbYIUW8lRcd^l{De4@!KPBK|@XIHa; zn|t+vm3vds7-49*C547*Ipou6$K#KR{;-A4%$P!2e#=PDmfECklEE9Bq4-~1_`{;q zLv%}*xPP`T9lc?E6zpAdSVm24aJ2cwaQieL zn5Q8rrqRp3bJ*J4TV8~L!9rGC)u$_})J}dciG@?Sm1LsAX}_vPYALpeQDJwMq@hYj zd(vrAZCT?8gH(d&^$uf^-b62~D0D0bi?8yytIvLZ-)sKZRRQlcI{B-Psj%A{t$%({ z&Heh{2Wk9N9ha^1K^s`u9YVfTA!R=0r_GGMl|n80cKCJbBHe@ij{8NPd(^AWo{?5z zlRU-;p~(UnI}e&guqAA2w%4e$caxJoD<0jMv~P8$e&mne;Jq0nnU3YH=)Y<7i1Zx~ z9T|yF6w^_Y5Lt3Q+0A56aU7u~3pFa?v7L5Z`408{ z{&GGiY!i{l)Os9ek_3ydg|HHp1^+Ws$Y4~~<><}<|RVXxW` zaoN#xub&jm&tKlOb5)@D!wJVfSz`a+r%RFkU!8FLosyI1Uk;j(zm5ceLy!ODp!sjf zQvN!3`1g~5zYG6v0|#=70Qo2WZ#Hk8h^}YNbOe5f!p)pEWMpqe$!-$1bCDG<;VYhA zlV%Y!la7m(n)vdp@5zz@X_CFLP*}~Lve5C3ozPd*p!&HsaDl#IFlc5?u9O;S(`T0x zucJT=X>9z$tnC($#>#YU&Ru%2Ng$C%=k9iaS>alQP?zmV&22UE^1F4mV)8$*^6RJH z6}rQ>n>n$HQx_+UCo|6zv+J6^(GWiWs8A{rWocU}ZTJSai&eGH%(3>_%ejlkm>?yY zVv92NK8Hhm*ySp5smHc@n{{WE;$g}U%52@e|l&F0bR|d`8VWufdLYf!hy6i|D2gX!O;IjZjo9|cL&vh zs>?6Cv2mhiyW-5)jfwiPg0co!&(M5ty#?KA5Fq$Levkhx$S#to&5xj2J>EDy>UxiS zGY~{ZH5cs>o?H-1nyArqZ-H*D*^?(Hl>=Hy7s_*EUbPTlA0(6)C!R%Y;{-3vF@ zHa{I!)SU}V4ll0tl^8z>VSY&4)?X?-GOnZ^cSaMxQ(j=^EZ<#=ZzcA`COkTFK#!o! zedM#vs(c$cVJddPDBOycI{B1}@RErp*jizDS*3Kqj`a~lFWgDGCU>#>UE-+!;3sqY z(JY}uw$}23rS~6q`!?w;Dr7zzm_1|;pL#S=>uM1mLFbJbv$MhVspo=<;n5fA+xe2h z5v)YUDdhyimymj6b6@KDm-RnIYWF*Kc^;KFM%w6XzZkd}P561S@)58@#^DpbZ0xACGMl+9_Tx|t;<#l!f;V#0?RHvPl|;mA7w5W}t>n3N z)vx!@?~ojh)Ob|bH%K0?VvJ%W+v?eV5$ZgaEvrD*c@r7;D^6|h9-1*$9sAGCe$IL$ zf3*FwS8)SN!9R<#A2TwaV>ji(yktDHD$xVSP=QcU<;eH+yBf)cbI$F+({a;a+?!pu z`sj}9)@ba6riEv#yF4iz?iurI2hp&M`D-VPon`=jN_D67G4*$M*Hufp{8CuPbka}9 zDRVhbcg|lsq)iHu9Z;U%86U70+@RF>j!<|PE0dq+NDD-$K)Hq^LU(W?@K193h{WORgZ6f>ChY3EUqj4G^9~I zw$5S2%8lVY+i~E&ju>5CD_J)_AaZ{*Zpc%*$oHJ;aTpIDg!&E@p8xpN;t5%`Rinp^ z$I}ap=y(Ydup^lV?2R7%k7zt6QjqsB`F3vac8R_D;)+2_F3A9S$){g}OsJQ6(aps=NoM5}usH4zun`%Q#p>2CalU$-CMt>Q)a4dGRyPn?E7i@$R#IW3>Y z!2I_b3yOrFfHF7)2gmtZJWNJQa5FSL9}gG9Mlsz!3yi)3+kj?pTSN&xoo&Xl56hm7 zY`-f#8(MM8z7S!XIP%Iu;@jx?CsFUhI>b^a zMgw~O{u8a1t}+gxpV24YAdOnva;eUNXQ7`h3rO{?N$kr4s4eWnF&SAXvWcgd%U`?%nk zXicZ>5@KPTO0Vh9%GRZa6hG{{&Mysji)G4o4+C4SJe-9+&w;s1Vt%MQ{zGkyQsb`8T;+0_mgJ$% z2@Q1`*^*GTKPI2`LHuG@g#tF^tm-rA+60VaE@5VG6@Y$PlU@hLhPYUm`00||k;rR) zpE0~0jY|K+?Z&m3jW-zS@7sy@3*{)wJDamYOXD^^ihXbhyGXbQ7k--gj_I(KryUV{ zX{eNDteLJU7vlD?&aaxxf`Xay`;63Blk=|g)bXOAiWZAsnn0RBx^sHe4uF|4dRZ}!QEzC`-p za)B3$>a^unz85&c77lb937YqA4x=}g9quJI&*|qZlh=8aym$XWilB-vdtpp?$`{T$ zdT*Z`_NsOx<4?ts!xB2}vsua<-_a7h)fRGY^+&CQK1;5?KmM`2tuWgrOTg0I>bxA^ z^3t5+6HkRr`Q|5lhH^rqhIFYEa*x|K-$%1U(Z~)jPE2K^gvY$E0+idqz->@8a>QO9rjPSpi<6Q z+Qrxk}Ht|AuzQ8%r z`_8w@sigZM)pHgBEkwB0BE^(bIADgfB^ud|cvxkW(`yO-O)6aRrlCMtsmg4P*F3%1 z%j6SoHiA9m*DFhz`}L-HiVjTP$BUmIUwGWlVx@h)k+uJHUxLJu#4$C0AL*PnBI3=M z?(HdA+D*MUA^F)-k>VODmP-tsl8!|Q^ESf}-waWLL|{4JS8S$EO%u$S$7%Ol$%iv@ zzc4+lEp+7!aiJs8PJ{kPexvua_Tx(960Mw3SL1u}OU}aR%77Txrqc`eSy<1*(5@KA zH@*+YmK=bzkzvxi85%YBJvAN2mP3P2M=i!xT6Imgh;ABP4-I(yz45e`ZLX&bM0_;F zsb}&)T4f2jJ5f}>p!5o@fZoHVqhbQhzDl+%?An6=gNJy{3iy>49c37uV|34@?wbWN zQQpydMJfYf6VCOckH*qKt6w0nO)ZP>p9`_T`eSP2rjrW{gs5CNtaF5_a(Z+)Zc zb`N(Mfv#5+fvz1BR$&c;uSPl}wZH8h16?Kp-ScMp8WXC9uH^8o5#YtosZ(_<=sjlD z#E&vK#E=eBSpI!)!X#XT4DfFG+9)xu@$@hEdRA^0Tg4QdwA4%!wQi@UElf2&=fKlF zN|h#Ix+7DhP?}Ww+7|1WfC8I#IW18`dAcy*J%5u3^DC+-3^qEMXfX$xdvkg8i4?Cz zd~bAH+dl&9YzST#FX+K6vb}o`m`B|{ZvIZ#gVBx=Mz<3RLoik;krW3Q?#*Hz+ z;U21dnU!?n@+Kwk$mrTO6^Ms*@f`+LrOA&?DM7=FUEj}w!U+d@bi_9sEU%`va zHt2a}KrSi$@cbYn_TA_9kB$#= zTrAt}*)MjA?H4FiuYISPt|EBgYKD__R~23Bb~UyGS;a+#VaX82v({xw})isLGOjHX99f_F${qS^BB_{$glF@xJ%gTrk%_utlAxRbPMJ z+~!+ambYzvdedE_6dUBS*EAq@bzOJyqAs((c1Kp1ae2uQn`9*Rbbbo-_Hj(*eFtlS z@3*+|kED|1Y9SoYpcgwqNrqBqzKPGmNK+JcIsN?3E`33JOsG3F2N134JMP=po;!~*;RFI2QK`tK;eyv&TPjec#nT%H@7 z0M3LE5EKXW|NPLwVK69+&V=rNU@$NY*rfjYp|k!U7#ISF0RDl$!GJ%2WT01IFbD$h z3j7@p$qSeQeuqI2yr8QvD2NY8g!(%kFB}ZNiU)%6qR;|CfZ>46WdRq!tNeq&c!4eMRXi{Pg%(~u z@Kqi_c;P_y&?{wmc|oXUQTYdf@}bfUg`>6&$_qtl2lOvn&+l!6@Bzt1uh0TP{2E4A z@c!^V2pEY{2bd3p!e^ivSNjbDSO`$@z+e;@2#H!21P7ti3xWdIkgw1T0wJ%~3;2LS zueJ@$2Zy1;pudu+{>B5~S~C>oeMlq{1rG@aUzHay5{lC2U?dp0u6m^o1Ru({1S8;x ztMq{Y&)TanARR5rxCA%=zuLcG2o!K>y;3hAlqfVKcmYS-D|kTa+N-h)hVwyD%ED1) z5@;=oOoHKH1WH*LA1ZyYKfDhZgDCuifdQA}D>U=+q4p&&0=OM~6^|E&!WUi$5`{iq z5Gr4w2vj^M47I&b$gkVR5mR zIthgqUX(EiNIi=DfuV5V&i>VQAiyX*h4P`yCBV=}Ez1i=nePEz`ODz?JFkFw5~aOB zx1r*}0j+qoEbR5##QRXl(42hyI z!Eg`^rH_E=1%>zUKhOuT0EJh8nHCiW2U0;_;T0SS`UC7&67%2Nh5*4(_<{hVtOdXb z$gi6jSIYjCvi%AShN6#v83(nG5D3(9gg~Owf&>;4SL;QBQDqVdL88zH%$+EFM#3Oh zbrcwhK-DQo1RqMj@qtk0=U?g)g@1sGN2!AkiaNG{bmS;)WW=Rg3xfLa!cI<^3{aCN-~fdLCF zlzNfB4s(B-6CuF#hav}nUO?d~99YC%EenUD?3sYki=saO7#y`M0%dN3z>yG?@e6Dh zP}+e2p{$1>2necNA)u(^7Z`#lvWo;3_*dEn1(YKy3<B_(b4%mrIKR%ARm|Ote{J~b zcpllFS2LJ#7 literal 0 HcmV?d00001 diff --git a/plugin/src/test/resources/org/opensearch/ml/rest/test_data/openai_boardwalk.jpg b/plugin/src/test/resources/org/opensearch/ml/rest/test_data/openai_boardwalk.jpg new file mode 100644 index 0000000000000000000000000000000000000000..19fa158886cee1c1ec2c0452427a1fcfba1d9087 GIT binary patch literal 103016 zcmeFXXH-+)*EJeIKtVyo(4;7!D53WnL^=pa4K4H{y+i0CB8WifO^Qmdks3Nk3%&Oa zp@kZHhs(b_&wIbzG2U@M|5lQdl|A-3*>j$goVDg!x0AO^q&G4!YfAt?SsB0z008a- z2ymVN@a{aEy9WS=27v!>9{|wCdG^1)A1wq9|x92@ZUNA z{*r$-|1j_m1OG7a4+H-&@DBt3Fz^op|1j_m1OG7a4+H-&@DBt3-wfQ&0KNcl{%yE8 zcWF2-E-v0ZygR#xfA3$0Pk{e#Blxco{@d>V%O3o%{cHET9q-*e9}wUZ{O|bxwR1an zm)A4jHUr4+lSY}>Rm(LG~C@86@ zX5-5wSpDJNlV~F_OKKCJW8LKVPIhlsE5NU*r7S-| zGE)zR$=uM~Z{KN?H2>?U=d!izZ;qg@XtcvwS@fPzFv)vitm$Xlm!MCz_0&U-W`6y* zjC*X-xduc-3r5n#wV=3Gytl;Q^)6*t!7OgwlVSU$#LFa6)1&oq^!W59cV`XtjaCg1 zT@cdeA|TMk7-!8J*ui=R8@Z-43T!vUemD1ElTzH?DRy7&pD3QOSJU|%->ly}6kbEf zKZMy=D(9|E`L-A{SquAmB=SwLEJUSJ#f(D$xTP`e)olD(b>oeG;uK@*Nj9x`gcU?8 zr7pr`ovMUMtJ*yx9wK}V8se~kml zgs$}Mi)kWt323 zqaCX)7{M0-Wt_98;>+?E2j$l4sj=ft!%9t=!lTUvtzuG^5A6 zV8iRQmtp0$9vaC)7iuQi@2e?^C~802^PM|CiH_FqPBqJpbaRDxsCp!aI?$C)S8siP zvqQTIyMPqVzqOa}eB6vN*tsH*5BCw?fx^7_-Ve3TU|f_FE7wQ0gBxSlR>Zs{6ft5v zw}7dMpaJ+QcHTVk|W(&h#}kiI0v4Kk(Y2i^wC}|?wHWkdT6_6#1!6Npn1jz4k3+iVIb=f3_Q9N zihqfBkjGw82oh|`>%@fZRN7m1fAok{M5in|ax#bTat_Q4*I$WQz1*YIvUx{3N3my) z@<|Ky6M@8^HR)GGlHYUC!I`c)Ci8JlX0`;Jh!LS3+#0}Us{^d{5jQa0#t{6VFEptG zEYx9f4U_2-u0{2~c6Igm4C8wyKc+K|afn@#F@e&R+z`Is-x4$GQ7x}v^%~?vb)P&RCJv zF?n=~;TV1|upXW%VR!HyxgI(wh74EK^<`IT)Op@7gs&S?kcNRq4R#BV4*K+wOfR!Z zoi(mhd26xRm5o263cnGPrKQxM|~bRjf{j-Z4xU)U>% z?w=G@vs6a}46RYo!SY7@_jvYC$p-YU_Q=DFb*Hl`z~k{WZi&@!1*K}S@ZXOr75IPX z$R*FZFIVd?Job|6M&nkqtG1V|cSrmIrF>Oi9UN~oqe@?p2xA{R7q?9tt$$l8*922D zd?|o@L2o|1Rs&USBc}aSR8MpC$BQ_2SU_x6)|skUch9<5(wUMc)teslWY0)R>N0_) z5~jN^?acCo;zC;n6{`16tW00LZenO?XnUC< zaQr&)^)Eh?X@=MI!JT@OBS)c}Ot%2#TAlq#`r~DEZqU!vY;m#5E4V5lt+Kw1oKbay zdjSRXWM|NgKgNM`J#;Z%oEK`J^X=DkYps5U;HJTnCZDfy{Lb1=@&9_1I9zB(O+Tyo4jUaEhTe7m>Ky+9R@AhU z6H*kvBgG1Z^LTw0#Z-bmR|g@15$vO#lhI|i-vS;R(C&x^#9Khm5;UK$aATmOW!gvb zYxzkp^EZC)GzHdbQFAy92rt)nU1@oqn$eFXdM@i90IVdI3@b^VWoY!L+tkvRO^~NU zKecQ9P}3ATOr(P$fPQ+rcAwTGZN8+Wjc@N>6^`b%6c!4vr@G0XfPM@JxxS{x=_@)|A4VBluZL%ck?bUCmu+4SXns(Mnl5pX+rv8?pG2Z`q)s_pzm*6W>W9tIjw z{3+G9Ed!%%Hq7|#q&@^3ba$ha%-1Ttx_)i2bUM>1&_j()=kfT zv4$qa>2yX}m(QDKx*TIu6)A%;{hP7`b6ns!{n^=~WU?I;5;!$|76{gg5gRFH@gq~K zLP^-?%vK48qsMD)^#PO*J>c24dJ1$4_i!-KmyL;Jh8;B)qPy-RxuZQJ+e~V4b{P&J z=?&hdJ*IYBJ?o+8te-3^A$?M%7exKnk^Rd2T&AVz3#WD>5V`D`ciZ=w1+#S`+q!?m zxkBL~MNINZy*%g~A8jog=E$(8&>jJS39v39)m5RWT3I}~%V*LYABa0%Ubiy5J7s8U zZyq&Wzfe5mIvOupNN@Nm!$iJgbHEGobGjCjZln-z}jw7^LL(*``v##YOhoh3X6veC#uByj6PlSXq}pOI8Z^$@c) zj|N1i@9WMxI`S_|>ypN!d3= zRrGx=4bJhTdgihwlhEzc3GKwZ@j;1&x$ig=4Z0Mwm2Z6@4?vX@?H>J*sCdf|k=x+g zKv3v>FL2T3=o(LF!a@3@t~M`}sy=4uRChp%$9@_$ST%m}o;2cT%~fGh<;YWghf3&7 z1Bm?Z3Br@+y0B*_a;-5I+wt9kO^Fz1r`?ytButYYHMV-xkh2~0VXxRiOd-XsQ#)Qx)s$v2_Ar!YhWwyWXG>O0TSX4q@uMe2_>9 zL2$*!6%EzbYK`h!z#~5`iQwP;W(Md3pm=1&(nhW>%XwM=>w4m2u#wwlr1G<^UkAq- zt>wEuR}F*nwrDsq>?~ve0qJ9ZmC-m(axqjnek7ni z1iXEDxG{7)H@MYgKBfHKr9uVL_svfD1ZBQnUpVh$RY@qH-a$|IYk((zS|O~ZkNk7? zCln)sSn>C8hfdhh#fDdBl`RNf!IMmC>qM?ZW-R=Nf2(U|;32N2$=x51T!}}cxDngc*<8iETzSk5qQ|_mE*X%(H31p>= zA74k%6ucf9-W`AaC+6Jt8sz0Gm)qQWn%#ZFzvgOFst0GC@>kQ9Gti9npBD$&P`}%> z(BP|z7LQ?|n|a&L6w#2MXvjY5P!kNGU#pK4*Q`=p2MV54^c}Rxw|k&)?r(KFU;8U> z_RT3lN7dieNGuu5UPScfzEWw*CO7O=ffe->rlt)=8^+^1gwWX_$H-%b-;834 zonUaInH{NDB1w`N4)*e~`dNsaCJowIW;N8!DD9Vc4odj(e#mM%s-p)vUz6bZrTuS- zuFHka&}vc`x#2+#(i5$FTv#dSAcUqX-I;C>l-y?sxe8s1Z0&PE(L&hmDTD2chO&wJ zKe`5FXlr}F{c?CX_~*nauQy7K${lnmBGmc(*j+DhDnSG?9bXDzWTa-g1J>~-+3xIjlbu-r7{w$r4{tJ=COUxR8Ekb)+{ zDBeTcDnqcXcowR&X_DVbc604{; zPcStTcKsum_v5%?&WolW3VuCOhed}37$lJZGR8`1;z>2VD$D4|O`newSK%RjJcj-m z_9T%{iGS-nHufg#4HZMMk4N+(JmmYTf$ifuvoo-<1g*2aGecMsZDn<}qV|o}-S1jO z=`U5Pdsf!(`hnQUGbYJWUqtq!m}x5|{b9lC6s zTaI!J{CuIc(l+&k%Q0KQafw=bW?HqGXaU~({c#V`%~hJ7^{ziLm1=?T2wj2i5I6du z5bUNF)!piNP?8}21UIWJIDZ*?TpkGkC*_}D@!!eexi!MNm*Ot8NXzAQ*vNt|Ci3Ne zx0B;Uv|{w4KK`-qeTyqz=vd5L@_Gq6K;tK{nsFM0Tl^|0m3!lEVvO?ztXDk`ln z{M*iwEd^u0^E}Eyg2Ps^r?vM?XY_C$327<+(t{JYWhi6Nk=}Wnx5Qja9c4XUx?^Xx za0@8b`_q!=(ZZWDST@KUQM1DD?hNy7la(8sc=X=RCp}o?QE{XRyM^;9?5G%4vB4}^ zO|+4yY+=Y9rk49$s$y}1wv#S#k}O$M?HL%B_qj1?YtAP z57&vUYfEJW*jD|0M5|pWm&K-jQQ3T-^lb+K2?bQZ=aQU29`ZT_Q zgX5(1kU9}@eFN?Z9D|n3eB4z3nQq<8rh5% zc5SJ87_s8K2H($e+b`+^-*f-=#_97kuC~nLFBlfy*S)gJBHN-#{8r3x?R2OcfMYv#m)={ayN~tb zVkP{WOMf<-tfprNh*&X0V`8_(jB2mXPdM9`a0D&fXmW%C&fUcjq({oHzeZ4k3@w?$ z|L~!^LcYx?edXry9ZQQOmK{76o#PSO<))Gh+|!TxST_AA!)JElrwUfDWE2m#=8>F1 z*vpq`ZxK#HiZ$%PMVn>g*nvu}sojMq(bM@PnE zmXMV9&qmE%Mr zxpgIATO_*oFUIfzFsC$P?Oo%T&Ff=B3Ex)w_#w71inHjH-u0{8p?oncs^&)Z7f_~> znm277xKzhqpk<_CRU2-S*6q}9TOrCO9$Tn%TORlxs zDX_EPs8yl6)~nO?*8jgt9q#`c1nZ>fRZgjB=NEh|||;L~fNsMHIq! z^SaiWmtM|hWm+kQD;I93k)YoWEn2i1g$x@DK?r39g896J!Vk&R=G8?)#*d$c6WAB3 z6&e{`g)gc3q@d@~-M+~o~$DWi&<}_`&>qt$tZ+Ms7D@6pSI%$uI6(uCo3Da>u_^Oes68|AXmS36&?Wky=#M|ihg|&`<(%tBg=fKf zn%AjratMR71&!Fu!yNKV09%VUo;?rBe3pMeRA^^SFLwL5|uF>B~UvLGc z-4~wJwtj4DEi6%w$xc+SrjEXFo&bIm<6h@g8mTm^M@sKTcU{@!>*@z}A~Ju)R&VcY zIE|tflo)P@uU$nLi;{!oFKz+H1=f6s*=0}7lOocco@Nj7@W@JPAD^q_EW1lL#4eE= zi`&XTyOojhgLUNaFUNVs2Sw-n_l*to%x|79>M#29tr6G;_a<{-56WxOvRYaSg$@JIr4qeWY+rI(2a@4 z%m&$e!nYv%ZRRJBelAzBxpfimnw&$P?)P&KrN!mvvWL$&lIXr&5Lw5TeC-~VS$m4( zACGi2hp9gRV&>@wE4HdzxTwMLXwz~-`FShk*A`c>PHI(RjxbTTiXS)h!APz}ZxGcj zKpZrCw0D0}f9mS{*LG8zTHg;AkSNe9<>UPRA@H}=)aW`fi?)O&E>yPE^)HWwItg}n z@Ay=M+Hma$$nQO`xr7i7k-|cA>B@S_qD;)Qvk2LXVZXY2jt;Ed`t4umCLxps^Geoa zQAc#3!AyRZ?ns*QRfv`Sb*W8ei3Ck&-Gtr66A;S%k?Uu3F&_x8Yat=DRSYutp^Fx2 z1=kl+lJ=&kLFkL{DfGcQnu^{PDMcfvlbKIz!^&oM=QbJ&zl~S)z3X&Hpxd77^K;Nm z)W|v42ODv9^edhS_u1^zCPc_<3{E7gy4SkWOHyw^`{v;VHj1tZD3j)*y|cHDUwmdC z`_(Kr&Jb!xyxikmDSf&*>dgeN3sFhqX)7mb4bRHbM> z6_SH7B#If`{iY(6*S-%Fv5omb_b@%H0mJkidlD#OZ?3+cLyFr#q7V9IY`h_lGK`;k z#5z4aO1r}niLVvyHip;F#1v_}Mi17a8?ax}Wa{>c!X~8Aa~!{savsi!-5$@DDS{I4 zhChTr3ZPSZzv8}hA2l!6cyZbzu)|F03+&!;1uEXXhpY!E`D8-%@sa0m@Fr+UyK|8l zI`wjuPXz8?7JwQC>m>@Hzh%sx@mU2{LmFWkOY5-?G&=y6QIhz9BxjCi59sV&5yY`t zCBst;+%TV|`G&$kv9==PqZ_sW*5jFgueE-a(USI!vNtR%7m%TW zp(C(ol0b(qvc$o7Zql)!c<4&PjDWh#-szq)G^!~bwgKkW+@@4!&waFkw}9`-k5LYP z!AXT97zvwO06d}j2&j4H`kpx4;RqoRmffJNT+m%OEJAOQ4}(LJe{3HJpVCeUy5^r1 zGtJdslH@dG-jL0Zr@y8Doc!v$39fN1sm&?&JeaERHNCO{+x!98H({@R1fJZ<5bsLq zEf?xwF>7vVlTcbq1*k>AeNSP@L0i`O=E??28c?k!-S6 zM=^Ay5q@P~y56qt^G?Xlw{8YLl!!t|jJoRscn(Ng8jAS!%Ju2BJ`P9bK!?6TdO}Qb zP~x_sql4$cpJr8kjkUAa@*P$}7w`^BTs)9wjn;4JKpReRwBGc}c8cCitQx#mb!9eX z(+2Kh;}wb8WUo%69*Y=jo2N66utU&mZkBHQmCf zgQDas4KDk_f$;`CUx_#D{nS>zbxau7l8r%zy^{in-VsN2Bp9@&Ag18MWG(qZ z8ReR;ynHc0{3nH*n)FYI#Kyvk4DFJNL6woWTmcmJ&0clyF^4x%d-cB42PaDUczsC7 z-zqg(bhM%xlz25TwPYK>ZcpD5RCL_>@s1-tACLag8&1!vxP^Hu_@p`=hTs zQzdJpcsA-$Vg-XpEXVJ?Qd1Ov@Oytczf75s{2_W5#!4nT+b~d?k#XgASzds6k8}ITVt|gB-y^WLo(=(O=QfGn4(Y$ zHB}McdfK9nPn%V!=XNVJis)1u=~%$b|L5^y`{OD_{qQ2giS@N^SwNk)V1JoT~c% zmirDCGi=pngXm&!kK~Upro+zhhHJ|2C4_GG{eyi$>9e4@5Y7B0AwIJD|Cv7~+o^L#I@(dkHT}Br1PjTwVFq_{1 zhX%4{dIQ(+{Xj!&ZqOW;ySiZWzR%XT&rziOs7*m0K7WHGrfeUrwLpMN4b@SJ_Gv2@ zLo#kEWXOb9@#DU;&YS5@>?0+`1X%qBj2V(@sJH2H2+%4MK@T^UrguGu_h=H*kNw@^ z>Tp>l%vHQJK!i5-!SA&B;94O^(w^+>jNUrS8;&s%ITPxJODY@|g4YJdchTquL%Et8tOFq=OneezeVY0D$;v`90;orU~u#EXFi zmcS##bEv=2pE5)em7CcvZHG10=rYF`~f=%*5$jFo`82=_%XNt;bpW1Fa9yw}3Gt zvJ9tmrbnC$#neYr669)#XBs~ALq5w}A2VNnc_J=c}IeM_Xm+fgNnn2*RxdU z>y(RB22#mmFE(v&6SA(;P1#SMR zA(e({mD}2y4elO?_|4>+>HIz-zeMq`Lu%s&AL_*{geb1%jJX#EmQsaX5D5dvO1$D- z5lrJH$L2aG9oL}Pno(%2(!~T5e#1(xTdykyN-Ls;{#5W~KV8OCqgg$(PN={V$cix& z@yd;j1~GJ$W`Er;QxZf&%|2d4+5c(N-vB8Nw9xFd`eGMJ_W~rL0lL{hk6~fIGzD?8 zHBN__KMHv%1X4%K4Yi~p5Yksx&zyvCf3O6xnYomG((zV%IR4$^`?J5An?*0#V(~ZULJ~C-JKeATpZRBZ1VfP-RuPyz7*a26QHgLcc^}{w);OK}sfa`j z{YqrHEaQn*z~9_}Xs2##xwBZgfvNzKW9kpW5#{F)qpYh}qa4rHzte%nCzd6G7S*dT zsor7vEbUud9(wuB8vq1Pj#a9vqFa#%ia1efIj#57DDcdn+gbm^75#|)jn|LEyQ^v+ zd?I=Iso4+o#3(eS7lFLDKK$n^xmfyPX?9F%!at=NvR`)dQYzi+hC<>VsauTvFTztbL@8t0{N}1l zX%|gEEdaqrsTW0^`VifqhdiWZ;M*n~Nf}R9T&j<>N!|<~5)~~>UU{asP3zGK9Dwq0 zgWVSy;AdK@e_H0GYUmr@44!%DcX`Zo1Y~3ru^a35F zOPnt)P_*qA(?BVMmC0GVw2NSRJ-~z)*-|U&$DlsUAy{usjklX{3m_U}zxVxHvgTdz z6L4G@#M95JtoMDSbGru{_5FHtbj9&k;A1JGdvt|AVs|oECF_72pZn*f4$ydTy>ei2 zfk5v=KTudYyN15O5s)K*>2~y5wYoe^rdt9WKhR@m_EOXTRfEW$oD<3Fz@TYOKHqYQ zv~8f#JWR}hX7I+jO~Y2!g%wWKYD{WVPNo1bpqY$SVG<%dDEO;4hKx)Tfn+;a{AwNO z`Bk3V>dE^k;{2p!c@jEfEb6b-Ihh4{512zaqcBilad=s2geMJ__daOThsvfQvbN_# zybc3b;2tzS4XqFYA)Ky1nRr*LmNc3QG0VarO}>6uA8v#&D5jP8y&@@^-`F;3RG>7= zYtEU~?Q)SAqJ`UF+>~j&G)}|7No%#o*vV$FA@>Mwqd=9w(>Zw7ub|ZN+<1K$%m6$U zx7I)?hfV)M4<4dotevqZLwAs`74ui5aDP=Zq~{d5>x&Vk@1SH4cmJ5szwmZs%%kN~ zWXRKNGmVj5`!?s_SCxtG9|2t~NX1?sLib+du!CP<>o)@4JURtAhklFLDxYG2qqEgS`A7W(O0=@SQrTmf2(Wyiy;<7`@ zVL=A*sE`B?Hs_RUt2Z6*Qtbx%d5P5HN?^y`(CUu8W^L7}ITET=A4p|D3(d2y;x?36 zKBdzn$k@{1H22{7^tn4EAI(9q9m%b^l5kmMjIQ+YykhhS&Fyu+0x|kgv~Wmb4Cl%D zgQn;$?C!;krL%42)!FwiWDE24Y0a>;q30W9mAR>QBe+Q4*~Fz7#2{mMTqhwTCZ*le zA7RIT#s`;w4ee4Pl8){9%E`a7hwc2r`n8?WK0hAdpFMM3x=9nCZg+y64%`9?3L5=n zkZ`aOh-q74vPJ`hMzMct4R1^Oqpk4q)UqioB%dZ?lfh zKecfkL6V1`Fs9<5!8PIU6X>Gq`^+zB!rsYB8euaZ`!((3!%3<>!D&>vQDn*-Bo2D!7w zJT>a@mvTPz!hVZ08>8H`-ljDpnNk+798h?be(auEemUIF1ZE*jev4`1lU?r4%{wXU z&6Y|xhP;`SK{wK{hcP-KJd%`jA-4d%n!E1EDg*OY0&!wzBN^pPm`7){9gdy1Taj)N z-)Pq&+!r!w>?-0s&JDID4~iY?JP&r8)As6Cs_>K1ti@ihsN07Uuwa2lnQ0`Hv(19y zfFjWbh^j^vv1%LoVq$1yV6*8nVKjs@t0E4b%1=r(HwRz}EoGRc)t0ydi5qV0+sXRD z5ZN#H*|_pF510lR=CARs3~diPlX&{z7Zy}{$9ek%BCJDUa%p09_mA!NZHsc_tSMYqY5ObL z$eIb{A)VEgV&Zx70E4nMmHvvgti65Kj4d&@_{nn4A;;&{=MQf7*Zy#r?!Tq7?TotA zV%VnhUMF#M&Dj0LSetJQnT`>DnWSXF&^ceDG(~crBaL@;1N;WgF!!pir>6UgFU9hi zbLC#c41;mbuh`~1vpKdh($hcq-jgCvGk<5yMiNcd!neP_M1?FvV`dQybJYC0zH*C~ zOmV?EpxNd0M*Q;r4++nm1c?}0*h$|wBwd7^=9^%5bdl$@q(FVQs4`#qN+i|D(f-+( z$64Vryv_<&mhktXbULtFr!53-qKd@U*C7>HSdcrNldzeTg+RCuT7u3- zOo55>&vJg38QN#EZQ_XlL-}1lp(g`2Os^oBr5?u`FA%^u9P*?S;;q`L1 zJz3i-aCF^x$S%r0qWnxdj^=|Q`5Hkb{!Wn@nf{I2aYJ?AdDWML!jg&wK2v zO7A5TV*=;YMF?amPU7~9@F_KPaWb@ly#ubyVi7-m zQZDOr!+5E(3N!;IejNxWsD27)&52$gp9hRAekpEKV>~9z&so4wcw}W)D555bFb?i? zX;uu%mHkQqSx{%TEbOU5&|UlUsUT=D@^&x>322I+QQ34K3&5*f?g=0a9bdmWwTYo;=#@dkU~xgPYg6CEpKLr=)Cg;5sy;zt!0qa(*Bga8nf$$O9k?zM|EbHO-2<}V7>e#;P9N)0dfB>aZkRrHB1Gg8cB-1t zVQPFT1S%FRr2p2w>;D}#Vh(>nxX%uJ^o8?w?2WOOe)y6v% zIGA*y3y7-OqAq*n>0fY~Gvwv|x8%>#9a1AX0g@WG07sh=;i@>eNnqG~4ky)ET6Db7 zQbOrygYD8ivqN;sMEM9CTj-NWoTH*2K8U9_LSfwlj=yZmd^yo|xz{822AnEaC*m^d z^a_l48&fZaa~3*JR7tkTX6$vIl1yR3jyZSiIo5!bp4_>#>%hfnqn69OopC=;osZ@H z?aln9EW!17HgyQcv}>eg6S$Yh4rqf6mc0eEQrrSMFRlb^!*W$d!VHV)f4MHx5ex}T z8??g>d2ADYw0~Vm+MyDVsOxifbjB8Rezw1wYap~qZe%`FZ{Y+fmTIBi;=3)KSuV6> zvuwr)+3ep7Sly9FD5X8FpD0m~ca0QorxX!rVDl-V%{yX-JdmQem$ZaO)%X%ThxZGe%&QQfAePmMvSRk{NVKPK_ zjAIU^KUw{xD)Hazh*k8LF>1R71VaKIVTKx)ycn(zR54nS5mwdlY2O&OO&jXL&>*-J z3X!sGnE1xPmZm>F<0kG{7VJZgmK&f@Z%7VwANg<#f=%Ui%`%O0lPCocbk zlC+l=H@{DESMgj6j+Z@}@G?JEJ)lKdl$32wL{=npBK8K@KoU|QkBhQBFaDSt#9t$e zwhcVWnI|D#KrR~Cn7+g=>v|Md04(m_9nAT_`jh=9Yu- z9Xnt^=qeSqota|H*!c1^noWS!$OsVk@^yBUGK1Q?@dt0_h z^VsHAoKY^nagHO-Xx7%MMzCh@kHY8O)VjEj!93xNmW4;UkrwJQ>bHO|My=ZQI~s5U?W3!=tpjJy0nR-}U$&&qBA8Y}E50di-*<+X z4RZcimDk6cZp^S{7a@7BlZY+)k#~xlcL|v^MH=Y7u_y1(kCUaTp?^Rb?3ikjiJAyQ z-U9B|7e=%JKoiR0F6P|qH+}_7zrU`$V8VnshO&%JF2>)3i;cMH`zUB-TlfZ$+8jUP z{V@ETQKB4ut|fHzada_IU*LqSD1h;%582Qs#)GFK`l#)wg{(aeYPh_cG zxCf#Z;Jo4DD_fdm`9>0NBCIcfRtND!Xkg+B1P@u#I*LfdpV@*k8{tne&N<;;TBWQd zC`1om%9--b{d$1n65 zobTy>*Vqm`n0YJz`l$y&;|wR_6TB5~bss@{^CshV{3eaI%sc$JNclF`TI!XuK0Tjm0Lj4p*V|5 zljZyiQ}MS5*+0Xa<`MGv{tY1AN;=j#-^yxD`XFGeGg_Q=N`Srjg&%0mb{rAv@r5)i z^Vpm0G{>Rmx6P{sM`r_v?4LHT@9TyP7pm@a+#f8mPY2-|eKVD7ru73E&?~_k&L46M z&yzZsv3wXtphdmW#ja&=rVz=!cJ~Rr)9K&yJ{J%=FVCl`hepbQxuSZ%#U3i_7-G2{ zIf44o@N5^)#DkzOcBD2F4b$rYnl%TUm3xTd=A}f9b#9$D_i3HJ=OS+K=h%6O96c%2K!CSe1KD#@P;Q#?cFm%4o5&q>l0 zt_qQKVY!Om^R=^Mj<2G7f;95gG;;4PboqPJwTDUOq9VZK{OCV9JtemH1!Nqm@d{t6 zA+ftsR0xIAGoNKDMqR7gc@-QPF?d_pS&^)LLJLtv!K;kera9XTh$>!^ zqJIb9G}2aoZ}RZ$cz~j_j;+7-rUmEv%?~j@-vz$1{f)^R9mMaS*j8ORnxvJejCztm zLi7!=-mMU3ruB7_mj=tyF$wEAeY$*OghdrpTb3I&`~bhd`+pT}hp;zVdNS;3jbUGI z`_A)x64z^tZ5rNF7w;p%*@-rImHo?Tj^qXEvZKL{l42Aw&TYih-@fuVv&=qS=~ZBs zUIdb)7%urfY)|$N)z?V@aeR+5*xS7aPN>Mv2f0d1*7n>d%3!(9%Yl(2gtu^nu&(d| zR*WU2pi&Lly-wg>^MJ9x2M?oc+wjhSFDwe4?fnw|xn3c!YvtTfkv^D`e_`_K5#)RS z=*#tzXJM_BZ8aXx&dQ9~m7S(IM9Hp%FsBTH3haR{92`o5KMQso9bfcP6m!%nd!e)5@a{hNXrI9zsG_cgS|Os<;YkjaI2QipcYhZiq3{T z*NFwKe_uBRa~JQ$X8rV!?*of=D5=_5lTfyxZlE`WX?j=umGJc$ z1r83Jtlk0=ewUUorrlV^B61a9WSDOGYAMs}hh!B~J(gQa1Cr1Ns$7x=7xNedMUKVt zM+;paGMH!?Nyog$g`FbQ!)F^sdY1|{;`N>m4bpkGYKN-db=z%JMz}6tsyvw&M3s#) zOj3-a*_~wlKq9tutbFIw+IU+j$A6O8p2g4DEw!2Lzj2cR?UB9TlzthwV~{pCo)3m9 zJ+`d~gW}Ex(WS-+TCUszM8*T%p$cF0=o9&hU`w`JSoK`FdCl$mZFow%R=kCkxDWFb z`rYsaroI~O^=y!)TI@}J39-I|+01Z*OTYtP=_=8+N_{7vpsmca;;2nOcAoq6>^wA^|6wB~ zbo!d?)R%uTm?+w{mj&^$X&iMIL*L1FmUNCi+M}zHz+IrAKW>8}>O=>-$uiBk#u@3c zNFqUW(J(wkpcO)Y{KL6$x<$f^(b#8VO=p1WaQ1z}bZTShH4EXAwn9im8i%9C->fr< zr5+{K(!Y;2@u~=K0fw0){zPXUtNO-MG^c9g4WAF&4e1{o)*}D#cg?~^$!79aYMyFAF9p6=Kq$dzlU(T zqVH2B@LGi(|C!t#&P55+JG_d7s-8K8?%aVN5}0Z~g#m~TMKAqiZ$3!d6hiH8II2vY zzZT{D>e`@!jV|wGc=VeNt*l1k@|8nYHyeqtp3c>Uw+-Jyb>jo(XeUJTK0;is*E#$K zfEzqpg^Z2|iFl3E9f@sZ@&b%j8=$Q}!j+bLW~|(`JrP?en0>JGegBuV$YR;i>5<9%Qn7 zRQ8Gg*ZsZI;MFap-KSV@9{RX%@9WPST-ChZ020r_b zKjU7dqIkPfvr!hHqGj}SR{ny&V9fYu5TngPSMqlK5&4E+#aMo8RgJwr;m=#qyg_Ak z0zrR!eHQ}G%Htz*UVsHJiXB)n)RO`gZ*5gS5Tzeb^9baEL+U z=E-oa#T4 zMR`0w73_?wc;$O-8U9sn*TH@tEEe+SQ}_{ILumg1475K4TX{A!G}Cu7ALKh$X1Ql! zb_lqKk$3xRMHM?lw#?$8kat8rG6&&9dB*)7fe~&ekGd$`~_c5|A zaLJ&*dn>;qkkdX1wB*Thr4XN$dwxQ@OHYFKHrOBnJuoXCUyHiI8)4c$iCNP8Oz|Dh zJIML~D;ec{Lt6g&iaH^mdE)pxB6?Og6exaM<9ifLy*ZN0}&Eu^O)L|`r-}go- z@%Y12yxRJ9o$}+2{Rh&$~d z!=389@A)HaF9l~c<;!%vEWb11i=P$v$)qARj@#dF`Nd;hKZ~_qC)8vs=@{B8^%f6- zpA9s)k!7$;i3jn_OOyRSN~?MB>%vk@vDxX6fV~J%1$mTyka;{``!*Hn*3n<)&q{s^ z=5-^@TM?&afA|OHzlHu2>OL&g65b`6aNS+b5PeVhSJComtplM4NuJ1trDxhI49ehwVf zCU z$u$d%I{{ZM^gS>Ex6N-(E2H~rjV+;XkrBXDsrJrUXG?GyM&dmMZdeJm5|M~P z7dRQMfnj0?%?pw5#cSEy*`dr(5spF4bJyZ1a$PM>nps4X)#?3t7JuOZm&y$zCqA4C z&(wS=E}63y!s*UV$lH*kq3kPm=I-Ux#H8Sy;GQ_F_^z*${aC|}GCGf@Yv!tF*vAf7 zRJWn@6msk=CY-6<;$OwqT0rvKMx0Na2>g>KeuP(lrhHq~+8^xAAHtV*l4N0|TrNLK zsJGhmbBWe@2?Orzj~}20w_@?0uL^9I65jd1{Hw&P^}4CXq}=^k^c1l;N^LrDQMY7r zI-kYg9Gj6~o*#1Tb}MZok?Id0t#mpbz3~EDjQEnqI1{rYf_{ZT{#7*IIK7bL?EH^Y z+Oe*@OQ_HHV*|Bz;BZyqk;RY9s7p`K^$5IQevti?$~O9E7_7UC>26!*m~-lD$!5O3 zesBQ#RS7RIzl7uHE83xw^;SoRS2_DG_C2CEXpj2-dje}_FBMB-n;z*<0qtKnPk*eq zG5LgjFgW}xHh6Dt#@UH}G5A)s_;z!K=!zUc2uWzyq4fFqgJ~)l{@$ee0xJUV#5VH= z0%A|)Up1zJzG-#bt>66Ov{J@p`8@Z);{lCc7lx+@?63HKM!1e8jHSD`r~DJz5tmVS znXeMsB+WE0A3snEwFB)VTD>$MT`PH>J*|JEam2 z_JIEY3goGfXFk`Ol|-S@B)>Z|rk4<8@uSIBn{w~3<)SX@9xm{FHsb3}y^>YH$s=gb zx2;Fv{{R$t7gV`ebzM~3M%_-rNcICbtQ(I2cy?jsv;za&)oVY7o))q`Y<9qt({@|= zn)LAbc6AC~&Eh_GSNxAN5t`)FsH&_#%T&KJ=qp=KgI7{b9k!!1h^jU>n2N;r+M9o! zahmMD2GC<`mAa9Ws_$h79{8^;li~jWhK#03ApjnrcKTwXlfix)waK&7Bb~O8JmzeB zabGs_E+NCu{q}NQlE3*ReGM)k$ZErv>Tru*b|XI*{3b4qnoqaT^Nb}ht{<-P+R;uY5JPN>u%XHX_9zCnohE^2Xv=d8NhUC>xvSE~VlfzWt48g1OH+piu+`tOc2B!@ zNYY(%P%0)N!V`dT`c&_#-Yy zI~;|-i99k-mgYp0Za^w}eif$PEYajE3A$z-mA4Aje+{YhUvN3A5a>Zb0mEa~y$Tt= zDs6jqzXO92hQLBUxWDE43u%9BfnE!Eh#sRgyZ-U9=**`$UYugI z?R4dh!b%7pxXpcM*YNUJotg3NR*U6}vL>{+*Y2Esos@b20@uHMIKJ`Soy9W5J%JV!;D!$IwiThq-hNDtE>Diw-URcEEYwfHHoeI)WL#|iQ99O978ZM=CCjS5` zDAf9%e=6pu&@SQ2NDqIzkUv`eGYOeNt8*U(h{H`iB6*e8g{CTbOAw5IvybOoZRdk5 z?X1;8U~>=?91Z!!n@eF~iY05xvM#1^)H zyp9GQxrL73t}-jT5#kgnuXZT&=

j9|H5JJAt8jcO*JNXqE1vDZ zuSb81kXg&*Nfg=kh_4?*ism)nigHb{?N;a^Kfx7wkHnn8t5Znyq2bmAe-h?;d>p<; z{{Tpt6min8NAQjiMrPyKS0KJA@s_VJZ3gmNNLTL*n0OzFC;4$&3$FZ3!IC@M8Q??n zA2ktx?aqItX1qYnB<$*_{n`~S0n1{!p-sQ*M`mfT7|>7m{FSI>SQ(|lv%c|5CqSyjOK&`I*Hf8Fg~i)ZlP#8%TOI-cX|GFR}; zd6@k76J0f5A8#dp(CNZubZA-7z^CV@>U+#~UK_MfAh(JzKiL&ZON|?ZcSKSB+%0k+ z*t(5{pOdIutDp9Bl0ETWe{*H4wZo;ooEsOTOCu700gPA5VfiJ!R{sF6GwJJAoKjJw zm*M#y_ZGR~TSz>-#gZ}e5KVJUscBwwy_C^gZW!DZ8y}r`(9-;3t0T){X88MHcGdY)rRNR@;Bk9{q2x=}{H?Zl<$w zj1^J;0HBK0Ul{mZAo70DR|kdLB*E8Jnf(?SLg z2e25WQ!K;jYy7SHBQ=>tJH3y!4!07~ALON5{CA^4mXxPub~YI0vBceg(jHGTPyO=JH10s0ES)IKcu z#p3eqED|%=me1*m;3d{?T3xo*V?n8+hh&N%#q zaCcU)BL4Q){jza|kNTWftAot4+ImxjBk2>(#Nl%)iuENY^Rdm#qw5TDJ;D|#Pe9Z6Mu5wB;i-`{*@Ho50cfup5(9{b|3z=dnmE++_Npb2JN}V?;=2} z?ptmEt5-UIg7i`v`%sT3pai#1_eT}xf3H(c%3mapMm!qY)<22ji{YCEeqMIRrfZ|q zJRhgeAlnr9_vu~5w~4$I_X5*Tx!)l;0a3q^6}fgKxQk?N%Ds5Xk^HI8jPWg6Eu%u zoi|UpwT$_TF$X1+YUkLf>0SkM;I9MRI~gpKpZ432)~(ujbHX}dNp#CG8>R?g&)e6U z>~yPCR+OLMaz`-3LHn^^*U0)&ygg(G5yl27Kj9Uyko=_=(-q@i@R8o41cM-Y?xjt8 zSYec7SwktsKec0`dG-~3gV$y7b%<~Est!G>i}+FqkM86`K9%O*+nVby-!%!;b*4Pd z+}Bk+rxDhoIcjkZ7o{`MCDClHhEB@JxQ5dkSfTnZE01sO339C|KaEy4nuapZ{{ZT* zN{(AK@l5lpXHk}sfzeF%G4udq(;cfKYrAY=MhE3t-`RIA-+rq$k+W%J5)qx*C!bCS zOjn`F8f$y{e@Nn|igg{dUcc}V+2(;k3jY9xYbt4CV}Qe^D)-qnJ7@$E$8N)R?o%Y= zwtYRSo}#+t%7mB5g;eq7MT`$lwbzEs>OxkHCvQ||Qwd5@yQ-I!nVQZQt9=j%wg6i%-5qNuB3`WY_=EEijA0#eJiDg zYiR!Us*{f5uSI`t7!H=v{{Y`L-TkE%vNCYb7tQ(JwJ^_$2#FJzWGZE+V;;tSb zN_OR;FSsYbl_Bv~kKT%L_!~}}0$ErC^sO=A9}Oyl=N<=t>57Wa;%9=j5?6PXo}Of7 zN9kA3jQkkZ?9ojdI`kRZFg~KZtj~-w7v;R8=#Nt>;aD|$Mf$(cOFsu_Fi0J|r(aAT z=bD~<55jTCUPzSSbik@j;~xfnhsvH#KjnmoAi(0)}DK@L7(IIo}lUHeRY8}Oae zX?ObaT;0UmUf$4|q*61Sgx~>;a%<(?YyJvB@Rr9?wXo9c?{1Rne;`P*N0sCPOl0SQ zlY#syO@3ql01=}gScQXFsJXo@87Y2S9!*XLtfSA<(ES&Z7zZAeAC(a}Q(uLj6#oF= zrT+jCEcMHu4R~7S%G=12;d2r!t~|G8Mcz)_<7*xcSD>!~_(A^w1)TVUaq$-GLh

{Nk;NIei^-Ti6uhqLNK{aFNLf`ErfKA!Ghng!iBQABe(zpYZr$(GvEx+X+j7U5 zYJW>{+AA(AQVl*s$Qc;zURB{w8)+UX&}_U(qTFq@zkw%+pYAvWdslU9cy83H+$*W; zw4dl}^m%av)DworlHo-OI97IRO^&5B+nf(AnLd@!Y0_%ie45BOp@xjJz(|oM^oGnaT zprcXU9^~3-mOnn)`W&eHq1CbZbgolTmsY`uPq<^TuO^qlUMTYfrb}RbcH{KNG?UtR z_R|R#qK$!!01iJjrIxjO`&BQIzGDlogOuxJu)df5Y2(tTO+G}+CnKgft{GwYjlXfY zhD>&3AK)u;&R-WX0{+@9qp69upUClFOtS=*vPXunzD3*X3oDEZ*xQc4;C_|M>KX@y zrpMZ$`ed>GRp>A`?mlqNM_u?I{c2z~sB!>OJ?k&+dVLX$@by=_eGe<2z@8n{6-0aD z$-pSg4N;rn6{4g0clS^)uJ*_C=DoA+_DV65eX2LsEr7xyBkPLgt(i`HBWiYsQ|C+H zgIXQP&E~mq?neL@IQRPnz zh}0jTt$(y@@^ZOZe80%S%9EEeSAK^mp9*{xFkmgV?*0)g5I+EGkkAR|{?+WzgC9OZe^X&^q z(sa)*-%8cB4Y7@(8Cz~Y?~3&3ucwUyeW{K($o~NKRyFk6jjM=Y5uZZxZO7y){*w-) za~YRVhB80pnTk`-uTfr$%qca^Cx5u}>rk(6-NwbKu9o8-bn_exe5Hr-70yqmczKg# zi!tLIyXX1WV+EzGXl2zQ5M!tBcW3k!LR~|`!I=55p&W9@@aWZJfF)odS#a%=$Ept2YBOu<}0_3_dskcR~Fm7GoQk=W3ZZGx&@8B z4mhs$o6uhpe}DV~%Bz9Jz1sOBgR;ET;zE)y+ISgg_4;CzII^6j?%8~fr^0nPedhlE0P-u@ z_#V~NO{raFyCi?qnz)`F(zLn6@dK6VxbgW{7kO*rv>CRRC1La<^4nRCpAwxg*RaR* zjB#ApjLx-Zt!+DOdbn(#4@E0e*S_brpA}p(`7CmvbXBZ-jYzjKqo4Q=E66T2KZthm zghp0n##?fY!{}DIEkXVvT?Up39v12Tu}|fS@}YsGoz;(0rZP=x{{REuf3|O9&yx>7 zTBUQZ>+&6{6-VdNzF4@l_=g!HTf2fY$VCi4ol;#B#P_OA!%jU(jA!s0#d^@d((e<= ztB#)6e}VLtrzW$j+na0IKXyho1?|^s82cPoL80Uo(k0tWVqDlgU~Vp zKP|?z+rfu8l6mf1`^XVLE^E3KDO2}jCHEX$lApzYy!A;mD{%+eOG2n{a00I*rA-cn zI-fG$<0SVFlU$aU;N2F&u_dw#XdAuO)f56r` zW17F}S(ik=bBQD`_|;a>=3JjIKD<`UaqE!AKhl}Pm%9_L zj}6do-jU|5quxXEOBr9ntR;j)mLI1AyE~5(TO65HPti~Eu5RPR8gXEy9B0u8t!U>} z=Dp=&YUDH_tX;~#WVKvw{5Yl?dl%$B)AXxjQH9%JjllI7{{R|}RllgQQ0a{HU$;AI_=G;(Z<$0cCxx zagv07l-rLOX*TK%Zqj6S7y`70b3H8o00L)`HTG?9G%{No(AY>5Xzow+#a5EaMPJ@T z3F*aLPZiiqaGA&e<7wyV^sN`Y(CemM3O_;091b}j_1T$DgaKLh)TdA0 zs;${6f05x=I%b{bZJn+>!`}q2-~g;y{5j{dNej;?9DMQr0F8SL{xk6>lD>6?lLPlK zS1awu^{V!|*12;g#;~_+44;?)2d@Ub{5~R3jozO$c-7mJ<&D1PdG?=ap=hW5H$TqS^?1C&E};t*{{W^;2OnDVy?^5Gf%M_6 zUEEyxW5bXv+Jc5IVMbi8R1QfUm#2JJk(hBc3=i+IQc=D2l&rK(yK1&czeIX?Yy_)* z>K117DdwA^oquDC!z4z8SICBeJ};^Q37lR}zp8L$kW=E5;QU z1#`}LugkdO=`oV<{Jo7?`$^e8ShVb$ORHXP%c6cquYllsn3$@I-HyrpuDW&6AE;KI z6~4N-TWg8q-Ha6hhTM9b0a^{<7$?fdBK(|XoQ(Yk75R7JpWDyk_lNvh29x7hEhqld zWb&g(8aUx!khc61Ln>~_$?Le|rG5VZr+(8~HnZW=sc8vqCZZq6i%r36iDcZVn9~`O zA-EFj@`1_4dweb9=2x50mLm&3TPI}nZARB|OLeNXwdrf!X%_Q4dyr6)O zFp{gggJ0*jh8z!vs~UL4B%v>g3F)TRwY1Y`eUjT#)X#HiJJD<2S|@Mw{VZDX2aG&r z@fX7Ic$>h5@29!A-0f(eyYmzSaAu6)xn=pW9Gqc`F901c0qd3$-P&u79G0+LYEvin zm=a4J?c{6bL128Hs_~tSWmY8|uGaZ?1M}~VeiwXC(C(}>ofk*Ggj-rid1Gt#hLYFp zvMXX$A=y!++0cxQy_Xf`{{RR4QR7<=iSb(-hVzZQlh_Fy`5>}qd~r#X-auo!19lG} z*Y);IgTqq8x#6O$leMk4O^=L?K1uVXW`9FI2K~E!KX_xusqt$`)FqbgJ4jOEGK|)7 z22Fw=vH8lV11TY63z8W0ujmWl5BwEh_FMRa;hiabd#_vgbHo=rV!5@Q8DsN)QN37j z5z7pE9FEoewCElQ@r8wj?}9!oYw~D%cZ!~5w@7@8&$FO~36?p$xc1%5?d5?WWeb3y zZ7p9q-Q0L1!Y>@UTudK6UDUS;1hN)(+YsGu!~&y<69XCfKpfY>WLcD_R->g-{jGNw z^V;`T)3>32U9W_XEl+tpZPh-u?Z5m3{US6uZ1q@UxUrEYie(Irgfg<8K?HC{E2}D& zc1?6h2%|arabL{m{1OZH?$rEOrD}fyzA1Qz&K59vnk4s(A!|^8MwUfZF@@Yl&BT+? z8vO&;Kj5L-E`g#;rs|q&+|Fc3WH#~v5|`X%0pIv>x60!rKuivvr>fE}@T4vjDNyb8>wl^>x0PpL^YS@=r z`wV;UCs=;y%dl14-2g3&^T745=YseC3Qwrrc$(kGTD_*8lj+jDHdfb?J4TBx)hrlo zV<;Sa%61lD1`i%<<6qiuQTVNIsra{7)h3Etb`HqdF#L_Yd10_u0DPcu4mr(uGVx{= zAvsWoHJz50Ub}bMZ*IqS4Fw;TlZKjiSkPzvrXYOl$6N6y#* z2L(?>IThx=8NcA8U$n>V1ELvxVPP(rr05eAy|=Tq5aLJ3Nf^|IM@{1570PurvRT#v*n;R~-D=@4JB+7d~2 zDbE8Az$OpyoZ$AXPuf%VyVE`j>2}th9JG&8)-M$$gcw6E;I`@WeNRsZ|T}u6>VO|VgY+NP^{J{BR`F-!#x3!dS)m^@B+Gw4x*zCz^ ztZPng$vwKiFE8@#x%#!^@AxTS#2d{xN8v3qPtjK3p^oL8+`51{U91FWc^<_@Sok;p z00mgmd{uughvV%zQ&4!&+eK!K72kM`hW`MYAKoC23n~t4^3v|)e`d?4=}|7IEo1VH zv}^mwT&XHuiOEO%*uXtIabFSZOXICYMb)O3ZkFYk1GoJnJga0X?MCcJIpB_*e2)Tg zzHwf>s^YN??5z1x?zBl;*SA)-`V8v>hN73ThxNDh)5!km^o>^6S<~(HCcOeqOk*=g0O} z{ieP+c#}oeSI2jEx0f;K)>bxhUBYeTw>IL}Xd$g{XDbdMxiJK3a!Bs~bgorv)&5U#LMn&(ze?qAjR^dI`H z=S%C6jj6Ql_q7%GFY?&^e(^W_6tDJC@QAmQQ`Iig8;L||D6?{yCjs;Q&iU*!z{P!q z@WbNYgFY?zUN0K_JJqAqw7KMxB$^?FEd!X`aslCo_*e1G@gv}m#(x)Dp)x#?U0FjZ zk1-)h=T%~}7C8n1y+~2CpO-v}{SEz>ziK}We#8C@u=qtOw!G5hz4I=vW@)9onn%LQ zV2%}KA#ipl;l*-3J;6G(=sNM7<kjMXKdSab5-;D5L)`XXe^Q zKs*n&Sy!eUX{SMwK z^}SBo#b%SrN1AwJR)6k-#~JpoR+cSJDx$HR=<@rl^+LB2{JWsRea9_IaYxw;Zd7uhW6)F$BEgc127Lv7ZFuAU3MKGU!@eKB)pQlqbr~&V zEtz3^OMV9%q>~43f4B+s74xUUulOp>hl@3vuM7CM!$(ilq|^S{rX{n;(a$3N_~vO4 zc{^0A9B#nlsRF)JIPoVb!BupzHTG9_O{9|Q)$Hxlf4vSEer=KaIX1H#V`D`j`U6-O zmwH{<{q3xqk9J1?0NHEsAL4iX6;tESgsd;TX>X$FQE73!i(zd!k)^rd1O?VX)|dw zz~-ykU~g@^eoydrep(#$H*jxMeb~#U%eF&mjKF_&#y=rVZ4XDd{qK;x_2mBmTIVf( zCwPANKrC-9WfvdnwoMc7us z{{XUit-Rey&!Hd6pqEK9$(ZkDPj26+uPE>*#SaVk>r1oMJTrL>%3j*Vc^qo!W)VyQ zQqhLozK0{P71?VZDzMOWd#!U={l%q}FFewaaFNC`#PQe)`YM>|sJUN6bNyZmbZA`Z z7iu}w<;S24{uKg(nMS&m!Q($DKgz!eJ}`g5P<}3HJ}J8Y0E8yrMuS0iA_ln+_Jl=X z101jp$0KxML0tNbSBS&^00p?#G_M)j+xU{s!sgN&dn)^CjVT7H3s=}q^rrP>ym6xEN z2`KwV6=U{Z-&uq?^7035$}#w2v|GfhGVI{v+ZEuR1Nh0{pNPH^)BJDX?Lym7(zQEi zuC3>aRA|~M$H;T<$R4C|(!Cxna?)XhO~i0;LC57^K5lY_fhhn- ztX^Yq+CA5%E6aRCn zqrS5isgJXNX1r5x2e`v@gq>9WhgU zSpA_q7x1ndgw-PZRmHWmmn{Tu8nz<<;|xIn`r`t+>*siUH9u=r*?(R_rGTkcT(H#l zi8aE5anynKNXHdX_0^F4x}2&yV_^Q3`8)C3{t6eV_;W)yKMTA%v)@4qO4ipUBzf6@ zV;n_USLI&$`~a;_fFHO1k?}9a78-Yp;j^=`)tl_L$!e}%JAs_E(m7DEjim`HK4HPd zd>&Eb#}DGMQNqxao#n`-%9L+!J898rtJ>*1JXa6aq`l?R{{YwZq581iBZ@UrFUt@) z8!=S}j1!0R8ONqEU!GsI*X>ED_%=Cw9q^(Z+M+L+HT%S`6biiTje0wfK4NgHITiVP zdGQzapz)uJTT}5Sq}PDL7IukRCwnJ9y@W=;y9%QP2<5+8}G8Nn^$Emff`EbI|elRd(@=vY>S32*@C1NcxZIUw}Uf zwf_K#7g|oKr`_t>jn$U9deKV*KP=H@5eWpo3IGV*gO5Rq<@G6iQKIV!pjcmAS_>Gq z#WLK=QV|;m-x_YfNccHXjPx68<}klXJQZ5r%9au6mG3mI>87^Rx8QvZ4+nE9v=wh@ z6|RckccJ^?seDzA%4?V+i6)-PF*7Qz?5QLHj>p=q=^iQ7^*w6t%T5cGmO{ogPyqQ< z2083=>FHmIpAG*2X}^iy57*<5#k!Qb+~>^x(ztkox7o*7V|FPLw0TF)Hxfs^1qjD zPHabpxs@omwXK)@9-TM0X1x#8$=Adz0P|O;sH{tfHKyWHGCjbrgzmp=&lgx*M{9YZ zM;xyX$tOundu36}u0sv1Fhc(T4nQQ=3*x`q58|xt7NMqD$7!2y+z1Dq8@n94y0{!h zoDN9c&MVK%ev`a8gLaIZ`&v8OO)q8BO^jpVCTmhF+QrA!`gT7<=Civ;+UF;?%m@^~ zmp1BIRB$~9YW&->{i;7{Ce>}Va=J|RDG!}JxV2?cjluyoKr-Xz3NSI(0~Nile%IbS z(Wkn(xt7Y|?NAxu!$-AF)BB}R%*->lJGsdDM}D~dDtLDAf^yeQBelEPoH&04vr5hK zM`eBO)206auE*#Onc``#T1gTk1FjW>@hn6!8NeO*$0LKR*)0&1?klC03LPX zt#@1TyW5Lf%gOK7R}x56!)i-!QlGzO2o(Id2X--lI3w~-J^EPCtCnJLnNqb|^1^Lf zMQh5=Hro8R(OErqKHDbnQx%7=Lluht8&`3?wX?K$@>_luJGj4ZZxrco=J>}3@MgwAHW14-T2(c;LT{n9FFAM9>{8p_x{dVPs<%a<^x40H z^s!1Y!A(+@n!ejxR+7I@HS#}fbpHSqODe3$vC00}+TA+zt!+=o`Zc|=xYn)hB0vc& z(ec3cKb3w&TmI3~=pGr;yfwZ!ts{4|RE!ogZO<*Rs0gRfjz>5KxwHFF-gv`QUjSRR zt-!ZY16rBZHi$+B*CEShK*Km=vjeUx_q<2ak0Y(_EIgEwUiH1+Yirr^r=|Y@O;67w z;l5*osmorn<-M(??RzU}X4>uPzK7m=-|b28r$yCLC%m{l(qnF1D_|WK)*&26veib_8}V zTWI5=1?yji<^KRm-XpKi4|WpP-sZ~I@1>oZz56?#qg3!y2Io%8-D>*2-klos+e;s# zo;m%bv>RUzBY0LsS(@5OZcJkmTQV~3U*2E>LG(Ow1%6Zb%j0zCTe$HIcW|?tm$;N! zz`|)7Nu$VVH#T7nw;(9&I&oZHwcv=nYjJg;YVkIZu{2JTeWjtfIdp~l+yJEW$i_gf4?*!w<;9HtKhz?*n)gAwL35^a43R9L?Ldw; z%OP#DD9G*4IIpif4f{9gdNs6qca7ppq)S(^wwfhr0pMS?rms@o|EB+(+hs8QP_dgf>Hxi_GC>qw%cL_Yv z5Io5~&d#K4d@PHWZJ_k&UWxlUc-P_wi}mYG3s8|FwYio{WLa&byt+{;eBMIEw8`bj zaTIC;9lMy<>fOJD{2k)24a;YuSzBrL@!5p8wVpVU!bm@OS%w%ZXXe1eR$szj4SX{A zD+h#pS7Sfg@BBXu(OZ3%-D1VNgBQrdaS@Y`nOQ-{&5kSC;qE-jX;Xz9RR``#YL(hc zZ8mmGy4Ajl#};|PRpl>hgQ%Y}eQd9%_3}sm)ACOd{5jG7I_VMr0K&ugcW0u!ZetfG zP=a5w`J^+vPXw=k>;NgiBn*sKo@u|bSM1O5Usk<%HIFTIFDR0dg7LyvV@9*4+#x4< zU8P4})%q`K`!4)3veh3`(Og*FHLF|9EfdR);RGKj@z7(py=Htu{g(a_c-{qvP@V`S zhzN|Z4=J+U7v=4blrK+ASNQdo_=|wT&NzGyCZ#$rc&=w}PG>K`{erGphE7!%BcGZ| z{{Z+3OYlAbhxTIlC-F+e_&3Aa{L7`rBeHu)WAjn0?UT!nIc}w3?YYkBA9h=XCm9Fk zFN%NgNekZ@{6UMv(s=hsgKQC6$16s=#BpxG$Cw7}=L`TKoD3)ce%E+6_B;4(;FCkOV24tVH9%2OQ0^Jbey?Sf=N4tYoFPh_EGpzeQ2wpTHRgg z@yYfrLr^kaBtCl}aE4|fnJ^VFNdmu6^zYd_!5V#q%dOM}E{^3@1akiXdpmaLfI5OW z&uYQ(H|)#cFA-_pMgFlJtdV0MXj3u>!RfS>Jv}f#5neO#mQ^eA{{U!D**A8h9z}P% zw0redJe+o2hlC>Vv?8CI*S78Gelt(|TX-AcCbxfi@Xp@DOSaOkwo0%?4W#o60uWqlTwvk^?9PmYP-|(`&An2YkiqBRfOw$+2 zw_T|mlZ$}~U}Lv}E%HY2n9j_rD*|(#EA+47_rRaoPr{a;@QwIlduv-XlVG-sECG#j z7Y)bD$@JpAPJf5rvlgAHUh1)}v{wgdnaB-*Fir+I-~;(r%)b+|yggadr9z~j_D(x& zy%*xT9<>~zl;|fa>N38zdObUT!AIb}qw(MNg0qI-Ub(iopU$|%N~s#Dco!;iSY?SL zh8%I9;T6KK#-G|B#Ttyu;rMUu?e9^1s3nG3m+b|Yf&J_*TOY$-rEl=t;O*v?mpXo&=F(=82%KOk=V`~e?0vnf z)R&KWE*A0r&ezRZF4|bn2f#Ec``=o+xBL(5{sH{f>i#tN`=j|s#5#G@E_~*-)CA@^ zE{&tcB>meWV8Y-sJ`V(d6oFqy{8aeU;4LZ}En@b{=lgU(fWf4_z=k0h0}@Lzjp>X7 zv=hNRDC73U{tHxUep5zFzyr&fxQb*m{b=@dv^$+1}&NvDb8zZz@C*Gbi2Q zT|sP*yfJ_{u3SftnIs-nGJXLr16Be z1zE1NO)Bl8x@n??_Dg7rTXB{^5dFXk?gd6MjEdCof9*fw&lY&1(^6A?VWs)dytdZs zHMGImFpl08A2otJmAoqP6gk?B8W{ z2(Y&dByhMx=5CB(zHDcw9@)nkuWnyRY?d%|;g(4|cVFfI01Qlc9uTy?y1wJ_OUB=| z560W;JAVs&D$w2`r`$E1r%ZJh6U_+R`_!u9IM9!n^&yuVg?#bi4~hQ(wzz0DOv=kckv zJwxE7tX9@HmbTtpz#;;lE(zKdayZ5@$3B(v_zxd)$`j?Il-+si{{SYBO1B0^qID|c z_0!3H%VYUqKOFx6YySZFMm5*hKHm+%&6&7lSp3Ol1*26mZcmv&?YFaYUVEqfSNO4_ zXut53_zvG!(T$)#XnTg0*FIofvA320%VZFPXW?JhEsuwO5nNfd$a1~=+p+M{a;DN<^RQ{8BOfMK^HeGDhyIS18cVo}S z_;pIFQkF9MY5iSje>1)V_?Pk1;&!zihlus%w6eQ`2a)c~<}&i;K!svM<{0sxl&(0) z;=f7$GyFWa_=Tq(D%-|dZKHjSc73AW>USf!W}Nn zE*njP-Z=KEyl|?E4X!;$k?~j)QpQU^TedD$t z6^o-^1B7&yE85CCyMK4GJs9w&8kGl1y;VEgz5f6|Gx8JQ*Mqz}@VmiR7A@mFO6uNW zGzlfug2ZCMP}0SKIOl3`04F#mw`{yy@ZUtYlJ`;ZHnIiTYe?m2C7M{6e3?`WfN*-S zE6|*Z{fE>c_;;pTrL?ixm0i-bW-t_Dq6@3%y1#KFM$-z*LR^M&sL& zf&T#Nu37l;Dzznvs|uHWoEPOz#7~E?w4)ky>c+`gD_`|LG4Fh5;E1(L!Qzh+>ed$a zHvW0Cn$;wfT3|1le6V*o%921EG67NrXm~Nc6ZksRc$NHXsU0Tn-X9_fZKpB>mw1jw z1998@AdR@kYW*FX!@mvZ)h;g%nQtw{xMAi;Xqau-`Bl#(fHtt`dJ56fz8d%|P=eZd zG)q#hHWc#H?_)fLKPvlw`m5E%{UH;>%7m3j!b;6AWV(OXz5Mqx!{zjqZQ|P7NBG)b zhKTsj#X5I^b=w>L0kxTJq_v1!ymDK}=me-XozWNEBv!`k@IGD$HS^zt{tC@M#eE~i zz7O#~+gS2$V(x$M3!F~vNuY5hHM`v+zcS*TG^Bsmaw;k66dk)p{I6oFJ)FV@p zok#5x_?#W?tnYY!N6=QmWp!&prFna+epJ)bYkSu6*)J}K<%h(tgE9Ep6XDObNf@)e zv|vq(i)OcyOt5mJYi^8$cTPuAqZOH?e#V*szQ52cZeX>BJM@(xg4fEoiCQ!AZ4Z*g za9x+S-1X!2tayS2ac(VbBj1^_*;NHFrx@ya9V{{XiJ@_rhWzkJ)bm*M#zj{gAgthC);`@;SW@V2XW z5@`{$s z?uX~vLX2)8brVi0zuqI^7h_-J!4{~p&3WoK51G|O}6gdl+)FkU@+M=DxTG9LNOB{+7x3Xbjiv6f4uR^J*hKI5PPXoACn$oRnGwM zKm_sXI@X@4<0yQqi`gz?cw!;}gbdE;5E4%ubNPA!S4+nVv?;-1=u1sLB$l3#=yp`X zWiyJkS9Gkjj?VXA*QaZKP54{k9;@L!5B?Kt3w>>F8qFqYtzt1rh=4-Nw5b73bFD>??dwGjv^C)=ckRdyU9S#$zou&!Oz|zv{j|De@)3OzZOgGxV~jRI#yR@nS8hMY+~XNS zbSramPf0FhuN{)o9V~d2^30LSFreOw-wNN$PN(Ewj(!z*XW|Z-b$Q|)E5rbRl0oI# zC|IP)5=julF)}E_uI!ZHn)#2#pR%9ApB3vdt@e}SS(aIVnnsRD+I_$ix>W*S-QXrk z^c@9$`&#Q-hMoJ%i`IoFFR*n`h<3AJc5U1+?N+bn)*_5u+xd%YJb?tQ^DgD!;E(+W z9V;xa#vH=~YFJeyme!t&W&FNJjg81~HDmGr0OY9r#`tyMZBF$(E#N;1_)}K!zLleC z(n8m_vU%`HjvM7-Ci5jfGOo?5ju`Z>J@}FEW5JpSh(@Jh;ja;E7cLNQvMmq{yOS!! zA<5vmOm5?h17EARej&HCPcq{6BP?zE=gIxqlaK=S;cz`Ne+uMn{vzo2Hw8@hFE|@O zh-Gr50r!d83vdq=o)g8)+MRb^j9SxMN=d!l+^v7eoK6!0se30qbhET{NhM_)SouG~ zzYa9N7ejU8FA>@5yX_>4?D|#C;Ts6kI3g8OBW~%vi1Rbsn(ptlZ;IXphT1J9_1G-6 zIW}Lmmo(E!8^q;fWLXaL9@gEsF(j(-zys)3_=j^GuF)*gK!B4T<>iJBPC8>dKQ@pQn&pE-xa60|3itVI6B=Glx zE}lXof+(Slp%6ukZH;$>k@J4=02d=3-+pQOsI#`Ymj3`zxql_R*CcHLLEs?381(DU zwRKi6r(WGH^W0s?vL@1qp_F!Qql|j{a0NvV6Dv`SXC=(mzZA7^R`l!Vq4f!1s>S;` za;I%H)2H(7<^D(KAH)3*U+~0VZqa-ltJrx`AYU%xSto3Rxs~TR+6R1f=LWk^9(+&u z->vCdSA;d62Y8bDD@awN{q3=626wOTNKEWJe-J*LSLhA2LsB2vx?z?e$7)HE&$Nzx zhTi_3-75o6@Y&Xq>f(81yYlkE_Z4g~!si4IF}DJ{v7R;7tx3_WuC#XNspzk+zK>5J zmPv$%?&(+ZO(?HlvV5Zlf~C~$w`lq^Tg_n^-efWbGRy}4*dJ zaNUR>UNcyl){CI{qUC3j?nRAaNXjTxWQ>7>lgS)&>4BQ~=f#wv%k_A;d9GvXp;IKK zPgTzTR{V@T16}xkd2xTK!)Fb`mYEUEo?)1_0WO=jzsr&UAwU=$aZT2|Lusd3oA;Fq z$e_zSExRidZ9gV6xG&u};P80jvoueJUJ>yAr5A{_xg}2~<+p;!a(-=qD8b_=C#Fto zx6`~i;9DIl$bvZ+%4f+NRgU*ALF1LjL+C4u7oB0fr?kAKxAniF>gAkmgHUnAK`rfL z#wPfgC)l+sizM0P?fWSUfYM6A3V_(a!QRXO!*QB!i>}*SXz#7+P@+Wl8x2y58W9bfvso6*kw*prw7yS78i=}Qj%4trOySw@i&GvtHPcXhwWOe z+p=3s#28ja9%0EKCMFrNu{^-{b+Ds@SkbAq<0H8t2}pN z2a4(v%NE3KyM%67V*{U;hWToyrKNa_Lh!}zo2F|jx7W&tM@Ngym&%(6n}#ykKYBfi z8GGk7(deEG(Y1EH)b)!9t?eNU_C9IyyvEvhB!rJJjtI^V(!C}PZ^9lGTS+fxVr}Hu zUM2G+-6+O9)&+{ML6%*Mz&IwmD`vDKB~FZQww7Oa_2z|&&!}P|l&dRmrkxi~w)qwO zE#dToP8tV@+DnGhQzBc3F#OEnBaP#X?QytpLgODPz`(2iHt^qvCh;Yvw`*w~uh`06 z+s5;SDJUO1mD<5UU^l1$?EomwO;^3X@WskS;wTh|VFD>7!yhSB4ckJp({tZvV5!mRZTG%?;n^*|*1ah(S0hNYGE5IiNDLpfq&NXoos-+vR!}YQH9#>kA zvHG?hb9=V@HuJW-f51AuYg?Ym<3(LE+&9`tojlXO?NUmlG250vpn%M(1_%eat~bS( z@M;&Akz7Vv^UX+OGB)kd1i{N>0TVN-?;kc-9C44E{7LbLSkiQV4RHM zcma8?>@N@1brYJhPRaMzPc4_HqCO`&F_m*bdi^sB!WIz#0Ewt=QsV$zZ`_K4Zsms9e_ zgL0gbK?jeRA=RaxHkZYXaiaaIRCYdc{{W;qfrzFK&WQ|}UV)C%a58xEZ-9TakB2Wj zf2vuFn526LXd*YN644SY$j}KvQSxJrqCyv)z!0_i9Znd;SHsSv72IvjZ8XyBO+U$c zSn#-`HIl{qvb0l5@4MSax=Xj^x6@uI@m2n?-)hrzcp>{FhE;pH))YdZfD$zaYO-!p z2nT8BfDJ!T_;uiK6~STR20nJPGCZa)v_@czOND6Sl}ML<;qn(HPjJH(^EbrXZyD%6 z7gVE*}Ns;9~pS2%g*rJ)6W!o zm}h45VVRW?WZk)?QrjOZ@Fne)}Ply&Z&Z>iK|i$ zJo1y#Y0);He@p4EjoA89-{QA{Exai6*p!$1GFYwSg5pMC8Y8$LFskI2W@U50=K`tt z8(%jQ-RaX=#b>3-c+vwJER)_{EFSzFTkA2)^Ld_3#QH?w}~>*T*Jd-_=SkJ-xWN}luVUJ6U^ z5NR;JY_Bg8+-@vJ>Da=-imIQHUn+6*HT1v5e;mJt^vIg&;+7jWiG&PVNZ82aBAC~6 zGMNt7Qb;+=`+W7_JDV#{9?PV7KK{ziz;Cm(v4i)PbHQ&Sm0>Q$+c*T~IM0@&hTXQk zUgyLYz7p{DjGh^>lGq7ki&k$fMc9TgWo1{|Kvn(OMovm)m*(QXI?wR@*vQpz(WPbb z>h`{!5%;+^Nf==&`zWVpvhAkbe=oaJ|J3?ZMbj^}yAQXu&CSx?xNk9YZw?UwVC;QU z<-qp9u9sZ7)AVg4Yua1sw^GTv2UKT_p$Z}2*O+#YPZ{R|ybHo#6nt%Hjv@?Yo-%&U z;I>Vd?+40yu|UM&DwXTfv|#v=t!mcI4ZM5K!otftRa)Xgcb)tUFF*iZGtGaAbvQ=1 zH@$96HvX;UvFu^9`U;)3*URZgCjx~lu82~Sq zb|R~V!Oni~Ku5JltoVvO-`V;sGWjsOdB#}>`oWMg-AbtYqabsPS3BY8MK=ZSKDJg% zqTA$iV>4yV@BAxe_;fp0(=WA+9t%57ZvGDZiu!ydJczeNqF1PV2rnRZbZ*DR^&X{NO4pu$b1LxuQ}JgD$8-G+(Dqmo&J1CHG*)xcypxXLx1w%W?UhRyJP&)M(W z`uTZ&N1|!p4vn^-9;u~Vn>(kq`JQw~bvsGtCmGKhMn3gyUugaT)}fn8xf|Q;%N@+V zSj)6uIYm|eBBvd5z~a1K$Hc<&32ZNI;V*EH9C7mL8$@tGSnmaD;Kd^cAx&h`bA|eV0tqt`a4*8@7|SBC6fL zgatn?eQVEk-y7QMRu_?8+r$V!qA&v-vv2vk5O;mi{uRvJc-KnM^t7~^#wipP^GH?l z$c#2HfMhC?JN2yP!WfFp&a;!BujBU}-|H_?bJ#o>2++eZF-H$#s=D`QK&@M*A;%SuO{ z<4E_`BgtZ7SyH zQHExUCj#A4GEVL=vGem4$KJtF&T-o-!#*m~XNt@I5;c`0ZMP6YQw~UD_uCzCFg*r6 zYs_wZNvYU)W?Q=;#@g27YR2k9ZpS450J6OsA1FUctFP+XmZfl!U&3aB;IJtf2ttf! z=Ie&XPTl)f_&yGytrVJGzFvoeE+(3-&iY$Re_tct^(*Z%23wS#Rkn=bE!3jz$PPa4 zI`QkL0n%L}eB z4iE#kCm<*s;~jBSbW1&RSMdh>*+N@KDTx>WKqD-lf&BiJ*++w@EIpjdsl_gyA#z0@ zQXOU_vxsW>jJ%2#0C!~xIB5soP)G2n82}Gz*|NN|y1uk~nIpW4)@F(`8tqIl3VLv< z5IcVZiuvbN7ckgCadmpK?{eQE{tVz{{YV&5*+pE@?XDn?`+csNV~nAaG++TN9|`{e ze2?W`Wlj*K?AI){{*{d?aYSzUq5l8?@4P|cy=+Wj(R_%lCi7rHaT=Z9Fbdr>w`e@$ z9X|}u32NRZ)ingt?4`1^u+xOAFkC9MVS>l70g&8~2lB2GT`Wy4uBt9R({v|wQskBq zkfkt(00jz1KmBUz^c(FgU0IvbKI6>n2lv0$bopD7pb|fpE7bo0R_MbGXhlDlUBBUv z8lHHaMCN;h{u|b(5NVHgkjxdXEn-L55lWxGwTGBg?LUV~)6ss&EalW}u3O0#Uv2W# z;UXcOBT{l3d$Hqy1~@h4R@&!;ZRUlY7BeJmB$Dq6%^JSpmT{C)i_l<`j^`DNqiZRp zJ=*AUmy1$~lXDOeV-eslIV60+vVC~38jlNe(ouSU+qUcdSoYR)K}}98r|F^e9;pq* zz3jT>o}~^iqzqw!@P1%OD#wqMKT+w^ny0Bp;!S#3Z?zpt7+Bw8EH38@BaUSU{+Zzi zPMO7cI(VKrbjbA0HCEL5z`VIEV;u%EG52`t0O{Jj6GGKA8+gn&DjCBhzbxY?XUw@B z_NX0ub*>CvNbs_|jo!=MUY>==JfQv64LwfF$1q*$8kN4?L1SBMk1lY2`M|>GjD6vp zj1GD9t?M<0#4eqCa&-&1VTyHRUz$S$W&JWk9PQ{a(Bi&N*LA^bbvN4X+BxK%nd69( z!3NX2C!i;cb?;qohkR3EX)L-{zjDtR-o+5&MhwGp1J2d?7a$&MNbsLEG_RuQqObiS zVKd4SpSv3~-gM1VP4NJl{@tTj`$K)4Z2+SPfK#3nt_k3bD;49Kp`b+b>F*`?_H&KZ+V;Fr~CHk0uZ{gc#WU0OP9m$?J^v70g5NJ{j|A zYJ8h`(mqJ*)Tjs0?!)%hG0bo4ZN?FdVjZ|?LxFIKhDX4I^%uD@_>l2F74zn+t59^9;xp8NZ!XaUM8mn z$)>-TZ^-(KPP5T`FEQ~Mw}JH;%R&(ltNgK$100e(z6VVFqaCV`n{BFEUB=B4t+ddgJ6I_Bv5v={PhL9L%6=30gL&Y6 zV^Z;L^2%h5-ZdVnnWxnjzG)+-7JOQ9D(pmhlBXJntrDT;BSsvjpn49 zFT`sTrbRd-$dvAIWIS=vdf;SmYwgP`eKSDO?Ps=}FUSt#^KJ}(iGTw+;2Qkd_|@U< zX5!T~Z92nj&_9@188>6N?l3^cIIDjTe`(q6^oG+e(8%bc$z~sWZ$QiI{uTT)PgE^}w`J8Nrc1=wGq8GI#ab0R^TX=R1gC4mB@(UvE8Jqn!Xx6C_ME$!3--}@notnft_ zn39|p9&)^yZ*dxdj-;Gcm0bzRN0Qe09k88P^K(e^4;gr?Tk!N>+BdBnR?t})Bm3ub#48Bz&zrA7WjtR<@E?rbe~ES4eSX(EtfH#uwKUEuHPXy4i`H$2|P%2I5aV zhCFA`6O7kgq3brIw^Q>}tkBFcJ1g^WgvVw}kwohDF(0UBN8azkg z>!0l(D#(d9tjzd}KPlW>gS4)2KZ}ml>#=x_%54MtVJ*_)$>Wyc;gT`)%)k|6?Zz;D zvs&@^*w|e`5u9(ki^`EE*xY{$6zPzPP-EDD1#6n-GOY{83l$~$e_xT!M+V^U?H!JD zP161$UHDes_Tu?(p^cB394bp;kb#Ze6uA4{Fe}e4B=HZ2tRcA7^uM=AE=)2om*xw| zNg~I}LF72n9P?T8vDke^AoV@tQrg&FRkHe8$ytkT+ z-f@|*Ck`^IBOSLUG8Z1aQCk&=l~+=YEquSOpYRXTa=#ZV;b4@K(Qkk2(G{TZr;GJ{ z3foH6wCOE;nZ`DqjrMJa%!6`uQXd`4j!#k-!9F_t4cD*d)Gquv8%d|hXCwwOJKKT} z0$ip&LWNGDd*=h$Ej)eVnJumU!G5T&2igj+%zj^xCVo{#!5fD>bSAwIQt@17(bm9| zX&S`wGa{sKEDAx5bCzSCdTkwusD?W+!eZfxveTx&((*icc#BTGc-5;){{YwN_-WAm z!q$Ecd{FTpg(C2^u9Wa%_DR^QHZ#dwjM#+S3F1slCm zSKIWpt@wY(h?E6s_W-H$^TLo^vty1rfzq~o7x5!N*LT4bNgamRka@o=p%M`4dXj*SAFdA> zubHEp=d`f0sa^Z8OIpg-Nomv5L%aSWy-H2OoMM~&9-DMqecEhzZjs@yiar^ZPZw(1 zPMvTsC5%TMwX%7Nin~NXR~^$iD0w6eaHf6}ilAXpkV zWgD=dF!Mogl&~iQKK1pTzl!w9FQtO<*4k?;z0gLAd4M60JCEEBa!wBJG19lSFNq!= zw!4o@(=I%OYpC$ShVLvk(I5cl=28A4m46U(gw$z6@Y2@NTWt@Y%y_P*Dl+!9K7M{( zw$rwjJ}1(DWKW3_r-&}@uSL!Nqz)gfUe#^v zZ6;-nASFiVVUK3%8CQ4A+~9j-iu!W%<6e!TT+ETh3}Q!el5QpT^y3{j4!{1ac%|mI zp% z)UNfB86}bi$Oy$iP*2JfeAVn)cg0T+MP=osgLzhQ8*~hJ0IjpB$oa5A3+}Q+v&DBHaGqR4YBYfzkTiU#{wFMGwFm?m_c8&g?nH3g@qX zdhh%h@xQ_PUb&MiOKo@`%#7{*=gta%M@@~Ahqi0a&GM|qy643~w{0}+@AA6;0Dyk4 z!G0@Ya5$yOHyCJ|zLx2~XZU&_|JBEr#dF7Sx3jX#7*8%yoJhnmOfDDro9GJrV!54f z;)V6a{k5&iKiZ^?=8>X2k1v%^lwfefj>DyR)|KKrKM_v0uv|)!=6EoqESZhGuvZ*m zI18RQu8YCiY!)H!tlDU8ptB=rU*4G~k&J9y_f!L?-N>)=2M>_br6$_>ADTxrtMEnW zdY6d4C3uTc)&pE2EjQWXLc`_r3BVk7jX~Y%w-_|zs%yX5Qq6E>-x+k;xcP%F^1zXi z@+ilkJq3BTr8c2?t7$S>L?Mz_YqKFAx>)Vg%TS|{fH)b(?Bw&B-?_VM$PKIuxZm3% zI%I&kMh>grd!4G@nD?f#N)fVgMo$?zRBm$I^uG^yws+GZduz^olq^xk-zf&>#s>gz zLGOcHkBjcD?ezP3p`Xfz-I2@A8h7ui*_dNYV7KvI7iXW|Y9{qvUd< zX$pV3rwTvX{WyEp(q4Tr+ge$Ej(s%A;U zl1}4qHEZiR|2u2}stPvKoPkBACdd0Kj`X(yK$XCEs3rJhe*kTRmU z9bqgJPPEsgmD>AIo^*;92Vk~|e5RCuKaZ22sjpUo%HHos(=@AgRF>vg3+@19J8eK0 zJQ+IUs9$Q@v>_NIqH@O(IeW0^RM2&eM@+YiQn_YCU`$a-^Ad$|(0{S>Hv@tzPf*vN zUeyF~Nfzf-SmliKxl|3S=nuC+isbxNW3FHLa@SS2#k_OLaTE`nG?Gb(wSc4`G5w*QH0S_=&CKy1j?WnskWyNS)qS01`jg2j#6*DEu*AMw7?I-ZwM^ z^IB_iUd@$ZJaESne4r8(XVc}tKBlHkawYpjK3b;VF%HatBTv2feF5Fqj@$={BGr67 zac`#io@BO#k;rDtwlYX#2TnQ+^c1!_d)Te zINC6H91gX|_{YQ++CYNo#zXADJeCDUSPTGh{&=nfPrRDiRfZV$LfbwBv4}^H}~7@cHnx7=dQd{ZEXtw z0B5@-Xw9-O;x3^}Kht(SfbCq;>Xy%|TRqwHl0DIqa1l$ZVKLL7+(-1UGP~8@-&fPx z1>ux=KrVM}`JCg>0grb4D}shkNv>5N>tZn(!+M_c{{RUTudGE1qaQtFSJ(#S1Mdu; zNl<+{{VGT`y+t)Rw&Fe8UvYe|%@`v9bs!O({W@39x8D%0bngw_UC4ZyE;q|8xxvKJ zVRrWcvD&@Y!8h>@I@zqs%J9P+vU%Wbj3_XM0CG3rU<~ArE543V2*s{VELIaRl=gj% ztFISoT1Jxvl+onK<-E7RCt~`0AE5W`Q0e|8)g!mIwr}4x#BuzmoU*Q4KZ^|eSB>B6 z+NJ4&+fTKSxn_wPL5XDBpD==OF}aC2;;349nj2(}^_Aa#BaE;icE*`(6UGKGPZ`EN zDPlNzM@Yk=$BE2m&tuRciP7C#!i~Ec0KyrQ`GHj zA`5r9yuGo<7#U|_gVTT)K9#X?s;-NsKB1|~Tgr?B+21M#)fmEpf0t_U-3InZ^(`;M z4JPRo_sHrtlFom19|U?2Zs&@nf2~|k;uEG^PMhB1NTX(K0>}%y5`JBYh09rpSn&wpxG>3t>ugZy!;wOPv-YwQM zrdrsurKHy}$lDN>E9K>+mOusu%!C*iCgXM^y_eSPX)ZlckUhBoOYnP3r_|~Jxe7lT z@=a$~pGvozLO4~0x2q^cQ-O!iPW%J+dvvcYzSnLpEwAPCSA5%20?05qz+upl$vwJP zt4kzuN;W!b6-G-5BSC}?R1r<5=If5?__QOU?xEB0M0nCJ@L1Wbsbwxmqfa^ zUoPdCNW>DqE?a2FYytD8 zRq5Y_?tN>d@XYsmrm6jft=Vq7X`ztJ=^*(?6YnjYxCym5Z*B;r(LNGrSDpx@kADS50i7IOlMIYypuEVT1Z#R^3Z?JJc8fd z$<2M#@R)2QEk|>q(^S3rvkT(ih!Wz)(?@xq$kU?on?EPs%Lhg8oY$`Y%O4K)Ikgn= z2Z}A^oevuj5~{!s6t5Xyy2l-IE9KdIpA&08+L!Q2BzJdVyv8AvNg)AMyN-BVe+cvy z`!(=iNwBi;t)880f+CH14p<&K;9&Ljujmg2cv(jHtV;Kqy7YT}_J1oLj$1{h>d$w# zl09?7{u$6u->{f;?T_j!(UZd-9fm#MVUrZx39wOC_xL2^b-YuGie4@)>cFk;fgn zSHoW${u=0-fjXPJ!8~6yau~o@WH~Fq2M#i=kCz-A8vL8Z4jua51=5X_Ui)qOd3o6A zsV`{<2J|`42>7zk#GVS0(&`A=6-)u}85j|g2+vd_AZMt}cU}tkn+Jn*Z?fw*Ya>SS zM;wqYe8tEHOq25CjP4lj1$;MWaj1A%Z3ZtgJ_LmqBP1$?8TZM@ zY#RLk0MdM2;!h9iciKhVkLO#+k%*gyA;Dzwjxa&TPRE1kUa78lvg#ck-bgnoVdsWX zw-UD8ZeF{%<0Fds3cLv%J!J_k8Kq7nuAScJQo6RO;tMG33cNR0jdGGYjox0uanA(r z>5kngQhip##8y@}mj*?7JgBy=(;BXFTW&)!P*<=XoL3)f;u-9vo_SjkB)~?YcqVlK zHy(#SPsXw&@m9BUa|G}Po-~zSM^X;L71v{@_etdsL({e^^b9UfPEh4hRz^51+6r9t z5!C9Fc$VVY1k$eMns>fvAoP;^mqc*~5Fjz?PPmI{rNaZ9#~xA`8;OwNpwoFl6@q1OCEsA?B_d@x;&GfK2B zBMV8?n{TW4D7^8ZC~$eDe7E{_amGfFOWA)8()V7;)Ekryy1PJN;ApakfF_ zM?5o$BT@dXWiEENsoF;v(V2DmvwOGgP^g-D z&lSlxg|yEP$91V_*=LsP1dAOQYDErOd{Er&b zJYn%4Q1K1TozxRTkw~IuJpe15U>{&c4@_~4SI~OCovDom-b=?O^9bs|a$7i1KiskC;b*5kb@U^_RF@QiD?P4WQPNW{5quQ{qwY@qU%V?!y<|WE*en3OGWAGy$ zn6E1zio-gQg(zCq*v?fPV5iO9Thia=aDUsZmRfXzLjk?GF)0`TDjYixKtBPR;rw&) zXHw8)TR0u$Q@F#|?)imRr>@=yO75MYOPH_p`+WIstfPi8{on}$f^x^dzol^+f5F+l zDR}!%@ZOPqCA>Gb^NbIYCY_c>RFYhVQaQk1L7up-oDD1^mFh;()aRj$sO?IY?#X^f zE2MmFp8D?cAt4)+cGJKdft;Sa_WUc`{5P#?lUV6jK64e8<^0!gnUGt?Sg6MXe;6l$ zUT2|vCeSW)q_c`2K7X@Y+N@~VE9J9J>%gx!@l1~`j+=92jR)J5>{5q28M7MWz5vUf-70d)ImecyR0PvNbe;KN zv^k|^WZBwHXJu^z>X>;Q6?Qi4<)zA#j)y#P{5Tk_O@1wM?QAUcq8P~TXO!~6k%Hs5 z05H#N3g&!4<4Ls5A)4R^664DW#(2Z~4a9NjS2RBrrnjpwRbYq%OShskE#Pj(7Cxzr6UgU^(9rBG{7I-byQxj&q~21Zu0is??sMOP#yRW; z2{q_Crh#SSNaygH%^kFJ#>n4j0b~)A^UD2!WgQ1Sg=tSQ!%&J&I`uw_0hnR=4UoBJy0#KS z5~9dSm97w~`C)pgC+{9O_ z4A!53{{Uyb62rvSQrKVITUuSsZ7dKSg`p&qfNcqadmo!89W#@R3gvDknlBgX#s-yzkM9t^(xGMAR0r$M z&O7wTuJ`tJH~u;CUA&r%eqDmxUp#05z=9-=yv&ZP^AK43`d6NOYt>@YbuA?{Xui)r zt2@mwU|#z86f=!psox9e%e%KwM+W^j_AevDAb;pE^8OBsOvVq9%3Md)**Fw zASwure)nv1mgM(r{XA3Rmx!)HX}2;m!D}3k0>is#3xEQs1MV*i$MGE3v3Qe4)NgJR zMzNg|^Ie(Y9%F5eI4%DGe9}MhZA2m2R~gy|0={a4!PB+W-Y3L&HX3!QOF0~!(jrW_ zZ!G+%@=YKce(_*?3iq%$YD&)f6NrS9be5~j$n`7vSH(Uxu=stXT(|FRqp|Za$gm($ zB(gEb-3rb6SEy+>T7QV|wS8|&EY|VJMXNqJj@l)*jHn$qzIJ+Kn&Pw%gSxkcekS;f zQqI*kZkn91x+q|f@gpLy`tGJx)UG?kU7x15l^!p2M59zlF7N4d)wY2w)>k^h% zGRZ45oB+%*j<~FG*m@1FN#9SG}h%(`h!hF{vHOcA+1&={rL;Op#vDJ0hrMthj zi|mcGi%A;e_Yl6w`zZNT?N#UvOa;3&&rPVsS;4@`R39$f zWuJ=kGU~A9r%hVkMi{Dz)tg&?U+@k;Pw{Q7=BXy1Xcbo4-C_#)*rfwxj0|ng0^_ZB zegURe>*ctFT00}(PCi5<=Vbkp8 zA7+v=sEhL908x^}b$pYbpzT^;8K%>v)#SLfm+eqPn+t4%^F*Pe4f2jmr5XKsYVs|2 z#I_8OS|2i3FqknI*f9_Q;B{Twm4@eeJW<& zJ+pFl;uy(Qe4c@@eSlo|uIE7T&D5)>-s-Uyj``U+SqQlpXg*>|LBJqS~i*n4qU8eX>t+G5f# zfx4RBb%r-BhLsmPn*;mo%)oJx>;+Ar_^M4?#`>%lu93i!yvF4+-c7U+EPIP|P@~Mp z2XP<@cA+MTqr4hzoKFSY-YhozTX|JyCH_!4ft>cmB;_dbTXoP*y0^LgBaHDi&8EFQ zmxm_*0GHd8ja9(QtPWorhZ$S~MS53>JX@mpd&6aXue7a*-0AmAPZt6OB0~b;^dM*BM{ut@N*0k=Ar@SSiTr`*Zr;9a9GaS~C zStMolyk_Afy_jSm3I(#Nr#v7nfH9vo@wvNpveI;~G8WW6%WRSPY`$NbMa;@ohYRHW zo*|qN4?MB2X1umC>(<(SpA>T3>Ls3KIcN}zcu)x&7{EwlkK7aMTV5^jF0Uo~YFf6# zcW&`oSTC8hO+C{WGM)!390k6-9ssX0c@k@8(MfntGWeeUdWpV~nWlUUjPcGrWiF`mDBW@2y^2 zSi}qA9pOck7?DRQ453hy!>}L>Q^L{0wef5He_tZ1;bNk@{{X-;JTZGdxA6x3^Sk>V z7~zeXhz)YF`PLhr->0BLt?i`b=_;evpRB6)4GhZ*7inHb3xlVf3FqxgFFS<^O2sa``mz|(xg z&yryb9dp1r+H8mwsJVG<6YJ* zKjB_#R#!-^JohiO%G}DptP#xIVHf4y{=fv+1*1zFnY0^kG7A{&W}fEaRQ>TTFrg8! zBl!WrVY{5*b6(b8&bg;4f0h3L$nApHNgqA6wWfjL8GKQ!M*5mG5i}9VKJ>Hil4)gC zIPymlKR-DlF@;fFkHfzm&mM_?;(ObF^u0+05X!)BSmOlB_)nE%>=gz*GhWH@&gR!u z@HMZ7En%KBb2YpX+rf|kapk7G zou`b+b8~Ek-P&cs5wq@qGR3jeA4=rgTAFsVg3v(_g=BFQtCxiW79$UU%tHO+jB%Rx zaF{q#o%wJ701eplDf~j{+Kjic8(YOfzEVTNkt?G}Fca7C=f6Q-lc~*pu6PeZmqCcz z#U`hFZ0V0T5(tcOCc*(Ap)I^;YhZM*mi!&#IW<3w`aX$gdZN*F3oz`vPY&wL%tGOF z&;T>X99Pv|7x5OM9-*aa^4iU;Dqg@eP(6I$Cz|^Y!=4(^EN;czck}R|6=yqn7|sD1 z;B_Xyw|G}2tB0h$uP@2Hx^zCVM%o*e*1C$4V6avIX9W&F@z?aJQM^FYf0N@V4 ziF)g8E&L^J3Qed42d+s`^v{|4R>H&Z{&2x{SebYjXD7G)b+4 zaC5Gp@wr_t<(mTv5@)ca`gRRF;lV(IQiDFD z2l>sK<@`ydcna3ec`UBdNe(voW;JY{{K!RkxY*NF+igF){EqrJ+TVfrd+{^DcU}wB zu5|15pG`v>PO|LEc^kfbsOR@>hXH!7;BnTtKMiY8HKeyJtoF`BZ5b;2e(u)b;5HZ^ zO8d*=rLTo=bziaFs@v)o-|2R;5GvNvK?E(=FGSwzp)KSI$yK zJCxvc2LKJALr&Ko^IA3@MF0K{{ zW11LY`3C60{xVyR`+WiTuji^* z6a>dZjCRF#J|59Dog_$-H_dSy#kA+lQmp$|9-o#D4{GtR3tddtf<0A@x*La%1CZpM z!M753zaf92#|jc?%2;2<>Tjwgk2>5steBiM~GLI_n|p?LstfyQz@tH>>N zy-s*$)gcB|xq=ZIHVAb)ST|JQOt1~5w-A0%2=yHXF~xXii7uqk*HN`##ye?)4c!q`@sEAk&(gl0@lCFoqTk&3 zf(S$tXzY(}oe?d3&ndC|Um+`#{n9JP{{Uz|30Z6Z01Itk(BdE36UTCzkJ&Mn@?_r= zJ8_3G%m+B(N2Pc;Wg3`>sO;L0&1?C8k@dCsQngq~IcpEcZ9lGunD}$z3(w-4nJgGQ z{{Uy0xjlYmeW6Ew{{WqEHd?l`FNvj&c=y28gA50lbGJPQ032<{Ue)Zs1^yGsqWB^W zds~v)X1(*`o^&KM9$mSS0otVQbR!Bnayo-v@9?|eYFX(sTWgXm5v9m^L{PJlaL{?H zvSL6hu&=gA72lYBR|hGk(^0ZKcyOjFQHx!5xAo|J&7|GymwL9FpnaY=H1QG%q&e9# zNPcpqe&_?{bAhzsP6CSWei`T*{{VtMBxxE{iY1pwyFoCM19zDzgUo+LVE+IwV~_=V z@5A4Q_gWW-Ves#VEv^FGSozVkh&S6x;|S`w`B*Cr#GRyIHh9Oaj||&upBeQnBS?lR z^zS=Oy+S@*TiMGqENio%mQ(-?<7gy$SIXnKqJnaz9jxEW`oHLBg~)47CkO9FNiVzp z8_@VGThi~mAFSPI(NCEyS>wX+2!YBLKBqfC?NxkP;w^O=O&L(6hGq^N6(Ti209=8= zB=zfGQhaLoX>sDeh#I!7;mdV~8~a;JrF@JNac~(>0msTW5~HH3maiJ|{{V&I)jlQP z+-ibJuO;(tB8f-|i0zgLzG3%NxZmzG*iyq+qWzyeu9xZg9Qlr2R*hPTX=i2H+MLd( zqP#b=ct=c}G%urD+$6(+8MuT$$`oL?nl=CjrFA|a@Z263wrw9zl38ccbm*p?rNeHJ zAYFsD0XV{d4%+$)!afYqbsr7cY1Wr$ml}2AATg;8@%MUeL28CvD;9y`m2)wcT+zfNZNc8PPLehLK9l50$k^BDuJ~ww@RCEV={{XnT z?kiI3Pt$CEB-;E&y=1huf*Z?-!xCL&xtv6p0G#EV7VAla%(TprZ<1}X$1HoX9#3E!up}yugw)sL#CnE| zy1tM?{l00Fe$r4ZQgI_-e2IX5W55`%T=BNAqTBdCO4Cx>DI$guvwIbF+Js0!AKrE7 z0XRO@d*SR_K8xX915MSf{x2tz^jH4?fUHJg zmr#vQt3p~@{{WZZcUpK`I}PX)_3 z+FC6?Hh=%u`TOCYfc!^4gsg700~L+^%U(3;JiALg(7blwXDT+FH#qyl6%T-H>^>k_ z=sJ4p7VjR5s_ORA{f1EIf0C~pXL{$&S(oP+4c5I6Rn&Cf5?x(K4Y0HD>s(20Pu)jG zwjr43k-AJ3?|?DIW&9oZS%0A2T3w0R) zo5HLoDv2ldblXSx`E>N#@VYgjDeSC$1E6`{6i8)?c=XwBtZp>RIJV?@mlL{{7~}3H z2JSF3it#@YX}%xUJ|Ue>K_~o4{@16q%+5Ymz+$kd=O1_HJT7=O?w&1z+Ak8xbFFSV zc87T+FuOX&ip-`kLv1DSWMHbRl5xlwuM_x#;!Q@$H9rJ+O6BFcBJS1amkQ{zoxm=@ zxbor7+;`jcuQvmZRb?5&M>VfyZ$Hz`6;~9RU)S~j01R_bYc`sltKVsoTwFBPGFjU* zLdllB7O2~IC}W7NBQ-#y}L8mG<3h6Xc9o;Z&z*udqE zdkWRkd>D)2Jv{hru4lSmvqF$fFPShaCl4k$EMsH4B!wh(>s^nHJUMZx{9un+your# zd9}})qbjgRBjt(0?Uk5%V*<9To=~Y;5v4D4TkN%3FP6Q$&cCg}sLOp)`?dc7Bg%FE z02NL)B+_1%OnO`_`hiC_B+sFeo?H&a1mY-p1Jest2xB6rg zJo_Ya!dYU$K-mCnQm9)1=bw7{g5%*^L#S$+zPx_ZbAPHucEGU_BiqR%XOEfp5zz2> z9R+=(qUoBJf#ACs{7o3OlFkpeTv@`qokZ$9v@UY0q;Z~c#szk9CN~XF4;W$Echm6b z_cK*t8`O^W{=VdRcZ+pBXT?4|pTl;tNe7jC9G4PXkg>8T`9qZZ+nJ8l>Pa7i{vch& zVetz{@ZnFiTxqb)buzhOZ3Bqeyr)&$jghK?OCO&haf4pd;C)X;{{V!I_1k?m7%r|Y zZuN_ch_Ya}jx3F+2n^FU(mclDSB1uF@2l%G+Onhi*e`RBu#T0M;e@hdB$E+IswSVWqbx`JSC+3M)3giZ>R9>lEU{X zlPuH7#xNs=e(Yd?k*_S=6WCX-TFI~KKeEn|bu5Xe>M>i%sI-1ihB$4S<(>~V5C$^> zHZZC{I5@3ehW`K!JYnEZ+3QwGC63!x_=By+e$b#Sk?su4sKk$*a4-iuvIhdQ%Q5uf zLU?J~>ULg{>;C|N$L>DgFT%y*9iyyi%3nK4uf^Nse7gpUx;Ml9diPhn+ck>9*9`8P z5<6=-*hI&sBQj)&$oaaRqX)zMTi|;y4txOccDJM1i_LFWwwq8@NBQL@V79i!#zx)V zUhY0%GJ03AFT!83e#;t#mZNWT2Cy`_Zf`v1^A*fi5hSOsIFl$2`xj(HmiDFDBlNa2S(}$vkP3fIe-bzC5xYg=4__{kMa>V*=_PESka_jcQ1ym+aBU zGs-P(p;5FJDjX69=Fd17ucEKUSZT^?E8*pRJ8SY+N%`n<=6E>L!}G&?f51!bSM=2Q zi@=@=OWQk?&~;Y(CZ^X5G>Hq{By9|08Bx5fAo5vQGdl$I$>P3;@J^d$ug@;2r#G7S z9(3yM5~{%iMzJtb8$UY&!?*_=S8pGMw0SOkU*Q{R^xZR0TW6YBp}i6^TO_SAJHo0S z1Ung1+aMk=JjcLRH`))5EUaKL&2OeX-MsTIZW1dj-eF_7=ZH)nCLFB_Xi zq-5ai*4~X>d2~ys<74H%xSuaWp!mb^mq+k&4SLEU_KT;K$!QYd8H_3kmmLB|!;YoS zIK_DHfMwhJKx!~v84iuQ)&i=aBF7Xv5;Pn!`PsX00V2M=_=lm#uIswA6Y33RX9SW@ zJj{&7Yk30v$dQ1-^1A}JKg2#!UL)`e!JZzo_}6i&UB|23pRwOw!9CK&9lXnL6gy0C zBS|A5GcPZ+u>^sN(=5&6uo9JNUQR!T{{ScWqkI-6Idb}cU)N*SH9Jo?;?|E9(lmCK z&^^lAgxCzZEDMF{g$gp*=NR_wH4CV`Rj=6G&uX%y8m-KlcEQ4=Hp^-pp5z%MKsfn{ zt{X)7c|M!*0uLN&da;K}k}KAG7mbYaNU~w3XUQxhJC#)60CG65uf7TRGs5>8jgF0{ z*nO&Lf6@1uc1Wh+9fxpII3t*H$%pu}-cHO~Q;p;&?pcH|B-NgkP`Y2GEd(e&u!j>k!vR{m>9 znVt3kL?M`v32`3mVQ{00^sNdTZ9m4Bdhdty?V4_g_oCO&m(1F2$JrHB92Ux{ZvD@C z;Cu^b9oCRE3megEaeDK;%iFNnuq(IB$0b-3vp%X1O6Q}PM($nGTl~KT_Z^C)VwIZy zzpuEXqj;~x8uf(hJR#t^n&8^o+baO^MIeWAGL7!c17wy9lfcJMAGYx~iLQKNb!~Ra zWS32cQGzJqOs3^ke8zvh82MDE1e1=H?mrCEUhubrCh=#5Zlk}wnm_E#DS;**w%yI; z%P<^eB2qS}!m0bb3h=*-zBnEp@vg0*cy84sw$SZv%TGLv^GPeXh$h7uJD4s3ARWLe zoN?@6t3q|EQE4TvtJC`ak}hUqu0@@_u(f7X-0e9G z7S=pq9gQ;=Z1Pocj1yTtJHEV*%1LzVfGyo%X}`T7-WLQ8S=f)y+OKP#DbuwHuWsd% zRJy)qxZ21(fh=ma@F@N;5whVxI0KS8R}JDFV#>=^x6|~}w=rlzRz_&Z8={F_c+bqm zi6G+xr|{RQRZi5SANUSf1~5IV!hCnGYBTtZYFF{gByT$HVU6yiljZWGbFp$8fCdy-*WfT$ zsN}R~pKHqZHZ}hM5bAy~yNkmb%o4(;WsN2KJEk`C`>6qp;A0zxc<3=+0j^x=ns1J- zEiUig&U>RQK3w2__6^27iCg>25jYL^c_O}|@GpmaIpPg=^T*na+gfQ_9Ep7_FDpqZ zKsZ=3Sg3ZF3yii81AsB{oNfOA38#-F@s_0=cChM~jcU;d-bRK(sc6KifO%&u zu3rr6xDr4hSG(!jb)C)QKTPd<+KKs5$$&^s62()((@T-03m82_RMpBm^RPtTy8~%j1lkU}I?c zL9dz2E6)*LG%&93vBy=<5m{29t2%!U=@t$m@aCyzklI-mZLm&ek)%L8nB<56r#nXc zoMYt~2ERrA3Vb%uMbFyxE2zKF!bXece(mL88d->3Wtg3+2i;W)G0l86;SUUH`mkL) z!!muQ;`S)fu9+Gw;B8xpB44~dQL#zj4_f_W_+Rky&cjN!{>+j%1W}^Kv4Mm@08pTe z^zJcV*8Ugp-_$VNstcO__ML5S!>8VTJ`W73N7=`JxcW=Nejl|OOi6GIrAq)}Y%T*3 zFg}>CMYPa7KCuU$-fm9sz6TwKGupTvGe^FP0Uf;4D-JLm?In-*YtUfRHS~=jwOf}w z0LZQV2OpJw*$K52qYaNiQBmERwt5$YCSm@ADO1;TFK=U4*G2GgM~KD4^Td!Ll# z8Dls-xd$9nH=YvKQX;ZLD0t3xh5rB(UR`Qc9?-GXNyEAGmx(+*5?DCAoXc2wp-ixmRkt!V^28E7f9E4AP_f#rwUcN*Xo~$ zd;x8KNY)TD5I*{I`IfJMJ~;dm@T3`hLI$l~`M4XACJC#Y@%AP?^11TfD zenZC`DT%EmR-DpG>8|PM{&x9!9<~mZDZW{s7I=fhzAEsyh_5^~szQeD&Sa6~P;ntr z3}p$=%FE7uMldl|ylJG}>eg==W=p8cByx}lCN^I$a2@#LkIKH~_?O|Y3;bmGXQBLd z_-Se(gvsSy+ph5-5xI(VC|8Lsrv(21x(}^?}^wa(OBr=HsE`wRRsBbd65KPJ7v9wz`W{kq4HM9KzAF2@9UR2g`sy zmAmlfEpNg4WL_2c!D8!UsCk!=%Id5mM&WQ( zpS=5ioeQ2H@JECGHF!r-wDCMm6jrmC!F^^We8M=9Q#(`=H;e!SXO5+6?vG<(4}*36Cd0_nwHKVl3m0WcH!sLzd>}p0Wa_eRCKAQ;W zQKwE-Aez?w?Q`m1gFXek_>(oq#2*pI_K4F>Ha94(vLuXz6UNF&<)g!=`yJpQf^Ya(*FRGe|5Wl2a)_G_-?vygY{nmfsgG?Mp;={?v6X4 z2GG5MD(sG+t`eGQkIKYtH-& z;V-jlUMwy9yGd>y>M0pc#*t%LVRQ}0?uwI@#!hqAy^F+tD${JVu_eI?VW>08if}hf zBasiv2g(6laC3}ez6XjprnWAtrBCg0x9iHZ`@b%SpO@y6qs*UXm*;=P!?e<_{u_9Y zSJgwM+_9F^f(L0`H#;^Q<8B>xkgRY5j8;dB^{Zj&k+WjOAy+Yw z2GGpR+#I$6`9*76TKJ0RRV_R|M~BIR2|Dd*Q!*r(I2&?C)*tNy>0U?iKJQSuxxBG6 zM%U|bB@!?0Lo@uWk&JZo&ro^*MR|FC7PVTctxfE{?JcIn*UV+ClCpaI-QW2ymAVl4 z^Te=edOn?T;n*=7eWD%s$V^F>AoV0K0f5d2&{xd=02J-)t=7gZJDZ8*{=~R>5uH{E zARAj7muoU_P}$1owoQ5uhJGMw8sxf6)(n#A)^f*i;~^SBaQJRpAS$U~!_aU#R&DN; zr}&K%MMO)P*zQnW9b#zQ{`Ih>$z?6T`?wYN`7Ie?swZ20f2Zhtb~hg?t;s%`d3t|c z4)a5_n@iFry0loMwy?gtKqmeuQd8w7R|h3_r~{1qRGRmRZ6xtNt!(zbR5$QOu|lL_ zlr(W7smVKI^72M7Iq90hxxCYEbf|PYT)cr?Nh*=dtI1$HkCm`L8t}b8#g`X;DgMd7 zxo9Mk>O*RF1_2Ty<{~QeB$1r|02k90>C?$QqjeWWf0w8ApuV*l6wDzu^P7T448x99g7e`s2z;_SamCUTJYf(-`q zZlNp32A5|Dhzw>&lP@a}2+{X$!@|<6+p=&A7|^-(Zg>^*$nYkV-Ws-pSkvLu ze#c;w$>%fqCR?VGRwfie3aO2NRwpALE-|0gaxNL6UTRf#)9$gsJWGvBQvU#k*!nA7 z@y4Nhqo%L5XG=+xNjh!a0NM-g$l6z_!5QNj74KdI@dNmeQqz1gW|CjA-bv*^*hY}X z%G~bbYkv^q_X7M?UW<~Ntfp3qD~M7u9y9@6JfnfI^ZA7)v%m{U8nU#p+Ne2LM1&-VX=mf2fnFnUb)93xwm%8X{{UyRuvomk$%$nx znPoE{;6^A=^x6Oet}EJgEl%IScbayWp|eAGePc7veFy|v!b&T|va!iKzGm7mKt7et zYtZV^GhbBzsu$S(D^AmbZ)e%ZEtN=k}I^kw?Ir}jfNDN%Y%}~b_oO!K2gB0u08|n zJ|2SV^TQYHWVyGFOSOtO-b1SHk{!>($e}|XHm4)0ug!n>SQj^w_-Sn7j_w(7sT zNiJ8+jzjLHHu2Xy3NL;h>DFE`n^Ex{_mgK8i(Oo^M~Grf`L`06-MjaSHtxr_YF$^r zdQZfk6nJ}G@T}fOpBaMbUf}U*ZxX)!;2)Yu*gR-C0D#d2Q@O%BoeH;bbdN$^v_ zcNh8vkBId+X0uZaQppX;Ho_<*IVN|1xTQ*=U%XGtf=(*#z2P`6HA~x@*dco>?M5_4 zg>n!vX9_Yhv9g6M#OI8jIIQh^#}?WLhP5d53)pQm+e-(MId3OeZ!WH)F|*$JXN8Ej znU`{)F3^Ixj~)C_ux}5?;a?5UHT;*?uvy#9$CGTgR$@hwax&r3w(d9{*ap6BVez#q z$}}|mwf>Q=E}SXD*u!;io%H=b;23ko;e8uHfjqU;8rtla+qGY85*IO(g~}%2!B1|T zdF*@)<^v#ZnfUSnceu z?My2wfEGobXZt}ec#R`b(;Nb+$*-7;>hgSF)wHWGv}#eusSPPbvm)%ix0B3ov?^^e zh~a4DDZ2(W3y)vXi~AwrKZKqrwbDXB_Li3ufo_0^U{dl#gniKNSIvHUvaj&fzuDH$ z!xuX3-k|z?I)&egZKv@xLQu;TLKsj+Bvk_vODV&ye}^Za1L3IgZF~d7Vj!-biG9zv zZGKvGJ@{n_)}w`kw|Dc}-dfvl!10SuguW=zJ|nI5;aj^4%SE%6TZoLY;@Tk+$l)-6 zDO?rc0#6tf==Z)3(v!m;5z{p9-p^^`J834j*mg-EwMKSbq_@hlw>4$wzA}>mJu*nvFdZ#kK(G->)LTjPCT(oX5G@hm-73M(lY4N!Qw04){6YztEYeR zKPmiEq3S>IrfhUC4e7GM{kLyDx0K31b8m5c(e2sE!(7OQQ^&}M7#vs8J`nIWkvGFV zC&QYB)xq$_iKIlgm%$3NTG+y|$>q0F_Yz_x!OIqHrx>qFu<$Hao)@rPV3U2U+qJdJ z#hr#QOUR*t+_Az)`>TwNqbxxmLU@bAR(>e(zK!8j(PRBDwn&hD)wgD89G|};F~`lC`6J>#fb29qa{J=1 zjNa~PZm+zp8s298{N~O!hwTlJ1W6ik=CJB`PC5Nh@ehT5&3oa*(ZkwW&42x?cP;u2 z^Gfj_mumgRMw8|%&&jk9Ghdkh0JPJwY=UW!_|~Idh}j)-?qxVU|h`C#)L7+`iN0RVD-YUy=PhJGxYO`pPkJeuD{w~oW@x7v-|QQUuL zTkcY}Eb)=&$s^>5(TN`@>411^KMZ*C{{Tm|UxPYbuZH|JY{O6f*wrPDJx2c2;%1Hs zE=J{7V%w4h8#}PZEA~t#O-iL7VI4jczrwY@&f0zUS|6WN%ikhXUY?5mI$Q2NgX3Ss zIeaDIFBI$PW|tSCppwETw`q6Da}aqM#y)%@Sd?u5FU!=L%lL2Ot$SGTmx=H1A%+>R zJT*7(RC#lmeCVT0$~l@Xip`dUIbX40Rg$Y<{sXSJOn>)3xl6 zM-6&dl{KSpZp&o9A>R*lNAX97yhHy02@a8S9fjdoP`+psZj-O$7U!uEH` z=DIx|+f(q~zv2x$SRdKZXgYj$mwJRdogy*8`^c>pFvZ?AF1U}7*%=#7K1$ZciQ}Cw zRl2Z;RM9lqjk`e>qR_iMQU?mEw)r8xUE6Z!cUEi`uhKHS196`>mc2E1{F?OB$mXeD ztu&=AE%Nf){sG*0%fvc|h&&gf>2uvncG`{H+M7u$h{nfuByQso#@}?{lBzN)w4V@s zDEHTzHlZoGw9+)7(`0Uq_eSnZ>lL4H8^B39iBxnTtXW~#=KO2v>fmX{ zl7v&gOMJht>c^i_v$MXeqvP)sO`+=WSZEfoJojpC;f63t^Bs~uFr4IJpE$tU0Vcf1 z$2xV-hrDTjd*X>KWml3*Ycp(uJC8gkm{D69WR4&Vi?#AfkG-5r@KaOM-{KyR;_njN zCFP!(4aCm|ojWyGhgZ>*ssZcQJ}KmT6T}YP_ME zHjLy6Scv0NNyj|~YZKwe!;Lq`zAV(dQ4YNw*Y;M~Z=^Dr62~7nj2)o^6~i2{DmM|v zE5v@;q+Kk!J*?5&7(B_PmI&vR#L_X5^5uM>BMf$i7*m2rL0?*YG+zg3Uj&Yy;3ZjN z)%6vQ^|t)?P{LT5Cl8S9V^j=&>jB0G1MKQkjVODm-PtWVf0es_N2v-iwVCAp9@Flx z{55f>X?cp%#Tw%86DA8sYCP3IyKtf8ksJmAfZ>KlD{gNF*lR+@*GY~xxUqOdaiV<8 zV+=%qF5Ge%0}nykyL){bL-3A=q(k9`mhO8vA)a7ZnPO|0frzUxk9xVpuTj{>frbLsd3kdty z1ypV#L!2&t>UuRd!g>t<01~yA@cs3@!)bbXHxjkB9oE)2A{g@L0$F^v+8Y>M#ZKiF z>z4M{8mEu+y%$V?X_}4faNb`1mL^5FvyA@$qrg0}r3|}9?tbDCj2iu$z`hi6uJb%f z@`|?aZf}*X>wd?-!}uw}^Qi34X8oSN2Y9dk5-YC}-Dp$8ZYOIyQW_>yv4xpqD~HK1 z{;>4>-qre-;ZKIb-66HNw7Dr5$IcX=Y!fdZPL<;S01xy#T_Z_`)gp;N00HF7jyr#> z74JGdw84Q8*(wkD=#u1*TyOm=`tL5v@Yq(|BdgWz{zuTxQkqFyPcQfF{3X30Ga3uq|TWyr#W9j^Kf)XyTq{5tI7J)`RvMWEWM}P7)RAsTrYJM2f1IamD+9`>?fmb92{eSYwKH`MP}T#?{fPXuJno^2t4DU>0Tk@RCkEo zOmA?*hKI}Y{{Rr9`u!`x&FW$5Qfi`B^#1@OzYiWqc6_z)7g@dVcf`$W;Lfh89-Xei z_Uj$0Rx@=N2`ocAZkjwc2lq+Ee4X*1z?U8b@qD^ZmK1%7uI!aE%>Mwr*)l6&2J(j9 zyyG6cdhf=+4eAB0{7vCqM&=vsHtHL=12|9LTR!;rqXOF-8(VG;!hLJszBzbK?^Vjm9b{xhcuAVz0Ibd1nR5=TSKzQ zqdjmxyNm%^>+t@^N7b~g95ilWRXf7H^WJh2yJgEuokPkVp=PK~2 z^@^U(I#2rTZSH&UttUxer~ESgKLfARF7M><_06nG?Q?Fi$Yy*nMv$zF8n0YQyJiO) zPpxpCBhs~{@JxDCL1v3h)c(;D4jCeLZOgRto}6~Ut$z=A-p@sX*=J-H_fcD|=bQ7Z zMI=62bHOfls;dAGLs-5e@g$b^(Ar+!rKPpRsc|f_g4;JqkOC+S%&y0CWA66pT{(6d ze%dyUw_k>bH0nZB({Ec_@VEIE^lut#I^Tw`J{!H_v=Y2?J7nPMeeC-ou66)lI zh>kgrkG;mwN7lIsH4D8No_V5`P4rJ~*ZpkF`$7P@k180Rn^>F-SKHB`p;FOWbhW>& zk3Soj%1SY-v{x$fU-0dnj*sBoA5XA89ky8IlFmt_QMU}MZtEm(@|-fNHZkhLyy`eK zZ9+C}Le=4)R=$@`lgjy9C;eY>gvZQw<8^uuh3CJu*8c#uFJ>``AU2Gz^KLB&hIrF~ zyLQ~?V8_FVXxBipmQ zH{f{_#V%fW^#1@YKN2kl;j~Wzv%I5~M^`#gZ>qM<{cr<2;f`dF??@q${v^6Bfb zy*R1DQCojs=5oF*)Ftr^sy*xtaQBTpj7))07&n$KrzD=r#{pO_Ju8CvmEl$J{*_^= z=n)HPB!R9cjX)pk*m*1EvEfxZU}xqh73dxn@D1n0-8S-V6^*W^Y&@H5DD%2i5tlH= z-JL?n0%QU~+s$YA`WtxuAzEGC-YdkHOKWLwfX8aVu`-N!hy}sj&UpY34SSSx1qniu z*OvM(>+w5fI)260j5K?Bf5UClq4M^b@V@^5!@mY?bxZP(f2mJ!(_piyW@1=u7{GN5 za60ETTUzionmipAsd}Kvd2ek3hT|p(8VJ;2AMc*My*gLY+Liv5;=LD2@PCKnlHN%E z+o?&lyr~ph&6zx+atPXfRqh4{6$Ps3y32pU8>_&}s8~ZRkTuE6yDP}bKp%CAk&ZLh zwkzMQnpU;%X4`snjorF=bT}R1Q9&)0zHIr^SMX(pzr=eRWZGb`w3bH*KIo2Ef{uMu zU}KupZD&%@d|L2nv80h&-Ay#FDMbNFMyeQ|~A-Ob9h^BC?7f+U`KKy*Nntb-rCb`h>|w>!I3 z%8cqLRI2o9-=FpO9(|mu+6%Ybk?4B2gzxqL00n5;&88!Q6!OBffGkrx0zxkv<73Dt z?vPiw$5Zf*%T3mqw72m@5KV8dUfvtKW=Gj^IFz-uOUKGCS$7T(EH%P&^wqyROreMp;f~rqLkzeQM7Gp5@jBBS&+nrlmrEA6C zveQoXw~{{`#KHScSbg7f{=AQDlf+P1e$E=Up?7H&o>Oj-1&SDXXvw(!;RLZTsUW$) zW5*;`Z-f3LTJw5SM-alr#_b5KL!-xQ~etbPUR zT4lbY;r%DV)7jilaD_xSvdIx-jrXn@5yH6xs2Rm{{{S3*9%;TW(0niOr$oCNUYBJw zHsRw%xQ(05iZO2zDFp9GtA(VH(seI`I&;XkXnJ!lzWQYAy9OZB;@7MTHb(<*sLE#HD zYrP)BPnOylW{N9jmRm`piTuYnV7nN$eKG?QDY~`)0E#qy67s`F((SaDv+*XIWdSGV zlF`ra)a}7$c=xxL_i~`%?ZEoxPXcM$i`vg;a{^jDsaRroj$>Gu#S^Ge{0d!T!TYXz zb6*#qacaTxIHee0MW)wV+sX9%`ksTqL+Z*~>X&a$o@c=tSA;G6IpKQ`0j8>v-`dGN z_P090B=2zK`OML6F@#wrc5%3awNs1=`X0mJ){Clq8t@jACZQ#~-WY&S9FxFdmRv-z zNxt2SZE*_}h;BbLoiL)WYCahFX{hP?=ZfuoMQ0@UD-2eNEu`x4NeogWJd=cAt>tn^ z7*z@v?*e|B_@?Y+hC5gQxe9VP@(d*+uzzBk+7)I;9*%Hy)`ucD9yX7S>MsbUi#g z@c2}#)_(QcUZ1B`rmy62-x53?o;2}}o#XEg+pW%&+B|JAlx~gH6vC)6`@xTqa~R|t z0u4Lj?!9&3kAQv$@Yjp1{{XXu#jbAF3~fNl#ZopH5d+lA!A0rwS3*2NW>#4kM4zLBnq69M_co(E6pcsXX)CWTq%(&dd@c`!=Q zLKO}PjXqK3kC=m!cp|mPT}(zPD(?K6E|1Gymt9)uw$O~n7tIV^dUyA|KbbdWegojpS034Y7ha z_FG4{b7zgx91OPuj!&z64oRo*PM;szWOcjL4x@D?*9-QhfDQ4ker8BNdN(jtEV6NcV${07Et2x{w{J7rd|j^DTxwd~hO=m~OJ@Ve zXMZG!jJC;Y3R27EN9q_VA#tQ;M zo!pJp;$AE8pTH94wuI#LCF2baCfZ_#MBl+D_ftABY=A64=U=M5Y){-@i zVr?SQSdu+5R^6vQuIRoPl5cu97_*lsIR<{EF=Htjq zD8P;x=a*`$#Hu9y?4%6L7z7I0+BvV=a9FHWYEV(-Z$|XDeLib-vH8YnS}{s0ov;0A z>Hh!>eLlVyo8h0sp9ASvK31aM8t|R9z2)S#YS%MdOj=8XiCFB4d)Z}(N3{uJxX7fH_;8kb%g{XK};N|V*mC1sAQY1vU z^7r|1AN7prNcp(r@y~86J$Kj^4UE zd3x>YULC*D{A1#MS{*>8{r%=N{V<6DfIuR6qf;+CLL%>xfCH&jAoJxvv(JZgzlGX{ zs{~dre;hibI@DKJS1l};6G3<87&rQ96P3vT#_D$e0I|scbJ}Tk+BTG$r;RmvA@gXGXJL#O`$WvteaSqqTM6}#Ii`Ll%|C+??tP;ylW{wKibH4hc|N)Hl2qxgNk z$>C1~Tdk~7vDzN}nNlGkeC1#(xb-GQL7oV&1s@k7(R>rAcwY64dZwj4#og=NNE%J; zATKQJrxC=eh>%6qBhK-=f#sef@n)I-01Eoj!S80cz1B4=XOdqjSnQTmFuF#k_euh@ zE_pvIKk*9s9D^yvx{8F?EFtd3-P%dN!pm!1npxD8`D5jG=VhU(1@^tJS@@3s0A0U> zN{Gu7otY>s<8$UjM?0GGMLUmE60Uyre81w46vN@e;Z%m&<|}PZ^xa9GU6rN7lK8+> zi-3!dpK6ZvH;jHDXxeXvzu`O6qLWdFRMrv;G};j??ezP0WqD(KXEMI!7(3Eyb(Me7k!qg$|N2EOuo>4Wp2(3lPnWezTHfUiT$UBP(h6 ze_tyeY^UvQ89v2J{eH^lP`rZ8S`=oIJv=xMC=xNuXb;Rw2EzXUw55BDOXDp+O8AJE zdP7V!gteX|-8#Dg9B@SpF)x?7m8Fg+V4+!;ac%{82f~}1og-Gh)OG8u+}c4k_nj|4 zXAyuGEP36|_WtN9qny^SfxIu`?~lI`?L0f-DXy(wTajGIAp+Lj8UXX%OE7iXkwlDH zc|2?%cvtAuFdVbyld^(d@!Luo>#eQ-008v+>uV2(treGp;D=MvrEjz~+Df+zZm=*5 z5Ag0_<%xDs200nqpN;%kZ~p)Zhm5>E9gL*L+GH1gP-ToXlf&n;gV*L+6mD!40P&jo z590@nG|v{<>9^h-ys=*o3rp)w7B!eG&|B&3Vu-wMBHpu&(_DsbC1K7ZQa-8Gz7u#; z;m5&082Fb+fv!AEG&?z6A=rXDDC|Ij)X2b?_n7?YokH-)g!QjRjc8%L`kxfPHto0E zcHrrx<$vpQ@atTk#QL9(*Ws1G8ZbzNh_T5*mm0EiQ3%%I&&2fh!O+^0f1X1>2P&ETjg z(pJ;@`RaQZtZZRTLW;Au;&ir}SBc^AjQYW`n%dgq5rlZiQ4o$OU?kuLbuneea04|{ z@xn9TT3L8n-SwNRxfa?BI5JGP7ZZdoPBN&6<_>;X<3h)UuDirK%w8JsZnv-M@LI(+ zr#DLsg~Itskk5nrv$vl9;}!My?3wVhRk8SJo;dL}jP}?1#CzcLp^&wmoQexP%n%6P zRRFSswHLo8vGD%@gzD#coKv)EMf=YE7wYz3<$Fd+gPmD@YTIf3c@ycr4by*Sui4u6 z{_9X{jDdA?GL(>AUK|`ocyGV5IFO&a$8%oc@IJ>!)b5|gI+dhJf2x(0m`sY>LV%+? zhYFw%(!8_7`d)$iKzPP873`6COThEb3?M1ncXJk2dA8@2h^t2s^$bRSQ(vIJ1^gEj z{uQ;={7kZG*Y~O=Yi>(Ghy7ecs{y%F8?n&lzqj(-1sGR^FsVit`<*KQ5(XozRBh>YrYq|bdnu1WOO46ZQLpRSxNk> zQp3c$FWFE-qQ{io{{SrC*ZruyI@hKymGJ&lFDjSXK2>=mC+ULU;Z&p3JQWq1CAF*X z+A@6DBR_~Qn#byO7S(?QbtlCx)L|=+-aY z(gzhMi7yxBO+GE92oY`o9tgg?eMkHVLXobjc$w&gdzVF8V50EU; zi)0E<$}Yo(1Yqa)pGxt68~i}hZtb4SO=~+)KZxCJ{8w&y?aubmAjV^am{jG25 z9hBoPv9aNP8Yy+T@b$@)_p&5@WO>?9jh{D|hYP{U$9nu1iDsc);-NPv?QgkS`)$(qy|z7E z9BRknc2CIVziYn*S$qxn%F}9=u~^vZ^L?jJnl^2MQt7@mP(~TwEw#Q-hpq)=_$Obv z*L*$V4G~fkEHSmkj7~OemkYCNeZWUtoCDBvUuAyMYw%a&@59@F3|Pq|x`XW1bjgHI zBN@zPgA#cm(V8}4#tu7RetP^czk+qQT`yHgbqgpLPq~D~V9bEyXx*F?^Egm{9+mu= z;@<>O!&dsmIKElC^|JJK{;rRU;_R7Hs~U-D{XawW$HIOIg7aSR1^0?h#e-LpMT8(} zH(nz}zx{$^m4MG90|OOb!n%Z-Z-{gaG~6uOduj5=o0H}jv@0o+GZ?{k8H{WL=K}+! zc*ll3Yi?f38+K2%Y4=vrq=dPUJ-IQdW>8y|3USXv)0*|IQ&iBdbR8{p%L`JS}J+{;0f2rkRvA*S(OMk;02klX5tLT0_w$`+@ z@}QnenX?jYV{>V79JY)fA$PQK?Ev-P=nZ-V_xd%SrK5PCUHg2|9X5L(v&Ae+ZWcL7 z+~7vcSl}Yxiwr&`Fq;?hg z)^WkLd8ax~_WuAY+TYi04tU~Uvr%hDwZE?C(fXdFZr&QWn(`gWiXu}M+`#E5XwSJ- zWMxs?rFrJJ;`4K(TUzc*hIvrF{{WFn4T|8uqqKcjxW`K0)HMwf<_WA)Rr05g%!!8w zh4R9M=mx+E&Hy9w@(nXqmeLEK4M@(*W<%yU1Bm(gbC*r!opQ>fp%@tEry|Q7>GRuf z%l-rYwm7ks++d=Qd(}0|+rPDF2KNU}WNb#cGDiI15C~;MkI#<6xbG8cQyZhD+TPq- zMz74Mit4PZ_Yuj)NK$%a6O44OX?$>3QhOaF36|n`BNGgDRQVSS2HoFx9F+&x6`NzH zc*j@qBt9Rqk>Qh2l`ZXI+)8XAicIUAM4|AZ{$_3o7_VzA!M8P1u9|*}`uTP|`Qjxg z+W!F8k*DHs5_!=MYP z1>>I)Ky*w001C*P)*E0Nal}!@ZpkvKZi$He!IOcU=DMlURc%)X(d+1%)M_pt9Brko z_z9zUyF;*&`u-hESzcX*5!+qCC+^q+u3yVmZ<(-jw2r{>UV|5cHMx8hrQcj%Lw_V~ zb!~6 zQ;iZ z1{ma8hLgy}DaTQ8N>zaPB2@OMeQy!feQ9-b0q zg=U6UbzF$Npk-jE9BytqvG%T72eO;vJevNUa{DFnqAtJ$+vWVto!BITdH@G*mGx)B z)q?WhP}FWbohh{WOi)bm9i{GVltCYwo18woY&Fjjd{&#=nr%UW~Qp#m24MF+#_*ePq@;-F^ zsQwq|UMxq3{xtJ6h-6q~iI>i49#OSNns8N>a&z<@YtlXu_&j)Pz}K?)oLdb=BS>v+ z{LCCAW$?HgVP`Ta3CjrZcw)6*;=jVnUmAR3@cFXTY~jm&u4+z~!K5S7c+#Z}vi{ z-?wlWIIRs|SG3i1-3!Hj436LIa>5@_w@7mdxSmW6@h2dc1mtiryQOx26kAf!{26m` zq3SlVY8UuPCS`2hOh(jU0`Bu09d^h-J;h;o4^6VrbP04DxvgiDTVW*gI_-QMD>O?V z3?hwoY)q5i9M=B;+E>H-broybCAF<*{{Vokj(1CyrEj14e2@Rq=MU`v0OMYB!_57~6e`dsF)=hceYN{6Z!>~b1@mo^JmG|DFgFdO)wI1g!te14-^9A~ z39L;5TS0B7SrxakwLlAfu13fr&ON(g`EtQd{#Xj%(`}>v%(}kU^l$8)1=V8~Fu8eK zF^2L;3;W4DwTw$UHpuvnFf+LMcDLgDb-dL43E}81w7Da-x0g|dL{@Zy>LruPl4*B- z?{jRB@whJ3kCR%@HK~uQP8d&hMmFiD_tyG$zvJ7*&b)ovz|=2TGHkozMnQYv+hGR_e`P*^KBeAm^UD)AM^gS?R;VJ$3@ofyfL8Z@ejAnD;TXC?JKpWPepZ%@M3RyTRgsQ&TUzlATC_5T12VrbtFEHsADto%oH9;YswH1fiBDT!_u z59T@pxt+){gVP~g50qoWk_Q|oTzfFNhZBl;D?E|PZXOA z4QV9%NlVw7K>H_ac@YV^DuZAf#zHaOpUS>pJi}t08Aa&rzKgrjZPRX=7m`+mDk_aQ z>np#z-CKPA<9EgyuZp}IrJ`wiTz40ak%{B;SdEQ`JxpN&FW)T0<2AtiYWR_)d@X%5M72Sy-vydz zEnUGW@)IS+&U5#RkR$GKfq`C`;qMaa{{R%N4u|my8-%#JSZ7T(;vkA*kf?i!3gGUQ z#@JUQYJz%Vyy;=9PMi{ty4NlJH+JloeLmyoULsX1yfrDlWuok~zx+2{KJD)U_|o4& z(slm;idXO?b6MHH*{yBm2W*o^J^i~x zk9_V35GsA8K#fk;c;!VT5xDz~et+@R<-fz-MEGaJdV6X2Q(h!cEAGl!*`niM=nVH0 zs>C?Ndz+vc*Z7n1zr))1$6IXz?Ie8}Z68IlKmfOtKg#JQNIRNFWkY}fQRpk&qmtD6 z#VXX-xYhksdM}>Ks!yi>06{|$Mx8ZAqg=|-Np)`Ry}y>q+pRqf%WKUmTF^hVblJBS zhIF#DvqiPFQdi1^?Lo1~W_IUz4gn);|dJ6P^fFBvP ztwzH}w!DrAW7PcZ>mk6l94eq8K?m$_OeJ>W^F<^nQLW! zU+eEWFWQgBdcK2yXW*>`%So(iJMSiw_lTCUm513B$ihb(h)^-;7QL6?#=YY=_*bv| zNz`N}!b`!Z*v%|E(Hx++idDuoM7YYJ`{E#)(xEg+mhuHsX|z0)!*gO{oaSg)=I3_E*mc0`hHt^TWu3{FNzvp#y=O> zzlm>J*HpThW-OSMRxnJC>azTi@V|KD0=}agr8a&8c`N0Igc2oYN{frQCw2Jf( z4|s!Iy1CP(Pc;OZl$Oz1Ao9`>w*Ff)@8Vch5t=wkuy*-|+WPvOFG>5cicyXHx_@7H z$nx;jqYBEbdTM=zso(f@j*+YQs>STDZ*{5si%DdV%{Admb>FYC3HzDE+L30|Cfe@NNYJWrwD-CDhtq_?^;c|6%P zEAbbIyjSC?EG+yf;G1@N(jsjv<+u+Nc5oGhVRugD9AJ^RjGFQre*^e) z!(KS>zlWspZLQX=;!CxcP-}=TU=#hRM3Bd4ZhX)6aHd)|JDBiD3ySHXlF@OflC_s@ zy8NuZ?GHYt0%|*dTOE&!XYl_3hjcqTT|)ayvAngglVz35w5=MdhGr33`J0hZ2OIKO z4sr#2jqw9sip1Mlc$-y@<4)6{d#l;vic4EnJUmv(YTjd(F_Q8}9!SXDf@|qLOT+P6 zcr#P@q2kL`)dY7D3o9)yNLbxN6nkOJ*;b8SL@wYA{D3b^SH*uA{86oH@JZrbD$?C9 zWz#HL*Y`2`c8w~sFpRJbwM=7<&4Z5h_FOZCr8J|xy)=EkU#7=STShj&r*FefhoZxG zbe|41Wh`vOdkX$%-h&huY*4s^<~R?5sC4O27k+wT-3xYQD3B#J&sgf*&H^ zN0QAY!lAp=ZitK~2-_TEX@(SD0WMBo2Dxw9iu=V&sQ4dF_?fBCCx>5Ay}new*b*Ci zQVKO_Ok)U;~dX2oNl4 zIN+bW#=XBs(EMHCui4{8x7A>l%Tv0rhAm3c%?d$nIbki_s~{zdNYVr(zaf66zB18t z9~s;HAl9`101|zod7l0QDV3UX@Lb)gkr=ti4R4Tu{HQx%Vy^2O7fw=EO(p*Tfm=;H zmCpP%WnTDp)|!4c{=AOb#@Y{vKN9q>hqgD;MRQ}Or1!I?Jg2+WWLc#9LWtM*cJRR( zTe9$RGVKQ@y(vGiEw1nNseH?Mnqd#x9zBLswRRD+f=)*}!NC3yNg}-d{{T-<4S2G` zd->Z;(rzxL)h75TAFuH(tKm*f3(K8XF2 zd^N>41H%8*L@?<@j4Cq+W5(S?4Ye$;;$^nV2CHh%>4 z$V|HTh}L5rx*_5#U59(^17u!fA|J0A#w$Z2z*0GhU>=LjB^`<$3RGG_f%>xv46PhQHL(F zE?Ia7NP&0B98AChHv}2_AG6?oRcb#0cz#%mx?2KIIe0%W`1h#b@Q;dQ0VBP*iHyL3Ggo zKj+Tm{{ZabvWuG4!}2%1G`Uoa=kRTwm<+11`T60KB}c#dkK`*eRMK_pn3NP*A_J-i z$ogUt`tetFKMi>9+9o=s`vCkl>uZjG18}^31$oT(`n`7^L$q45^C@Y}??4yiTvp06wo7*hm+b^tmuGe}Avqa88EE9QTRz8<@{ z@b&ej?SyM-aktJ-Ha-6UyU9YkK6KmFabI)znq4XuYulpR4Pt7CE%ZU&~ebwxcvpfP2TeNf_(H_{YK8FM$3fPvfS% zBSrBv;cdRv8UT}wZizgx{8;lNKF$Fc0=-2%Nj9(IPxQ#7lHL{E?AFg2ni;=$I~LZH->$R|xxCh9*S-k<)?F zzt7xf4NjD&O0;b#w!V5Fl2^mZl(}c8^<(HM{6k}9;msRFOXxJ?sgdSPLGxx*K$}ow zDYZZ!HyO$31!wu54vWC?c&o)w$8&Y9p}7L`CJ`BCJ2ptC4A}!Q!;G-!Mayi5Cw zK>>^`NU^*O^2r)56a`h+z>;X{1MVAV$I_*{~u7PgPVT@$&*J^P64qCY9jNg*LaoEp(3B z1(->(h}>nn+7XH54Ww>8fjAk(eJ2V~p@>S7kG*)NwATLs@ZFA_Jz2^TpLe(Q@;^p= zC8_wIRG-b%5^I|))ybA16n^xCD4FXdfM)<;edC4(y1Ty)Xf~)U{9}D?Z!aQL7PcOB z%+bmdw9AB6Pff%goMyg6u=t0pP9f4~g6eyfU$#p%xRk)e+cF5%V^<2XBKc)_+}n;a z-1Hh>#LXJ!>2#~PP2Q^UxwkJA!6S50ClIMmySonL^}qv+SCdzUWU7AZHdg8Qd4I!q zCY9%9EkCZO)bVOoo*1=*S-!V7#^D5nkMvlNl1nQ|5X*VxKp5ZxxKqy^wx97o#7m=F z_}*Be(dUwAmu}Kt-ZqpZAxF-|rfq|1ASuU9roMAJ{;#h5JJU7$Q|0P>*`JQ8+ly&D7^UN)z_)FrRtKqrsu9)3UwrHpuGPJT7ok}W|2+Yg1yA0swy_3WK zELZUS{{Rs+c^B>Lc{1#mn9c3FnBv-_Js;*B_Mpvh^Y}Ag@h6H`#6BL9JAFp_8zpH> zxds{MF-7HM6(O#EM{dNXep>jvtJ*5{sY^({ojJPn>0{lDT~268>Yq zz6sY(pW%6}?UER$g7;0gicprXHurMxl}6nn^C(igvU*p!{>uoyH_>9#t~ERRTcp#a zf>`gZ*KDZ_$vE>~CfZ7&bXEho?O!DL8{lt?^)HCtC-BSZBGNf7Z*AdJo-;M^-!AKR zhC%Wb+($nl13AabU!h;J=Z7_q4L#qDwJjuQGidkU*%tOU%9vH1KYMbJ7WveL*%^_~ znBed+w!cZ@47QwbE>zz$vTj;vto?iG*K50@>#|Ime_4bTH|*2*x7lyD+9%U}50TKx#O9u(JXqq(utr3>MU3x&0|uqhSB z$mM00)*`>V97ecC`BY(jtDTEf@OOhWYrT1Gue7fZ$)`R2w7y-Ww#_xu1$an~2&7Aa z5^a<($$&i1;5GS1Rf+nw3Y6z1&b3OrwG` zwXaxMcvr?2aKmpPwP@}xZ5BCaVdbJE59NoAzFo>rPtaA3b5XR^?56lHp~v1W zS&3!X9Qm?sht6~$Hr8FZTt~R^fGfR+;ope7M_@IWHN8Jeo?o*oHPLI>5snw7UoFT^IUg?{r2hc4Ec*73VPZ7o@@}Cp#^O06gL-ek%g^%cRek%9 z0dZbQ@lU~;MDKUv4GA>!3%nEkuF5FflajdICIRJk;ZAuZ3iS9(K3>X&OlG{k>b0+Z zwer>5eUj*L=6Fh!+v$^p8lyz--+VL(@;KvW2I>!~C&E*-3 zwobyN9fNY@{Sw4mB>s#|Lgo-6SOLhwvK3-nD}a<%4^(a838 z^GgINauQV|%eDFC1AgG6Yel~y@qD(@@8oO0|m1M}}GE)Fh5;foBW^v3<_}0PDYblq{UQgJwuL=DsI+g@z_^ZV6Xqt_R#cQX$`<6M|VkAJDruji>9UEewFt%i1 z7v{#_iT?m-Z;f6T@Y?D=Cl+YLN)ihNxLnD0?fcT?1y*QNhZ~O67*WqYv##k{Cxm_$ zCatDjh+u0-B(oDm=dAK7Lm>M!F##>oK>3v89%TCzxB-axBT>*U{x*2aQus~amd}Rt z7^1Y3QhzTxh2+hjvWtQgJeL#stVOuk7B-egD9O9@?i1nabFDsqf*MP=p1yh}mGyo{ zpI(kGr6o#A@5?`QL&mkU$bs`b8dwqOoO;AN#;32(b#t)x$gsNgGuoZ zjQlig#lvZFJ%|7ci5BKK&_9;xw5brl!H7~<%-Roe`NQzT$NnSnc853Gw5jL4c_)g? z?CqIdWNrjXy-E3U$K?TcmH-o9Q-I-=>rL5m!9`hG)|Kt~U(opsMM=xumeF5^zt^Gq zC8YS{;rH7-8}SQUK6F~|!?~r2Bl%42Wh7$i-<H?skIVWZh8L!LlhW`K(^!*?9anoS8nmuLw9dmbO zaW#^8@L74c2|t${qZ{M5`Ic8-muMrB20p+2mZ#O?@I)3`%J~-a-Cx{YL^6gVE@LXi z$O9!`8@l6=HrI{C95KmFATu*0vbR74Xg*~X`VFU8T=-Q#|=1MspgL%XI^;J`wm=@mo^yU&U_^d|AG>g4V~v{vC4z8JNOX zRMp+YK}ZX}C^X^8&O?r!E9xJM9x2mwTjkP*r4^d^S4V?Kd&wP*x-qqTsog)k%_YQR zzDUWh%Q%Avi2B4ZwHLyge~yaYuy63`Z_;us4jT!FmMLlYTSaX;-cgeO0Jo2kd|~m- z_>1 z=vMCN*$D(2?mW%#qcO?^21RBk@aKy2@7hPgGin1w9tym?vekTP;j}Q@-9_fAi|K@R z1)=$!E^Y<9?(zl*s~UiyWY=f=AKY5}B>kVfN%5=1_R(Bxz8JTCBTSceQLe6-qy<64 z5fM|k1#&?Whmg#@-3rxk}?d#JFLK`bR|g)9|0ejj3r?`V?#}8VPPByOpJ2 zlDS_nD`gPpl0vCH@m_P|?Q-JZ;#z2zR@N5&8q@8b>h9v(fSF++c;PZZ418$T%CTB`Yur~x89>ZP&s01b9tI`NtCUxl?Vi5IsDr(C{*&KaYO z{^s5UUFuZjN|`44xW)(LUS<0td^-63qFQSD=9#12YC7Gz-@&KbMJ>8IEy2$BX_<59 zZOTWukiR)2IIp<3HeWteucAx4>;6x&=(IQ`7*2$pP5%I={{R5`H&WI<2Y7P(UGZj* zZqw|u{?D>+Dobm7tC5nHu|+(L^W7j*G?JLoFPv>8xL4DBev|NxHMs(Z#9jdSg)NoufOM}7Yxcj|Ef$dmrq(xpM0U?Ays|44IT0v3q?;aV z0e}z+^QdDKs=3kkcG2vTwVL^VT@DOJJxW)L<-XlJ9_``p2>6S`9uS^C8r;cmr|O<7 zj{a#p^NZ_EF48#yN5BY=dwKUqX!D-K$n<}R9v}Fb`&npu8B?d+T03iZp<-YrwzY-C zQJE8f6a_0DS8oUAJk`Gw`0782nook@(ylY5=;JRtN6Ee+_+_l!LE-D|Cs&r&!`2p8T2=74gv`?mnGnShhzF?N{{S(baWnZ)+S$T|*%kiGq!dySWtrq| zO>&+Xv+x(i4-D97{tK~&WYTYSC!PLaH)-yO&lcs+{Cqf5o@FXz&M{s227NksYo+eP zOZm9^e3$v!=2wVxskp~ztAAgYk@1(r-8bQX!VN~t!r$>bmR$!$v#_(b)8eyKOIv^~ zyrg@Q%Pi~j%O(IQk|qZIr2Egp9}4Nd0Pr52W8wRKLfRc)Rgx=BE>i=^bv>fVDzMy8 zG;yG3!l)x=4sl;G_}fI)W$~Toh`d8L+8$b-ys{B&$=fyi)&0aqe-&!3q)OqpSEPh;X zau($QiC^V&9yTvOC)YI(i99W&Xg(32U7kygN9_>W6t|WiDg5})nGc^IBp{aB2J+M8MBEvdtl>N=*V;Wo-r+$q{5XJfk?e8=@105B`Xpp>x+ohM}$yKTGa{LAvOp03z=rszqB>#Lx@Ntxn!3w8Jv4AcO`6=Xdv}UI51fj1ATCN9=>}x5S^c{kDhU zpBF>sX?k=q+-nxje3!R|LmbxD94<_Se|GugA#q>0-y8lQ+Wyaf2sIxY>kxU5X>93h zVP*p~rBv*z3Oil`J}K7{|?Ae}Ftg;*TCf{fn(= z^V%d4#b%O63d+YjoK81q8(1h0w|)^w#w&N>Z-Txg_~XW!my7&KVw2p!u%9YdD=3C! z42&Zn3^@$D_j1Zd7{z_B;D6bt$Is#m%l%tOi^#Z&9g7vi0<56!TzsQw7{@1^^cb($ za9kUR#=?`O7k2&al1V1D()vjxt?INmGo0RyX{5IFGrkx6TeFUHW3TAfM^LpQ7m{5) z9Fp8J$2FK7cjR%wU@Pta00n$X(q73D-$J`s;7|mQZ6h;d^JDdD@XrN)&z})A-w-YL z_J7(f8Kn|ieW=Dl@#PrgNdp3)V5#{|eXHoLC*hZg?XtFh9=o;&f@0LO{{XeC^z5I8 zvnmkfsW~>kE}v!kblBs=V<_9+OZxu+w~n?Ci+&!0=~nMfxD5aHh!?{9PZNcRz>q7}=Pz?$m+E91r<#jB)8#ExZ-t z(K(XiNYtF;?yjKhXL+({9g`cIdI-C@X0q zjQw^G=Ur*>29zV((ihGM32!4DegN0c<3{m5p2igL_Msw=`RS^~j-G;Se}!aR+xUpM z#+i9)7ph0BMf#6BziP?Q#xHfX`2PU%CXt;T)}JW<0Gaf){{V<|3(Y!M?_O3-ydjQM z0C(-j;Z?ukCh!%H%?+&FbSRU5LCCL{FD29#5cAty^R-)cvlzzXw13M}j(w|0cx%OX zcEzHzx>RnN)|6)*IzRHLP{eJgnAG3*UPOb+ZcHJ0+=rW~xd)FIv z;r$y^63=X!oI!r+lH7k6PBGb@o z=Fb!OH^EC{?O>M;B&tgmMgD#XUCKF=wd2 zw)f%*1d;i%Vc?tYp!^E1Fr6!r< zYgt9+k$wG@epVtCq>Vi5TiDp}A-I5z{dSYHt$#51&B4`h_151#&!DM%8R0!D%F|ZS zyi+vOM`NkP@fh3h%^48JtQUfYJ-v9y1oW;B-^E&`^{$zwOC!l`2ik0|qLl$I6l3MW z{JUd0Ok;BAIp(^pFT{45Ev~7o>z88jZM>5#t@6gRKoG!<{2+YFLk=6KYW_*As{1+9 z>aAqEUfsPrZ})y-ikkPLxBi#i{{R5>j}6$_%cn!A>d2qkQXAx*hsw^XI7SDLlnw#o z@~@6&_*r}6i?#5dgzObYn|5UKp;)C~FLB~YB}Z6;4xkYD_pf{SOX9e1h;u=!rmX_?5r2hZ~(8mv3opSAO>tn<9F9==uV$vTGX?oa>IWBynwsSKq zcMBj{1H45*+@zj!z-%4DuzzSDi!<8kHj=KheG<%JD4y&AeyNL<~ea=TJbF96*dAW7jFGcxx-1M+m_(glmMg1e_4+LtS zDAqKcDs4_^wK%ogNm6<3ClTov@hfk6?6iXUOXp=|G zWCim8Yz}dPae-YQh&7baHGKoZnnn40n{N@xXB=(vy~0^LM0~~_S))c}7{DN??O!d$ z^j9XBx^77c~#QRIer!ACP^xFRbcAl0!LN6S*i99uV;K|uxy0p2I?O$UX zH|`DO$+Yefvnpd5!OnfFzSjQ$X!$-K_|HoCXtoi@J@Y=9bEwRB&k2zt2+>AIma+N% zTXENBO?=Jp-RyK9ih9R{?j5x_?zI?_NYM`TwrJXP*zMKh8%j5*#}&?ebk=nnZ;l$x z=DnnghKZiwTwJgEELM9w!E(69{{S!m6$UoH%6ee@+%kGKvGJ!}UYmoLZETY5@>^_l zP{#W@j#T=8Nc|G{yC$PI#=UiKE-!4YJauPh;z26I=G#F2TyWglmlC9*MDWQr$ddyl znB;+9N8I?Y!rnIU+AXETI-iOBGL~z3Gu#N|Fqythu((|A+Q(s1K?iBUuZ{dc<9lBk zcvnF9;jic#d+Xm0r#5%8-mJ2{zKyO3<(ylhI>sF?5GgW8Ce;{RZms_S9QbPH@8d=0 zp3=c(;r(jQOP%kelV^~YH*Y@V!36xIs452x0mwM7mc!$#)1eF`^iYkqR=hcO>hDfb zmrLH(>VCI?<0)a(sz2USsULbfZ@-n)?$fjB=zX8zj}_W@2TFs*In^K-D6}M6q zqq>ok&yga*aMIwB^5o~C=D9zNH`93Q#}a9}MyKTI?vnXaGCj4V(rx|umvfd65lk#9 zH!%ccj%&s|cky-)5x%40i_6*b?HwKyzJCN`b5(Yq-mqL!9c z>*nrK;zjjesYW+#-{qM)90!?Eo;JIwFJ9{_jT>foEjw8o4~kb^LDeLebF6? zLrL=FHqoAdbbl58JdcI?D_r<)SpNXxRrYj-RbgiLCw{;;viyw91Z-Cs$oYUZ+G`q5 zio8GY#VkBR@ZWfdo!3mvk&rIs+`zF4sJtSU z`}X^fuys9g{v&uR`+G|tYu2=Hn^V)bJfR?(H1lGPYmc5piP#m%B?eS+vbe}}JI~q^ z;C8)brAKw6YPRB>R zmO!m_8Hg<7ounuV-Fpr?aPNcP@SABqCz@{-+*@g5K-3yIWiy|d8bBg-+<`xQVo%%+ z!+z1uYpT6E6roaZvi4GsuG?E@t(MnIOWf9ZjB8blt3vm>dT-L}w_eLlG&ygDR@Qc3 z3@6j?T5Hpw-MN+B|abm%sNh5V(y~CElWB_dd7C)OT(?VrApAn(w{Finpe|(v`Wourpo(iuh?bWKSsQxft-_+RkiiH zww78x{)hk6_b-T+elYQWj^Jk1PL<&=4QO^U+a>!L=NH$9p^P zZoqI~9{hXZT}tA|!hRdFw*J8I#lpv|-CiH`OBE<1H#4G<=1%zw964qLunJD%BmJ;e z-zD(p=g_y)N7LoKn87+U$Nq#8TzO&V*I<>u#Q(3R-s2e9w3Ik!kTmjuD3Vfa$}ZxV)Eh}h)YT2lcNIc%uH};^~?#13+ey`r=~5td}$nc|>#^i6B{xR$S022IR z@Tx`lQ7jrBk!Pyu{v6S4{Ldu20M42@)~7W41rkdmjP#J3!RF z6n@R#7uBBHJr2)Nmi9#4u)FEOTT#^{ z^LyTXJ1gm|Zn`q8vZu0miAPCjms`7APfwCa@ax}z9t{1K{{Uxuj|$mp@V=|!Jw^*J z3`OODksD-BKI-B!dAEjeko>EMm1B?YD_4>H3Gx2F;m_H}O0qXEbE^1`S(#Q-zFqRM zGOl^tRrtr?JuBt!jQ;?%7mGeF{?gapBQ~GedUl(6r)jzcx(3X8C7s%~oK?8P%a^#g zEHk=RL-z%HXTzI&h&~tiXT;Ol3yYZU^&JC1wvA7nb$a&_xs?KsnA^lo7Y)p*k~ z%4@6)XP7(AtkkWhp5&~)?h0j3sB7`kw%K2Wz3IxaeG`q#Ah`{OO{yWx#O5cAD@aV)xuM&LJ@=Cqd3 zMlya@WQ7XG0NwYFE5!US`#t#b!^XN_hNHBaZxQPW3~}4qlrY&N`SVNW%5oY>w&Z}p zQ2-^nSFQX-(V_TN@Xo{G@58&BEpJ`bC5KGbJWqmP`!%-4xJy0zw%d&PalhV~x&HuG z27Jt4)n!tZIJ?SyzZUywwC>i6v_lIos&iC)u)V!r_Dk{XujZ5QJ~jMy({#A}cjJrU zYQsXc(EKH(Gaz|Hk!a`+(-sH{E^Z_V)CMOY9M=)z--ueKxBD{a*ZwElq|bFc*SBFt zB~%CO(6cT8-Ly7(4aA-Wcn^woUkv4pr?gNyb;XX3x#p@_TXBmT9?*%a6i#?GieA?s=O`k{ZneHd1FK* z`yJ*Zq%m=yQRyGMlvTd&(tcF(`Dtt!SnTH@@x$x2pP08VyJiICxIeTka=1`?{1JOm0=mJ2?yNH2<6b)?-@CdRV(-D|hW$$_f@{?)hY6v2F6z9qBQ9 z<(zp2mmt57fb$Xu<@;s|9EF=AwLZi4C{s=4g<63Rk9*ZVeCXBl?dYDqLI>`2F!L~{ zqMXorp5H~&(>?U615A`+#`J_f`)3YcX%pB9Z>GN49Z^|!jj-R&SZ({PvNBFQvJRniIr-8_~*}o2|ChewBWrbf6}$LUv!>WJWjaz*$}aj8U+iJKBLzy zUZ%Flq&{x495$&-0eFhPe(uqG270=!>(}Kt&WVz`_e)FlZn_{ea%c)zJ=4zOc53yh z6@9ZW>Y-yc3q*>vWI7>_JiScb|9h$)uXmKqZ~@Dd?&&_WIZ+XvKdK($5v_h9*a0rE zQuinm)-uuAeX7G!UHttq13!P{86Qx4c6TbkupH2YUYIVWgMr`5^|3e)YzOmTW&MVR{ufZLTGt< zQt?<-dd=Ek&^4!(%m|~6Q6={*H^o3DuvVUNtSLAYjlPG-RfN>jntik>PkZlMuywnwUBrH5agk? ztrNG4Z*F;4=oiHZ757um>~7G^E`C*KW*BgpY+E;~uRa*N;_2_lleW5yxiQ0ol*cvFgfW&Wa*6z#8-J=&*ysuc%S9+i+F7UkxkFfR*s%Eh+NG!h93?`cfoeR{0yc z9%z1x;>R3_!+($1pS34>-ZQg=v|MnVSd-G(7Q;W9qRq!UQfkUhb}gOl@Nd$N|FPKI zzpGsMv1kLd=e7uoLEVLGx=7EWPr=b`iAEM#G%!NOaZ!_C=7G$q_kY^*Y~hEzAh_yts(YWHIRz?e>FKKI zXUxAV^urP%c`!fWcFqUMlcx3K^v~{*Y+2E6o}c38ud|%8o0Valr33u*AvTqt#*#(2 zJv{$Wesx4y#nCLLlgdp~zNLQ{v(n_x-h#^mPJdn7V`S#u10q7EqV^EnXq8|2DrRIL zx(=$_%UZK-JK}<+au@x;q2InK@#ogHln0D3qssL-*X1LCmVj48(qTjj5q3NViZ`lx zdP=yj|KJ^`heQ_3H%i0&@`A3j#<0qcxgasam!L$dt{~Vc@+9A05;G0b zg_s=dO_84$J7OxYko%4(V%_w{I($W9K#d27k9T^d)VZwfHm&t%j#5jR`6G`0V7$}_ z?R49xo$u>2zch-o#KYh!c|9LriGKw{4S7oPqq0$}E64-sOHl8RZd!vP0SJ*2iNE0) zgxwaX;5Z_4IPza43aorsuX4iJIO~hO1;a(6{V^BEZ%Go?_6Y8SmC3Jh1g)pPr`6Td z9^c598*+19=#ZuEs*xZ>slCkcFg1@DQdr;$wL9K|93CSVp`?1L^Ai2D69BEvMgBK9 z3HIJz<`Mi&(9yxuHu5WU$qdWv$Pk5DCnvG9J-x*Q)fv5;-#sGqUrB#D$Jyk~twq3G zICf;oKct6p57|IXkZ>8r@TzxXaJ7Rwsq18+7(PLc>fJ+bc7XbjHuOuFmSNu%N?^ML zt#s>C+Yp~FkguZ7D1`QI8+JL4Gua;GP@b`<7Dwk~v?HP29KwRTRW7;rnj4mk{cx3{ ziHmWk@&hXYWRqDdzh$*?K00+Q#{;$L2fZRP3bOW*Bh?e`dyTzsaCbhp(1`WE2HE|3 z*UqJdJ?~;8Bf-Smz?`Q&`|0-4FW_z&EgsrY>8i8#OO`CaJHVgy__w@dof{RF8tlKlRPAlkIXidqS~ z{oM0wLpBH(P1CHfoix^7g$TR^VfKMFBehnV*60rSlb4Z|xE>K|RB=BnnKP}-coF)K zhHp{HoPEzF{t@mjBk~tz&Ta(^ZJZ>FO&U@nVM!kWI!yB%j-zR(+A|p~8giGmZ{$v9 zV02<6uIP6a)h=I5)q9!vX44p|={W2EK(x?xTRnbi<#R|k`R0UPiE4AEFe1E%75#a! zCjRSY^-LGj&2?Or)mHIIeDR*hYktA-M zR8pC%>AxBO7~dNdh1}}ttAF0Gg)c)T0SFXcDUdgGE>glOxYa)Cp6`XTe5rb2V$xAr z;^4BD@In)cV!Zv$vp_C-LZWsB4&6@8p(encXnCMa&6b*HI5hJ<^T??~Rg>lDF>1Md*^j$s_+&ql`{{1tXaRQrwLIu>(@4b!d8x7naVPQo}&ba;Q@ux(wr8S+f5RqzH;{Z#qa~6 zz7j6xmS?s%U*1MQy38#Z%4d$mJ^I9(NsKc-H6Bfp5`#iYV@XH42&#iY5$)I6*1Y7+ zlwlY5W#n-it$-kZL2rBc6|}SM(LUoQ)a_s*{Nt(tdwpoT%KOHN$AS*1CFAO+ z`DVEWTuLX5hG>LCwv%4|G_w4YOidEJ_Z9*h`;~g8^Z6A#K9TyWpK}d61=L~i7iRk; zzhnFTI%3g(^mv-L{P2;u+`Bitniu)eW;igzLEhU6F0Nt=w{yN`ZSRY+23+Hzh_NTD z3$pbRlmb)9CYanwr926N-OtTyeBi+>5j+rjn2ujwvuzD$H#7Q8R7W)e{jf?$@wSoN zwm@^UPp(n?X>6+$!DeGVo={q%N^bYEIY#$OyZzrz@rcyM$Q0hb9(>U5s7zmMi#Ky8 zjG(oJvBgd1J1p&2;3&tvdOFiyVxlyPZ$8v}4ohJfetxBlfJm&|yE)Hv&X zP5PGEAp2r`LETE-9U08UF$%dlYx1dCFadBo`QMmck7Dyrao=DY-&>8 zxIf|oJOPd$RH3h{Yf`)b_p4yM8@@C`{TJgR>NSwNO;ST>J=TPpZ}!cjVvbd}pYZ+b zd;MxbY*uhXO&ptXD+6f2sW#yhCZf~cp2&{ZTs;_GpOI~U`Xk@OuAW{+;t53!Ge3_eHQ_itUPwBzqn!OKL8d%QoIq7nM3RcSOYtJquf3jJtDRsSrdj(e6}}^)JFt=}Lu=h2^xV7{jxWw-@h5dVl=f-Z{9B^_H9*^>f3-W1 zX2BWMw)M+>FXPQuWd`WiS0PZ|w2@@4d)m`#yB7Xp=Dph^{{C-ci^~m@EWhm~kWs?3 zhjE{gY@$yW7~yK8jXrGJZ{l5I0^R6YAS19G%+o?6NW+)<8f6*De_Ld3)B(Cxl(QWh zR^~=|T&pX#w`Gpw$retD(n^q2hF5{BayRmwq4~qIkXX3GGPS?=K8=lTw(Yi*@44}{ zPF79NcgdrV?)Eg8_1A?mG1Gat=cL=?%sxqi8Hra9R7;!WtY^1p^m5z+wxrrm2x2)u zorI&d`&~~cI6h4L*5Cx7m->hs18QP#%(XbSU zl_1p|0)MA1-vK!*dn;z8YH%$_KBc#wCzb`?5bET(Wd7an*#mFTGQ+9#&ia4maL6C` zC)1W1r6r+tk&^hV_tb66e<*|beE;6K5cO6_-rkR%x zOXH3G2Y5K+#}DZZfsy{`>|WD`)vW2N2-ij%6b)5#V-0@)@mI;!s(Dqp{^mQQ4or{Zzbgm!%BCoJa zNuDjwIM)!{!P0!Mj0c$f^-|i*Vb9sQ^nz2caS=p`MabtTTheLKra}iTJ8XSS;y2&@ z80mY}&n?~a!9HDpMY0z$&P4}qX0Q5{cwpC_-=X+;l0%fTpIbuZ8kOBWhFfGyb$r&V zd{K0heNW*uJPp~rP|lKcE(*-*)!b4*jfzp zKexSkb7m$xwyW*n11LP^`{MtW3g!5)#nx?Kb zI`s;Y%DgD1q2RZCn7IfH(n8FZAz!i%I;(SBN^LAP5`(fwx5ids$@4O+KUV7;20qGq z{gnFXh3F#4*E@xisR#jq)VUMK3Y;w2hgi~vMe(vo@xYH}nBcrZe=i1qT2_psxZNwM zsXPj=_sS)$%s0_Dnl{@n@sv4%2@Y-u4mqxZqTJ46IOl}?d#uI7zdzJeC;vOlfz7r0 zz}S6*HO7yoB4=C>*wA0Rx3+r^@0CSacd4TI79Mpl@4uT_I=RVBbf8C%ZVkFBW-ZA5b0Mlo7>3(jr;yl zLsZF=Wwi;PKn;mHaGCs|vU%<`_gcC~?RU)mEquSFV;i|o+DK`|@>fco=kMX1I>0o3 z;5GoiJL(3e%>(et{ygs-;Agk%K@Ru{yJ(@xcih#ACvN-OTu}jl`Bn*q{^V>;Nb_Ax zyYo}pJ1*~oV!?M+ep^*0S7?czXx@&U&~ykVxlq2xi<=U6ai=>MK?kTMr$~eXTt%FK zY?}$1HAOIt@fxRGXzz?_-fB!wT;*0#n0ARB-6fZ#3UaNU(xMA1^o!qY1uDk>O7Pg9 zv)mnasu4EOE%Pbhx(D#iQsGasZh7R^S$PpmQfyVrVH!0&t3=dIoNZ>dD{Fozqy}yR z$p6g8OCHqL%~I=1mOm8leEYeS$Fbexm9f~n#=WxVnVt0uG5hl32V;wGx1}SAu9JVv zi#k5LjGz`8Yi)g1U9Ns#dBU#oQq0TPj2rv=7(!=hMW{ImN!xBiaNT@bZ<`4whDkb7 za%BCf@Oo@)w=12(_B!UL3dP5W)9@YsH-JQ(;PORhMa= z=gm2*K~o_%!>btOCw~it{`R};r#IrJPq=PP_-hHd?C}}CNw%o4BOuT4WrY3kV7i-VJx z7o9e0!&#J{Zrce7kNwIWwQ^k$!Z)0{WAyS@>TLCmo(wSX?B0|E9~>j1R%;S##xw8Q zk0aO!fY9b5I{3v*TnyZkvY@%9Lec(QJ@k!!g=NQgF5aOm_UryradT=@JPw)!7?w6l z4DnDK@`)(HJ!frUGX*W@=W>qlfPYsq=<1n30IIr<4a1QO>??l=AQ^fo0+ZWoYov~soTa~(0>0tPEcVR0+MId(nru4~YWjQPD z3oqs2UwIV@9i#Lb7O&4HR0QVp4*F-|%}QU<`RdyY=;m)WCur3&`{Z~lJSJ_-T@ZWB z6t((mpZRS)_;71g^qxEZqOs&mtON_Dn{Ho*O`o>;3i8yK3`l(_>|=B*3gUC9NBJa{ z4~%`Bt0q?A)jNI~b#g*TzpBK4>y5PL%}`r-tjdHP*URc)f1ZYMT=BeS{dlL-cnRHK}eEz(5DI1mH@! zW-@meZN=*TqQNF>tdf!S1sHbml12?%O}1dB4~F9XiQQTN=a|5dOVm72@7VNSxMyXf z>2y-(P(hW|Sj+eEkqE-9=kadar>Vp-$^FdA>sf&A`t>^GV$qK;rsKEeT@7{4?z!GB zzh4#8dNyZp^|yXJb+DCWYjkfGsb=@Tqna`(0fOjO6iV8zZJm?KT!bU@UDIkp$dEa> zyjxJbjm7-NSiy}xzrB()s7ye}-wzpc!pVOdQq&F@3N7|o!zzldCo!EC7Tpg#xz6mk z&b)j}p)*MCLd8WY;~K8!bJ%eQvn?bY(g(e`ANTe(l}vBLKHAe+`i zqRJl}U;ZOPz~tnf+;5BV?|H_Mr=Dt5uv7uc+AD5b&gd9d?=6g463OH|XyDkvP>gP@ zf1-@XyFMkR>KRcuUvnvh_$NV%Wgpr-7W1`X9>+WCdbC{bkzO4*B@ zGo;$fj&z@VtWcB0ZLM)XC4Pmf!S@g^VA^Bb4H0lAi;ctY70o)^UQJ%ms{G`h0VuD6 z2bx-Fg@yVb2z?aZU3zjwMsk$!qZ9FTa_6Z&=lrm+OYk!=<;47U1f%L#ZmRh5L2fZ_ zRSn)O~iSulYOk(t$LMLX!gk#cOe%_hgTO?#8zwTzXVOIn}i8DFn+gG!O@ zo}OwwkCnMEN7+ixsCwmZ&(-?O&t(V0?Fhy?DAkOs8Dm_bIw{*H@S_Jysq|1^5tVO@ z@s)`lYP{XEvW!O6CE%^Qo=e7-;Ft`_-4~Q4^|^|^QyKGXW&p)-vzK0RmG*XhV?IRu zd5f7%9?@t7PQ6G{>l0kCczI2lRyH*vao{=2%`?m@&kKFNW;yW|&84PqrsaX6jJrhv zQ2d5}ZK*HQwJN0V8REu9cdJ66&cuFY8MBS^T??yj#v=ao#D1Zf)ZUtC^?K#JU(i%g z>eh`xo3%2oU|SBRR&+DxCm&I%HMT+3?8f@5f({R?m5T@Dd$M2VcC^3#TH(&T+(W(x z^+UGH@~($JqE|f$dNk2I6wt}gxsX~E%KSvLg$Lepd(JIn6dvfijAC>~($S5}*ci6I z95cw)J+Q4bRQXd)UHn8O=l!P#G@65OueOPAO+T6JEAiV7DB|sCAsH3Q zH7%u{39#G9c8j)Nv^@@2aGPpd`Ie;`2#JCXTfv)`2hE=iiRCpz!8`#$(B%Qro?8A%VX&IR-Jo z5?}lJIn_K9(G1~BurM<(bD$Tucvmdc=jkp(bywKUnXx`jyT7;CPUk)K>;Jo|=V@cE zQ$(vQW<;>MAln86?Hsv`=IJ8||NmUyK?sk)Rj}Sb6H!ycy)?HPn%*|;9s6eJIq16v zCLKm0{8eTXF1PgMskT+s4}mevEYfbu3iOv$@(5Nrt9;kPk%dc9W4`^Gq{XN__Y&%+ zpW3iZGdg}Ji3)}P_BodMr1$QpFpmD}539z<+#oXu4vYmRo()l=;FfK7$C+7ew`?_0 z#kmKLm@Dd`TDbesMSq#E{Ok@}&ZG=Wvc$E$W{lIU{53NC-AZpbKm@ zuf0~9(H7CTK820j%l>p|m%i4BSq|@5xZ}q$rc@5yAvm_wMhaZhvD9QxdjbDy?9NUD z>Yha9OEgTU2DS|(9aM4J*-iJ6{yl|{O6v`=-lot zr^KBx5wgez{b=}jg0Ho{u3b}*UhsnqnUhutvk5gI7+x%zKhgHN^(B06IE|LwKK+0a zsdq4G^*Kk=FyHddJo3OKF0I>62wfo9Cwpc^{2r4tcjl?SK0iGt_U^?yd8;qS57=+* z=Hn!DU-}EW+2@H@CB+6EUgX$(NRr;Ucydpuz@CHAMV)20l6CV{4e96VENyyBLn3<$ zI8IO5Y)XZeI7h|1o@vQ+7w)#?V%)C~ z8m!4D{ad(8_Bv@~6*IV0t z`1Diusec3Jx{P&8B5DU^|G%@Ic|Q%W>3@mPrHB|HnW=wtPa$!&pX(t%KQ%~zfO2X3 zBP%l$4-zcbAhhPQspns^pF@uH^+&6%fZVsU!#xjo@ym6}`!~En`JUbxTCrLj`v7N} zy`NCWRNHgX?b?;=sqOmz+sh-X3K^gOOCeKm9ZKckxsm^Vn{k4L4y z&>d2V4-zlL`tEaI>1@n`9Hk?L@PR+Ui}V7fII&_WJ5JqoNV2Rk(+B7kkzHQM%z18M zq<$*eOL3NFol1#%M>w)1U?l3>wsQZ)=p~3JhLJOccHsi$849K7kg zN(Jxq*$Y7$Ou2XeepFQjt@$p$%cXYs#%qA*FjqGR0%oBYQ><$73qI}0gjw;99uX`b z^TMz5M8Dd3oALP)yeppC3O)ODWm3!B0A zTgGH3&+#NYf6qvW_v(DCf2YalKgEfUf21Qyz02+o^Js~%Jep>U%TSH}B+NJbt=|3R z%^o}^->=cPX#1W^3@P)3%_6mHj4E8<5x18NWf=4Z4`;evi9a;F@m1Perg^T0H|O3v zug@Yq!my?WY#(dSG~O^vAxwI>v&f>@x5ha0`fpxaDW6~4my}Lo9E#KGtI^e?ioO!< z6YEKom1f6;2J01wOIdipdR*f}rWnS7+5S8a?FbIUV13^UaqBl46@5hxAGe-?SUbi5 zl^F^@C#`?j1Wj*@sG@ZIssXe2murc){sWov(=buwD?zmh>RPzcFy4PWj*)$xeJn3B zHySKH(9XrB-#hhU3xR%DlqCc&RCIR^6AUx&@?@+{;sf1+9?Xiw^8Klcwo?0R-Lv1& z^T8p0$M`&9*_Oq;9@^o9S5*QU=|ly5tN-<16A>;O<&2qIbvtC$CD?hzyb#RLb&%J-@fqy zkx8~Wh50*iHB5WM6vo|iA;9_a&0soB;QB{ZUoQINIC{eR(v2uUPgc8iV$m()8BOxP zS#VlG9QK8Sb@ZM0UBamZmaL6aB)z<7DPdnbj6gndHKH^Q!oDxjGT68oE+FkUPZMXhAg@gQCD~H^U${C& z@4QfhB}zv}j|c+GcWl z61PxeA7|(XaWeGY+{&&;AX2u^%puOS(HV?{XRucp_p4J3>d^bU9`uw*5fLO}?vZ)Q zYJ9HT0;u}x%7OScCnJubD+vcN#py?jz6J_iO^?cH#{^9t+TRx4>^(Y}KGC5dy2Z!Q z2e;u1il0ihYdF~}a2Y?AcEP5R6U{T_v%``Tx%5jp8V1Sonos@2rge)xZzfu0GQeNE zV7a&~H%44f9=G7M^If*aS{gc=H^i7le!wR~Pm0vz<`P5pyJOn4-i0xyV2sw2w}z7^ ztAD8PH867Cxvdtq9kfeAFv@;G@c3OJI^+@jy-Q57qSX4*vhd{hN|X)5*FElQOE4aC zO!9rr4UDDb@fZyHw7k#mr?$TIAsonar%!A*EK9Q-^4mSV8|F2>GKs&|K{kEd50<*& zeC#2-2&N^7A=wD)N~j;|)7cy6KP>%_7h-;$gxSmv;uu+ahY90k;2$!fiv3Q)T*TxtzyhGCP9+ z7TIxgb%kZ#2r9pID2YZkH07w8m<#}5Sl7c*PI3zs5!~Zq;8JkMD6B4FVepJQOl{5V zd)Asd=(XOv|8N@%SK&UtHdwpE_tRif7v^E&w+BGl+{$}9jA}f&%IX6zR^mf-uB!T( zb#O7cq`3Gml_>gE*~n5A|9U}RKFr&+L0TPoi|lt4i3+>uk4piBMW*pK@SX+Qy@}mp z*iF~{;tivY;D^6AMxUx}F{<(~sV3=!fLQJgyrfk(5{l&(Eii2!oYyXl!l;h~){Hq` zu#B#zClrk3ACuhO=^^T5KDZ8mR+q}eLRnqkZCZ-=_ zR>=diPS}^syF^Va?LC5%N!5ka_rN?-Bnzw8-tQ0g&e>Tz>0*D83cP2e>SX z4Os!!f3d1*QlXEoYrAl|k4nv|Hfg+i-!CJYAOS_dm2f9XH1*Ig#TkO4H5>K0`Rfke zjdd3^;cxwDt{D~byDx|{m+^PNXG-3-PcIt!8hGuc+eTnieVyjKw_>BKMm7Qkjz?Bx*h?+m!J%NrM&+rOwJfHahyE;z8LxJY>Rp}L6S3z&duM4^cB>zo z5#7vHS63N%XsfGtzNUF++ppB|LaT-QZ3OdKr}9^28t*cC(syi}W}H$LIxp~yb!SS~ zN5r+ExP+&LopR08p8ZpUvrMNC>w1)TkaXKQG6wGNFE1@+89GbLF10J-rOS96CHDQ+ z$Iz8mtDax4Fr0}w85(tK(}sr_pYs;^*3T?tL{wFJD`JGZ__ZE;Rhl+n92Oi_N9V&Y zCo3)_oa@UiAiEIUf>!N8cL?$7xk+YN=lGY0$!!>))uF6a^-P5|b9$NZ;V|e|Dh6%1 z0q$aX4~28;lu{Hrpy)h7((sfm#a!Q#SW#Q{)GT%U#uz`HImDAi+G7I=l#LI^F5n}g zz!z7hFF~92Tc}&|;+LR@{owOcuJfztvmdRMOR0`Bv!;)PjvRWgRcpsf=ljdeoi$kQ z5VvD-4DY;u8@Bi62^>PC{jF?vO0q3Q<%Intk29l7iEd?Z1HuZdYR*X4=Ib2goX>Cn zYOvglxXhtzvebOHL8pm)P0u;qo_N$4f$D{Tku1@qn|=xBc8+|>Q8 zjWxK+cDI?~;G$H}Wn*Dsr3Bk9Rd`TQTIr$=V011)ylbBEa3wU#ejzyE#*WF-+!GCX z?Gp2OKx0F4jEq~Vcvkm!q$Z5gDe3cVzahQ-y5Ms9X4vxn$7WH|e?-`UN~Yq_|FsyAYT{{;A-VQjx&xFCB;aDYVpmlaInt1K&oIKs|k0KCy9 z=!7qh^su2UAuqN6W(v}EM0;jDdaGt$?;er9(s1*B{8(()L`g*09sRlWPz6Xz#sU&* zvgqCHFuoY&x4xV=ZOMh&I^Gta~oi^|33;$-%Ez`v6?nhpWDw)S+jVdct9( zP}Is{BR624Dg-c4bk9rtd}C27&2ZC8P+0-$Xr05TMY;J&td*Ii7UxK5LPEOX+FsJW zJHu`ey;7p1$-8GZItVtc#7tgb*dt$PyYF~{!WRj1@-GeFCDe2V#6=ye+{yaQ7*m`R zi`NwM$aCB%cer zfBHjF?mwV6ulQUda-JXxELJEBa!UE)5~RgheF=I6KI5oDU0mCOo_p1l{lg`tze_6Q zEzSI_AKKD8S-@{p2n?vKq=>4&7X0M2#vRgpE2`7BmS3iw83oNI46X21)q317reaiI zyxl=VgYE!km+jlbECBPxPqlzn34Q3-QJu%Kl}> zE#P^;Bshc|e7t1K3S63lM3aGD_OG7p@Q@1GNJUI3DwvF-Ha{%4wgx6_sIy00ba07- zg-;$!op_lqx#%|`W;M3VwSN(WhC;!CoE zEa8Vw{Bw;N`I5qQkccP!D4@XQ;I8zMYEyi<3`{S5Je7@OG+=q6(aDVB!AkAJIfD-~OB()XO1`pq%DycQc8 z>|uWDEq_PX??!~v1RMrTU1d*D_j{cRXTuC=Q(py7v!n?`J^2>r^}147}pyr~scWwGj_#a!lhcDD=n>4uE>-0UBN%zdF#*2x92NM)f=W303 z`TYKxF~#}8r)U~;f26bRMcnyQ?gxzVCQWX@ATC~_3wd${D8`R6ZuQp#zlX6mNbiuO z2a%0^6LXra!@01-Q?vj7o2A%iSSLe+=xTtY@Dbj<44jpa09FY0`d6$<1 zmt0t7#cdH>SNG|=ik%mFxiS06W$pyYMN#!GaVr8;na-8-p-aL{>xxRHO{2xE_H9(X z&7MT2*i&C_>Ole2%Ik1)Yg#@zLhE#R%8jfI4FgX7uYruNQy=4JP8=$17d(!v7mxm_ z(UuNu%HR(O{^p8`iR0Z{^F}`J-g@_2@O-Jb!edzZSLW4?Yrmg$eYpEhNIw*E8+;sh z)oR;Tu7)wi49&lTv#P7eG-=eI$6y*G4u3nnEP3YHgiuMCmM;g?DK4>V5u1g^2oSk=+auQ2v4srUV7Z(7z3 zIiv2q1idD>%@CoNphuZskUMuFN2zEj$o(0uUJEi5oz<;ZnM8#J7rBkqkLAUs`nbzZ zuHiAA%@5;uytTdO@@KA;yOciQxna}p38pgU-uwN4OXYZYhwT8>(JXqw)jj+rjm9+1 zH|$vWi-_noTi>ree4qB+XP4ogYJKL>XPFXWk164QaUtXvsOy&?)cZ@&f4qoZau*8k z^bhdq<1jnZga)+`oclsuj>Fqm`)i1t7jI`E4hJXEZiYWyNIz5bKU{)-R-I)U!L=s6 z_bL4phjr?|k-oon>)%}K!S56sYwUHCYNiyZ&QuZ42oy_YF=R=&Ho95JuT~G|TJyLm z^jkr#`3#SP;S&^iU#p^G{p)D$=0}Jz(kro`!u=9>g}Saw(S z2_Dm3OUYxQoA@c(0eWLB@}xmOyuN8Rcyh*1dhnyT+mmI^aQ^#g{IftjN1mhq*|Gx( zKB;CLead+I*O_ScvAk^MPAQ6)te!RnXT`nsyk3(5n%a1iB6EIP5{L|)mrvlnkI=S{ zmFh`H2t$nknsx~^tE5Setf)WQw6VC+yUo=JG&fsTWlFdfQ9XoR1g&56f=vBbN!nA; z-avn{g1;JjAOnLJEF|FD?+D4H9I#D8{xz;rNK#M;za}Cn2|N$Y%Ci#rFYQ$Q244-4 z>5PkQjyzoBi`@1uanpv&g!x387{L)$I0Ag;|ut)`kk@9t}HipOwNhjnx57y>$z@t&E4ly>nhMYjmK z00%iUbON%whIDA5Nv=dl_RD&otg0*LqM`l7xbLqBPC@n(fPq2-=e!#iDLH-NPNV+A z*PQWmMicOVARuBu^MW2aksA?`O|%w@l?98NgMmFmu*4VepS10;d-_b%sB%#g6$czy7i4*;rHd!ff?e*4C1E#pm&)=9Np33NFf=pe^g#)!*@iF~-E> zWwA%L_`|bSlZ!$H@wivQ1C2o!&s6iA6XvQ8B^$pcr}WnxawOqvq3PbXSB6KMmvIZk z@O|VCvv2|-1PDONz>beb4xy6pdmQa=JeiB-Cz>)F7)!hpV1fSo&2!tL#FDJfO%%__0Wm=rB#QXZ;EI z36yru>*DjJu3*W|oiSa9RgDLg>4lwDjz1rXi*r|GMR{CD=meN@po!n07q5RKH@1!j zVPs_-NuP`^I}8_h+ZN2j@shwjMBk{#-W$uj%%YIR@AFJ)n+v)JQ5J=ku7lpJYme{x z`%;<6&la5J#D93Kpf`I!slUwe2))$ZzO4Xh|96%)kfuxY)Y=#f#uvLcnU|hC|F7lg^IoAyanxEwO%De4?QfjSB>)>$#U7UI&-Nr zvx7EL)YISn;A9W)BgT?;j?4Z2pvGW0_4-l+$JO_?+!uVvo^ar)zxt_i2~uyjt-(o< zbs_&hHY7tyTnYjHxn}5<1VXyqM*7Y!M(so~PNYQ0qeQZ@G&9W<7(XOKgwbl@M(p90Rrx>s42K1AE)14q2O2rg?v99DE-M!qNWAa+U{ zw)!IKA(L5_htm%?!K>!n1E2IHYCT#c4}gmkwilu2%WjvTa`YxD9JNy7cR_Arf}T4B zqjZOOd!yTb3yW#Gj4ek4xrer4!=1XWplOC=wwa%WfwRSRAnp=09g0hf);5#PWas}I zwIr|dy$H0DyW0~PRcHM7b&uJQMC&i2T*YKFKk-RoE8{)uS2kZ*}wgbVP-bLG)Wiq ztncho5Eamsy6VpfdDtu0GyQXa73KH*3G>^vOIlVQ1+ucaIJ3ZV!v75vp`SDH-t(-N zpwbVS7oN~x&3wSK@-IcAwaZAiWbw795(QdS%!VU+=`dg>WJ{}C6;(Y)XCff<#1JU> zR$+%nQ@wGmmH@6yF6Nz81;$mjvaRQV;`^nm*S@S^m5mak-PtG0qTgZ<%G!T)5X!kaF&eI<5iS6Ml`+>H zi+>Y5I{%P|A70xR@rK<=sGa|TXZ$(K!M@5!l_;C0uG!0A?Iws`rkZL*EZUpSHdeXR z;)PPP$r7?&PZ%G*2qp$#fB*)>`4K*pR`WgJDUBg_q>;cW$N-w-mzD6c%?N3F`nEnm zVzhU&%X`j3)FTIvCdQDsy6PLrlH>AEMPZ`Fl?8T~xC>h~LqbpC6*y$T?fQc>iJ6ZG^LN@Ah> tfTV`3{O5-vw=ts5Da_Uhh2jOl9@@_2%1h9|<6~><-O=qMV%+8Q{{Vzxc8dT2 literal 0 HcmV?d00001 diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/GenerativeQAResponseProcessor.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/GenerativeQAResponseProcessor.java index 7b1814c2a5..6e8a5544b3 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/GenerativeQAResponseProcessor.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/GenerativeQAResponseProcessor.java @@ -179,7 +179,8 @@ public void processResponseAsync( chatHistory, searchResults, timeout, - params.getLlmResponseField() + params.getLlmResponseField(), + params.getLlmMessages() ), null, llmQuestion, @@ -202,7 +203,8 @@ public void processResponseAsync( chatHistory, searchResults, timeout, - params.getLlmResponseField() + params.getLlmResponseField(), + params.getLlmMessages() ), conversationId, llmQuestion, diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParameters.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParameters.java index 01dc97db75..ba4f1c9b03 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParameters.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParameters.java @@ -18,6 +18,8 @@ package org.opensearch.searchpipelines.questionanswering.generative.ext; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import org.opensearch.core.ParseField; @@ -30,6 +32,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.searchpipelines.questionanswering.generative.GenerativeQAProcessorConstants; +import org.opensearch.searchpipelines.questionanswering.generative.llm.MessageBlock; import com.google.common.base.Preconditions; @@ -81,6 +84,8 @@ public class GenerativeQAParameters implements Writeable, ToXContentObject { // that contains the chat completion text, i.e. "answer". private static final ParseField LLM_RESPONSE_FIELD = new ParseField("llm_response_field"); + private static final ParseField LLM_MESSAGES_FIELD = new ParseField("llm_messages"); + public static final int SIZE_NULL_VALUE = -1; static { @@ -94,6 +99,7 @@ public class GenerativeQAParameters implements Writeable, ToXContentObject { PARSER.declareIntOrNull(GenerativeQAParameters::setInteractionSize, SIZE_NULL_VALUE, INTERACTION_SIZE); PARSER.declareIntOrNull(GenerativeQAParameters::setTimeout, SIZE_NULL_VALUE, TIMEOUT); PARSER.declareStringOrNull(GenerativeQAParameters::setLlmResponseField, LLM_RESPONSE_FIELD); + PARSER.declareObjectArray(GenerativeQAParameters::setMessageBlock, (p, c) -> MessageBlock.fromXContent(p), LLM_MESSAGES_FIELD); } @Setter @@ -132,6 +138,10 @@ public class GenerativeQAParameters implements Writeable, ToXContentObject { @Getter private String llmResponseField; + @Setter + @Getter + private List llmMessages = new ArrayList<>(); + public GenerativeQAParameters( String conversationId, String llmModel, @@ -142,6 +152,32 @@ public GenerativeQAParameters( Integer interactionSize, Integer timeout, String llmResponseField + ) { + this( + conversationId, + llmModel, + llmQuestion, + systemPrompt, + userInstructions, + contextSize, + interactionSize, + timeout, + llmResponseField, + null + ); + } + + public GenerativeQAParameters( + String conversationId, + String llmModel, + String llmQuestion, + String systemPrompt, + String userInstructions, + Integer contextSize, + Integer interactionSize, + Integer timeout, + String llmResponseField, + List llmMessages ) { this.conversationId = conversationId; this.llmModel = llmModel; @@ -156,6 +192,9 @@ public GenerativeQAParameters( this.interactionSize = (interactionSize == null) ? SIZE_NULL_VALUE : interactionSize; this.timeout = (timeout == null) ? SIZE_NULL_VALUE : timeout; this.llmResponseField = llmResponseField; + if (llmMessages != null) { + this.llmMessages.addAll(llmMessages); + } } public GenerativeQAParameters(StreamInput input) throws IOException { @@ -168,6 +207,7 @@ public GenerativeQAParameters(StreamInput input) throws IOException { this.interactionSize = input.readInt(); this.timeout = input.readInt(); this.llmResponseField = input.readOptionalString(); + this.llmMessages.addAll(input.readList(MessageBlock::new)); } @Override @@ -181,7 +221,8 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params .field(CONTEXT_SIZE.getPreferredName(), this.contextSize) .field(INTERACTION_SIZE.getPreferredName(), this.interactionSize) .field(TIMEOUT.getPreferredName(), this.timeout) - .field(LLM_RESPONSE_FIELD.getPreferredName(), this.llmResponseField); + .field(LLM_RESPONSE_FIELD.getPreferredName(), this.llmResponseField) + .field(LLM_MESSAGES_FIELD.getPreferredName(), this.llmMessages); } @Override @@ -197,6 +238,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(interactionSize); out.writeInt(timeout); out.writeOptionalString(llmResponseField); + out.writeList(llmMessages); } public static GenerativeQAParameters parse(XContentParser parser) throws IOException { @@ -223,4 +265,8 @@ public boolean equals(Object o) { && (this.timeout == other.getTimeout()) && Objects.equals(this.llmResponseField, other.getLlmResponseField()); } + + public void setMessageBlock(List blockList) { + this.llmMessages = blockList; + } } diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInput.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInput.java index 66c635b211..3202d56455 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInput.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInput.java @@ -44,4 +44,5 @@ public class ChatCompletionInput { private String userInstructions; private Llm.ModelProvider modelProvider; private String llmResponseField; + private List llmMessages; } diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImpl.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImpl.java index f6cdfec816..6793253480 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImpl.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImpl.java @@ -75,7 +75,6 @@ protected void setMlClient(MachineLearningInternalClient mlClient) { * @return */ @Override - public void doChatCompletion(ChatCompletionInput chatCompletionInput, ActionListener listener) { MLInputDataset dataset = RemoteInferenceInputDataSet.builder().parameters(getInputParameters(chatCompletionInput)).build(); MLInput mlInput = MLInput.builder().algorithm(FunctionName.REMOTE).inputDataset(dataset).build(); @@ -113,14 +112,15 @@ protected Map getInputParameters(ChatCompletionInput chatComplet inputParameters.put(CONNECTOR_INPUT_PARAMETER_MODEL, chatCompletionInput.getModel()); String messages = PromptUtil .getChatCompletionPrompt( + chatCompletionInput.getModelProvider(), chatCompletionInput.getSystemPrompt(), chatCompletionInput.getUserInstructions(), chatCompletionInput.getQuestion(), chatCompletionInput.getChatHistory(), - chatCompletionInput.getContexts() + chatCompletionInput.getContexts(), + chatCompletionInput.getLlmMessages() ); inputParameters.put(CONNECTOR_INPUT_PARAMETER_MESSAGES, messages); - // log.info("Messages to LLM: {}", messages); } else if (chatCompletionInput.getModelProvider() == ModelProvider.BEDROCK || chatCompletionInput.getModelProvider() == ModelProvider.COHERE || chatCompletionInput.getLlmResponseField() != null) { @@ -136,6 +136,19 @@ protected Map getInputParameters(ChatCompletionInput chatComplet chatCompletionInput.getContexts() ) ); + } else if (chatCompletionInput.getModelProvider() == ModelProvider.BEDROCK_CONVERSE) { + // Bedrock Converse API does not include the system prompt as part of the Messages block. + String messages = PromptUtil + .getChatCompletionPrompt( + chatCompletionInput.getModelProvider(), + null, + chatCompletionInput.getUserInstructions(), + chatCompletionInput.getQuestion(), + chatCompletionInput.getChatHistory(), + chatCompletionInput.getContexts(), + chatCompletionInput.getLlmMessages() + ); + inputParameters.put(CONNECTOR_INPUT_PARAMETER_MESSAGES, messages); } else { throw new IllegalArgumentException( "Unknown/unsupported model provider: " @@ -144,7 +157,6 @@ protected Map getInputParameters(ChatCompletionInput chatComplet ); } - // log.info("LLM input parameters: {}", inputParameters.toString()); return inputParameters; } @@ -184,6 +196,20 @@ protected ChatCompletionOutput buildChatCompletionOutput(ModelProvider provider, } else if (provider == ModelProvider.COHERE) { answerField = "text"; fillAnswersOrErrors(dataAsMap, answers, errors, answerField, errorField, defaultErrorMessageField); + } else if (provider == ModelProvider.BEDROCK_CONVERSE) { + Map output = (Map) dataAsMap.get("output"); + Map message = (Map) output.get("message"); + if (message != null) { + List content = (List) message.get("content"); + String answer = (String) ((Map) content.get(0)).get("text"); + answers.add(answer); + } else { + Map error = (Map) output.get("error"); + if (error == null) { + throw new RuntimeException("Unexpected output: " + output); + } + errors.add((String) error.get("message")); + } } else { throw new IllegalArgumentException( "Unknown/unsupported model provider: " + provider + ". You must provide a valid model provider or llm_response_field." diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/Llm.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/Llm.java index 1099b1e21f..9318b681d2 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/Llm.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/Llm.java @@ -28,7 +28,8 @@ public interface Llm { enum ModelProvider { OPENAI, BEDROCK, - COHERE + COHERE, + BEDROCK_CONVERSE } void doChatCompletion(ChatCompletionInput input, ActionListener listener); diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/LlmIOUtil.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/LlmIOUtil.java index ef9e9948db..24e38ac368 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/LlmIOUtil.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/LlmIOUtil.java @@ -29,6 +29,7 @@ public class LlmIOUtil { public static final String BEDROCK_PROVIDER_PREFIX = "bedrock/"; public static final String COHERE_PROVIDER_PREFIX = "cohere/"; + public static final String BEDROCK_CONVERSE__PROVIDER_PREFIX = "bedrock-converse/"; public static ChatCompletionInput createChatCompletionInput( String llmModel, @@ -49,7 +50,8 @@ public static ChatCompletionInput createChatCompletionInput( chatHistory, contexts, timeoutInSeconds, - llmResponseField + llmResponseField, + null ); } @@ -61,7 +63,8 @@ public static ChatCompletionInput createChatCompletionInput( List chatHistory, List contexts, int timeoutInSeconds, - String llmResponseField + String llmResponseField, + List llmMessages ) { Llm.ModelProvider provider = null; if (llmResponseField == null) { @@ -71,6 +74,8 @@ public static ChatCompletionInput createChatCompletionInput( provider = Llm.ModelProvider.BEDROCK; } else if (llmModel.startsWith(COHERE_PROVIDER_PREFIX)) { provider = Llm.ModelProvider.COHERE; + } else if (llmModel.startsWith(BEDROCK_CONVERSE__PROVIDER_PREFIX)) { + provider = Llm.ModelProvider.BEDROCK_CONVERSE; } } } @@ -83,7 +88,8 @@ public static ChatCompletionInput createChatCompletionInput( systemPrompt, userInstructions, provider, - llmResponseField + llmResponseField, + llmMessages ); } } diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlock.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlock.java new file mode 100644 index 0000000000..1dbfd4d13b --- /dev/null +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlock.java @@ -0,0 +1,325 @@ +/* + * Copyright 2023 Aryn + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.searchpipelines.questionanswering.generative.llm; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import com.google.common.base.Preconditions; + +import lombok.Getter; +import lombok.Setter; + +public class MessageBlock implements Writeable, ToXContent { + + private static final String TEXT_BLOCK = "text"; + private static final String IMAGE_BLOCK = "image"; + private static final String DOCUMENT_BLOCK = "document"; + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.role); + out.writeList(this.blockList); + } + + public MessageBlock(StreamInput in) throws IOException { + this.role = in.readString(); + Writeable.Reader reader = input -> { + String type = input.readString(); + if (type.equals("text")) { + return new TextBlock(input); + } else if (type.equals("image")) { + return new ImageBlock(input); + } else if (type.equals("document")) { + return new DocumentBlock(input); + } else { + throw new RuntimeException("Unexpected type: " + type); + } + }; + this.blockList = in.readList(reader); + } + + public static MessageBlock fromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + return new MessageBlock(parser.map()); + } + throw new XContentParseException(parser.getTokenLocation(), "Expected [START_OBJECT], got " + parser.currentToken()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("role", this.role); + builder.startArray("content"); + for (AbstractBlock block : this.blockList) { + block.toXContent(builder, params); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + public interface Block { + String getType(); + } + + public static abstract class AbstractBlock implements Block, Writeable, ToXContent { + + @Override + abstract public String getType(); + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("Not implemented."); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + throw new UnsupportedOperationException("Not implemented."); + } + } + + public static class TextBlock extends AbstractBlock { + + @Getter + String type = "text"; + + @Getter + @Setter + String text; + + public TextBlock(String text) { + Preconditions.checkNotNull(text, "text cannot be null."); + this.text = text; + } + + public TextBlock(StreamInput in) throws IOException { + this.text = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.type); + out.writeString(this.text); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + builder.startObject(); + builder.field("type", "text"); + builder.field("text", this.text); + builder.endObject(); + return builder; + } + } + + public static class ImageBlock extends AbstractBlock { + + @Getter + String type = "image"; + + @Getter + @Setter + String format; + + @Getter + @Setter + String data; + + @Getter + @Setter + String url; + + public ImageBlock(Map imageBlock) { + this.format = (String) imageBlock.get("format"); + Object tmp = imageBlock.get("data"); + if (tmp != null) { + this.data = (String) tmp; + } else { + tmp = imageBlock.get("url"); + if (tmp == null) { + throw new IllegalArgumentException("data or url not found in imageBlock."); + } + this.url = (String) tmp; + } + + } + + public ImageBlock(String format, String data, String url) { + Preconditions.checkNotNull(format, "format cannot be null."); + if (data == null && url == null) { + throw new IllegalArgumentException("data and url cannot both be null."); + } + this.format = format; + this.data = data; + this.url = url; + } + + public ImageBlock(StreamInput in) throws IOException { + format = in.readString(); + data = in.readOptionalString(); + url = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.type); + out.writeString(this.format); + out.writeOptionalString(this.data); + out.writeOptionalString(this.url); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + Map imageMap = new HashMap<>(); + imageMap.put("format", this.format); + if (this.data != null) { + imageMap.put("data", this.data); + } else if (this.url != null) { + imageMap.put("url", this.url); + } + builder.field("image", imageMap); + builder.endObject(); + return builder; + } + } + + public static class DocumentBlock extends AbstractBlock { + + @Getter + String type = "document"; + + @Getter + @Setter + String format; + + @Getter + @Setter + String name; + + @Getter + @Setter + String data; + + public DocumentBlock(Map documentBlock) { + Preconditions.checkState(documentBlock.containsKey("format"), "format not found in the document block."); + Preconditions.checkState(documentBlock.containsKey("name"), "name not found in the document block."); + Preconditions.checkState(documentBlock.containsKey("data"), "data not found in the document block"); + + this.format = (String) documentBlock.get("format"); + this.name = (String) documentBlock.get("name"); + this.data = (String) documentBlock.get("data"); + } + + public DocumentBlock(String format, String name, String data) { + Preconditions.checkNotNull(format, "format cannot be null."); + Preconditions.checkNotNull(name, "name cannot be null."); + Preconditions.checkNotNull(data, "data cannot be null."); + + this.format = format; + this.name = name; + this.data = data; + } + + public DocumentBlock(StreamInput in) throws IOException { + format = in.readString(); + name = in.readString(); + data = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.type); + out.writeString(this.format); + out.writeString(this.name); + out.writeString(this.data); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject("document"); + builder.field("format", this.format); + builder.field("name", this.name); + builder.field("data", this.data); + builder.endObject(); + builder.endObject(); + return builder; + } + } + + @Getter + @Setter + private String role; + + @Getter + @Setter + private List blockList = new ArrayList<>(); + + public MessageBlock() {} + + public MessageBlock(Map map) { + setMessageBlock(map); + } + + public void setMessageBlock(Map message) { + Preconditions.checkNotNull(message, "message cannot be null."); + Preconditions.checkState(message.containsKey("role"), "message must have role."); + Preconditions.checkState(message.containsKey("content"), "message must have content."); + + this.role = (String) message.get("role"); + List> contents = (List) message.get("content"); + + for (Map content : contents) { + if (content.containsKey(TEXT_BLOCK)) { + this.blockList.add(new TextBlock((String) content.get(TEXT_BLOCK))); + } else if (content.containsKey(IMAGE_BLOCK)) { + Map imageBlock = (Map) content.get(IMAGE_BLOCK); + this.blockList.add(new ImageBlock(imageBlock)); + } else if (content.containsKey(DOCUMENT_BLOCK)) { + Map documentBlock = (Map) content.get(DOCUMENT_BLOCK); + this.blockList.add(new DocumentBlock(documentBlock)); + } + } + } + + @Override + public boolean equals(Object o) { + // TODO + return true; + } + + @Override + public int hashCode() { + return Objects.hashCode(this.role) + Objects.hashCode(this.blockList); + } +} diff --git a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtil.java b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtil.java index 3a8a21614e..9b875c6f7a 100644 --- a/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtil.java +++ b/search-processors/src/main/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtil.java @@ -19,15 +19,20 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Locale; import org.apache.commons.text.StringEscapeUtils; import org.opensearch.core.common.Strings; import org.opensearch.ml.common.conversation.Interaction; +import org.opensearch.searchpipelines.questionanswering.generative.llm.Llm; +import org.opensearch.searchpipelines.questionanswering.generative.llm.MessageBlock; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; @@ -61,20 +66,27 @@ public static String getQuestionRephrasingPrompt(String originalQuestion, List chatHistory, List contexts) { - return getChatCompletionPrompt(DEFAULT_SYSTEM_PROMPT, null, question, chatHistory, contexts); + public static String getChatCompletionPrompt( + Llm.ModelProvider provider, + String question, + List chatHistory, + List contexts + ) { + return getChatCompletionPrompt(provider, DEFAULT_SYSTEM_PROMPT, null, question, chatHistory, contexts, null); } // TODO Currently, this is OpenAI specific. Change this to indicate as such or address it as part of // future prompt template management work. public static String getChatCompletionPrompt( + Llm.ModelProvider provider, String systemPrompt, String userInstructions, String question, List chatHistory, - List contexts + List contexts, + List llmMessages ) { - return buildMessageParameter(systemPrompt, userInstructions, question, chatHistory, contexts); + return buildMessageParameter(provider, systemPrompt, userInstructions, question, chatHistory, contexts, llmMessages); } enum ChatRole { @@ -134,37 +146,132 @@ public static String buildSingleStringPrompt( return bldr.toString(); } + /** + * Message APIs such as OpenAI's Chat Completion API and Anthropic's Messages API + * use an array of messages as input to the LLM and they are better suited for + * multi-modal interactions using text and images. + * + * @param provider + * @param systemPrompt + * @param userInstructions + * @param question + * @param chatHistory + * @param contexts + * @return + */ @VisibleForTesting static String buildMessageParameter( + Llm.ModelProvider provider, String systemPrompt, String userInstructions, String question, List chatHistory, List contexts ) { + return buildMessageParameter(provider, systemPrompt, userInstructions, question, chatHistory, contexts, null); + } + + static String buildMessageParameter( + Llm.ModelProvider provider, + String systemPrompt, + String userInstructions, + String question, + List chatHistory, + List contexts, + List llmMessages + ) { // TODO better prompt template management is needed here. if (Strings.isNullOrEmpty(systemPrompt) && Strings.isNullOrEmpty(userInstructions)) { - systemPrompt = DEFAULT_SYSTEM_PROMPT; + // Some model providers such as Anthropic do not allow the system prompt as part of the message body. + userInstructions = DEFAULT_SYSTEM_PROMPT; } - JsonArray messageArray = new JsonArray(); + MessageArrayBuilder messageArrayBuilder = new MessageArrayBuilder(provider); + + // Build the system prompt (only one per conversation/session) + if (!Strings.isNullOrEmpty(systemPrompt)) { + messageArrayBuilder.startMessage(ChatRole.SYSTEM); + messageArrayBuilder.addTextContent(systemPrompt); + messageArrayBuilder.endMessage(); + } + + // Anthropic does not allow two consecutive messages of the same role + // so we combine all user messages and an array of contents. + messageArrayBuilder.startMessage(ChatRole.USER); + boolean lastRoleIsAssistant = false; + if (!Strings.isNullOrEmpty(userInstructions)) { + messageArrayBuilder.addTextContent(userInstructions); + } - messageArray.addAll(getPromptTemplateAsJsonArray(systemPrompt, userInstructions)); for (int i = 0; i < contexts.size(); i++) { - messageArray.add(new Message(ChatRole.USER, "SEARCH RESULT " + (i + 1) + ": " + contexts.get(i)).toJson()); + messageArrayBuilder.addTextContent("SEARCH RESULT " + (i + 1) + ": " + contexts.get(i)); } + if (!chatHistory.isEmpty()) { // The oldest interaction first - List messages = Messages.fromInteractions(chatHistory).getMessages(); - Collections.reverse(messages); - messages.forEach(m -> messageArray.add(m.toJson())); + int idx = chatHistory.size() - 1; + Interaction firstInteraction = chatHistory.get(idx); + messageArrayBuilder.addTextContent(firstInteraction.getInput()); + messageArrayBuilder.endMessage(); + messageArrayBuilder.startMessage(ChatRole.ASSISTANT, firstInteraction.getResponse()); + messageArrayBuilder.endMessage(); + + if (chatHistory.size() > 1) { + for (int i = --idx; i >= 0; i--) { + Interaction interaction = chatHistory.get(i); + messageArrayBuilder.startMessage(ChatRole.USER, interaction.getInput()); + messageArrayBuilder.endMessage(); + messageArrayBuilder.startMessage(ChatRole.ASSISTANT, interaction.getResponse()); + messageArrayBuilder.endMessage(); + } + } + + lastRoleIsAssistant = true; + } + + if (llmMessages != null && !llmMessages.isEmpty()) { + // TODO MessageBlock can have assistant roles for few-shot prompting. + if (lastRoleIsAssistant) { + messageArrayBuilder.startMessage(ChatRole.USER); + } + for (MessageBlock message : llmMessages) { + List blockList = message.getBlockList(); + for (MessageBlock.Block block : blockList) { + switch (block.getType()) { + case "text": + messageArrayBuilder.addTextContent(((MessageBlock.TextBlock) block).getText()); + break; + case "image": + MessageBlock.ImageBlock ib = (MessageBlock.ImageBlock) block; + if (ib.getData() != null) { + messageArrayBuilder.addImageData(ib.getFormat(), ib.getData()); + } else if (ib.getUrl() != null) { + messageArrayBuilder.addImageUrl(ib.getFormat(), ib.getUrl()); + } + break; + case "document": + MessageBlock.DocumentBlock db = (MessageBlock.DocumentBlock) block; + messageArrayBuilder.addDocumentContent(db.getFormat(), db.getName(), db.getData()); + break; + default: + break; + } + } + } + } else { + if (lastRoleIsAssistant) { + messageArrayBuilder.startMessage(ChatRole.USER, "QUESTION: " + question + "\n"); + } else { + messageArrayBuilder.addTextContent("QUESTION: " + question + "\n"); + } + messageArrayBuilder.addTextContent("ANSWER:"); } - messageArray.add(new Message(ChatRole.USER, "QUESTION: " + question).toJson()); - messageArray.add(new Message(ChatRole.USER, "ANSWER:").toJson()); - return messageArray.toString(); + messageArrayBuilder.endMessage(); + + return messageArrayBuilder.toJsonArray().toString(); } public static String getPromptTemplate(String systemPrompt, String userInstructions) { @@ -183,6 +290,24 @@ static JsonArray getPromptTemplateAsJsonArray(String systemPrompt, String userIn return messageArray; } + /* + static JsonArray getPromptTemplateAsJsonArray(Llm.ModelProvider provider, String systemPrompt, String userInstructions) { + + MessageArrayBuilder bldr = new MessageArrayBuilder(provider); + + if (!Strings.isNullOrEmpty(systemPrompt)) { + bldr.startMessage(ChatRole.SYSTEM); + bldr.addTextContent(systemPrompt); + bldr.endMessage(); + } + if (!Strings.isNullOrEmpty(userInstructions)) { + bldr.startMessage(ChatRole.USER); + bldr.addTextContent(userInstructions); + bldr.endMessage(); + } + return bldr.toJsonArray(); + }*/ + @Getter static class Messages { @@ -209,6 +334,207 @@ public static Messages fromInteractions(final List interactions) { } } + interface Content { + + // All content blocks accept text + void addText(String text); + + JsonElement toJson(); + } + + interface ImageContent extends Content { + + void addImageData(String format, String data); + + void addImageUrl(String format, String url); + } + + interface DocumentContent extends Content { + void addDocument(String format, String name, String data); + } + + interface MultimodalContent extends ImageContent, DocumentContent { + + } + + private final static String CONTENT_FIELD_TEXT = "text"; + private final static String CONTENT_FIELD_TYPE = "type"; + + static class OpenAIContent implements ImageContent { + + private JsonArray json; + + public OpenAIContent() { + this.json = new JsonArray(); + } + + @Override + public void addText(String text) { + JsonObject content = new JsonObject(); + content.add(CONTENT_FIELD_TYPE, new JsonPrimitive(CONTENT_FIELD_TEXT)); + content.add(CONTENT_FIELD_TEXT, new JsonPrimitive(text)); + json.add(content); + } + + @Override + public void addImageData(String format, String data) { + JsonObject content = new JsonObject(); + content.add("type", new JsonPrimitive("image_url")); + JsonObject urlContent = new JsonObject(); + String imageData = String.format(Locale.ROOT, "data:image/%s;base64,%s", format, data); + urlContent.add("url", new JsonPrimitive(imageData)); + content.add("image_url", urlContent); + json.add(content); + } + + @Override + public void addImageUrl(String format, String url) { + JsonObject content = new JsonObject(); + content.add("type", new JsonPrimitive("image_url")); + JsonObject urlContent = new JsonObject(); + urlContent.add("url", new JsonPrimitive(url)); + content.add("image_url", urlContent); + json.add(content); + } + + @Override + public JsonElement toJson() { + return this.json; + } + } + + static class BedrockContent implements MultimodalContent { + + private JsonArray json; + + public BedrockContent() { + this.json = new JsonArray(); + } + + public BedrockContent(String type, String value) { + this.json = new JsonArray(); + if (type.equals("text")) { + addText(value); + } + } + + @Override + public void addText(String text) { + JsonObject content = new JsonObject(); + content.add(CONTENT_FIELD_TEXT, new JsonPrimitive(text)); + json.add(content); + } + + @Override + public JsonElement toJson() { + return this.json; + } + + @Override + public void addImageData(String format, String data) { + JsonObject imageData = new JsonObject(); + imageData.add("bytes", new JsonPrimitive(data)); + JsonObject image = new JsonObject(); + image.add("format", new JsonPrimitive(format)); + image.add("source", imageData); + JsonObject content = new JsonObject(); + content.add("image", image); + json.add(content); + } + + @Override + public void addImageUrl(String format, String url) { + // Bedrock does not support image URLs. + } + + @Override + public void addDocument(String format, String name, String data) { + JsonObject documentData = new JsonObject(); + documentData.add("bytes", new JsonPrimitive(data)); + JsonObject document = new JsonObject(); + document.add("format", new JsonPrimitive(format)); + document.add("name", new JsonPrimitive(name)); + document.add("source", documentData); + JsonObject content = new JsonObject(); + content.add("document", document); + json.add(content); + } + } + + static class MessageArrayBuilder { + + private final Llm.ModelProvider provider; + private List messages = new ArrayList<>(); + private Message message = null; + private Content content = null; + + public MessageArrayBuilder(Llm.ModelProvider provider) { + // OpenAI or Bedrock Converse API + if (!EnumSet.of(Llm.ModelProvider.OPENAI, Llm.ModelProvider.BEDROCK_CONVERSE).contains(provider)) { + throw new IllegalArgumentException("Unsupported provider: " + provider); + } + this.provider = provider; + } + + public void startMessage(ChatRole role) { + this.message = new Message(); + this.message.setChatRole(role); + if (this.provider == Llm.ModelProvider.OPENAI) { + content = new OpenAIContent(); + } else if (this.provider == Llm.ModelProvider.BEDROCK_CONVERSE) { + content = new BedrockContent(); + } + } + + public void startMessage(ChatRole role, String text) { + startMessage(role); + addTextContent(text); + } + + public void endMessage() { + this.message.setContent(this.content); + this.messages.add(this.message); + message = null; + content = null; + } + + public void addTextContent(String content) { + if (this.message == null || this.content == null) { + throw new RuntimeException("You must call startMessage before calling addTextContent !!"); + } + this.content.addText(content); + } + + public void addImageData(String format, String data) { + if (this.content != null && this.content instanceof ImageContent) { + ((ImageContent) this.content).addImageData(format, data); + } + } + + public void addImageUrl(String format, String url) { + if (this.content != null && this.content instanceof ImageContent) { + ((ImageContent) this.content).addImageUrl(format, url); + } + } + + public void addDocumentContent(String format, String name, String data) { + if (this.content != null && this.content instanceof DocumentContent) { + ((DocumentContent) this.content).addDocument(format, name, data); + } + } + + public JsonArray toJsonArray() { + Preconditions + .checkState(this.message == null && this.content == null, "You must call endMessage before calling toJsonArray !!"); + + JsonArray ja = new JsonArray(); + for (Message message : messages) { + ja.add(message.toJson()); + } + return ja; + } + } + // TODO This is OpenAI specific. Either change this to OpenAiMessage or have it handle // vendor specific messages. static class Message { @@ -233,6 +559,12 @@ public Message(ChatRole chatRole, String content) { setContent(content); } + public Message(ChatRole chatRole, Content content) { + this(); + setChatRole(chatRole); + setContent(content); + } + public void setChatRole(ChatRole chatRole) { this.chatRole = chatRole; json.remove(MESSAGE_FIELD_ROLE); @@ -245,6 +577,11 @@ public void setContent(String content) { json.add(MESSAGE_FIELD_CONTENT, new JsonPrimitive(this.content)); } + public void setContent(Content content) { + json.remove(MESSAGE_FIELD_CONTENT); + json.add(MESSAGE_FIELD_CONTENT, content.toJson()); + } + public JsonObject toJson() { return json; } diff --git a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParamExtBuilderTests.java b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParamExtBuilderTests.java index 49f164cdb5..23eb6f3d3a 100644 --- a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParamExtBuilderTests.java +++ b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParamExtBuilderTests.java @@ -25,6 +25,8 @@ import java.io.EOFException; import java.io.IOException; +import java.util.List; +import java.util.Map; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.XContentType; @@ -33,10 +35,22 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentHelper; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.searchpipelines.questionanswering.generative.llm.MessageBlock; import org.opensearch.test.OpenSearchTestCase; public class GenerativeQAParamExtBuilderTests extends OpenSearchTestCase { + private List messageList = null; + + public GenerativeQAParamExtBuilderTests() { + Map imageMap = Map.of("image", Map.of("format", "jpg", "url", "https://xyz.com/file.jpg")); + Map textMap = Map.of("text", "what is this"); + Map contentMap = Map.of(); + Map map = Map.of("role", "user", "content", List.of(textMap, imageMap)); + MessageBlock mb = new MessageBlock(map); + messageList = List.of(mb); + } + public void testCtor() throws IOException { GenerativeQAParamExtBuilder builder = new GenerativeQAParamExtBuilder(); GenerativeQAParameters parameters = new GenerativeQAParameters( @@ -115,7 +129,7 @@ public void testParse() throws IOException { } public void testXContentRoundTrip() throws IOException { - GenerativeQAParameters param1 = new GenerativeQAParameters("a", "b", "c", "s", "u", null, null, null, null); + GenerativeQAParameters param1 = new GenerativeQAParameters("a", "b", "c", "s", "u", null, null, null, null, messageList); GenerativeQAParamExtBuilder extBuilder = new GenerativeQAParamExtBuilder(); extBuilder.setParams(param1); XContentType xContentType = randomFrom(XContentType.values()); diff --git a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParametersTests.java b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParametersTests.java index c36dcdb2a5..e5caa70ed7 100644 --- a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParametersTests.java +++ b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/ext/GenerativeQAParametersTests.java @@ -24,6 +24,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.opensearch.action.search.SearchRequest; import org.opensearch.core.common.io.stream.StreamOutput; @@ -31,10 +32,22 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentGenerator; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.searchpipelines.questionanswering.generative.llm.MessageBlock; import org.opensearch.test.OpenSearchTestCase; public class GenerativeQAParametersTests extends OpenSearchTestCase { + private List messageList = null; + + public GenerativeQAParametersTests() { + Map imageMap = Map.of("image", Map.of("format", "jpg", "url", "https://xyz.com/file.jpg")); + Map textMap = Map.of("text", "what is this"); + Map contentMap = Map.of(); + Map map = Map.of("role", "user", "content", List.of(textMap, imageMap)); + MessageBlock mb = new MessageBlock(map); + messageList = List.of(mb); + } + public void testGenerativeQAParameters() { GenerativeQAParameters params = new GenerativeQAParameters( "conversation_id", @@ -55,6 +68,29 @@ public void testGenerativeQAParameters() { assertEquals(params, actual); } + public void testGenerativeQAParametersWithLlmMessages() { + + GenerativeQAParameters params = new GenerativeQAParameters( + "conversation_id", + "llm_model", + "llm_question", + "system_prompt", + "user_instructions", + null, + null, + null, + null, + this.messageList + ); + GenerativeQAParamExtBuilder extBuilder = new GenerativeQAParamExtBuilder(); + extBuilder.setParams(params); + SearchSourceBuilder srcBulder = SearchSourceBuilder.searchSource().ext(List.of(extBuilder)); + SearchRequest request = new SearchRequest("my_index").source(srcBulder); + GenerativeQAParameters actual = GenerativeQAParamUtil.getGenerativeQAParameters(request); + // MessageBlock messageBlock = actual.getMessageBlock(); + assertEquals(params, actual); + } + static class DummyStreamOutput extends StreamOutput { List list = new ArrayList<>(); @@ -62,6 +98,7 @@ static class DummyStreamOutput extends StreamOutput { @Override public void writeString(String str) { + System.out.println("Adding string: " + str); list.add(str); } @@ -123,12 +160,13 @@ public void testWriteTo() throws IOException { contextSize, interactionSize, timeout, - llmResponseField + llmResponseField, + messageList ); StreamOutput output = new DummyStreamOutput(); parameters.writeTo(output); List actual = ((DummyStreamOutput) output).getList(); - assertEquals(6, actual.size()); + assertEquals(12, actual.size()); assertEquals(conversationId, actual.get(0)); assertEquals(llmModel, actual.get(1)); assertEquals(llmQuestion, actual.get(2)); @@ -190,7 +228,8 @@ public void testToXConent() throws IOException { null, null, null, - null + null, + messageList ); XContent xc = mock(XContent.class); OutputStream os = mock(OutputStream.class); diff --git a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInputTests.java b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInputTests.java index f3a4bf8284..d70739b8cd 100644 --- a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInputTests.java +++ b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/ChatCompletionInputTests.java @@ -43,6 +43,7 @@ public void testCtor() { systemPrompt, userInstructions, Llm.ModelProvider.OPENAI, + null, null ); @@ -81,6 +82,7 @@ public void testGettersSetters() { systemPrompt, userInstructions, Llm.ModelProvider.OPENAI, + null, null ); assertEquals(model, input.getModel()); diff --git a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImplTests.java b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImplTests.java index 2dc06366f8..5e5f72b59a 100644 --- a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImplTests.java +++ b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/DefaultLlmImplTests.java @@ -93,7 +93,7 @@ public void testBuildMessageParameter() { ) ) ); - String parameter = PromptUtil.getChatCompletionPrompt(question, chatHistory, contexts); + String parameter = PromptUtil.getChatCompletionPrompt(Llm.ModelProvider.BEDROCK_CONVERSE, question, chatHistory, contexts); Map parameters = Map.of("model", "foo", "messages", parameter); assertTrue(isJson(parameter)); } @@ -120,6 +120,7 @@ public void testChatCompletionApi() throws Exception { "prompt", "instructions", Llm.ModelProvider.OPENAI, + null, null ); doAnswer(invocation -> { @@ -164,6 +165,56 @@ public void testChatCompletionApiForBedrock() throws Exception { "prompt", "instructions", Llm.ModelProvider.BEDROCK, + null, + null + ); + doAnswer(invocation -> { + ((ActionListener) invocation.getArguments()[2]).onResponse(mlOutput); + return null; + }).when(mlClient).predict(any(), any(), any()); + connector.doChatCompletion(input, new ActionListener<>() { + @Override + public void onResponse(ChatCompletionOutput output) { + assertEquals("answer", output.getAnswers().get(0)); + } + + @Override + public void onFailure(Exception e) { + + } + }); + verify(mlClient, times(1)).predict(any(), captor.capture(), any()); + MLInput mlInput = captor.getValue(); + assertTrue(mlInput.getInputDataset() instanceof RemoteInferenceInputDataSet); + } + + public void testMessageApiForBedrockConverse() throws Exception { + MachineLearningInternalClient mlClient = mock(MachineLearningInternalClient.class); + ArgumentCaptor captor = ArgumentCaptor.forClass(MLInput.class); + DefaultLlmImpl connector = new DefaultLlmImpl("model_id", client); + connector.setMlClient(mlClient); + + Map messageMap = Map.of("role", "agent", "content", "answer"); + Map text = Map.of("text", "answer"); + List list = List.of(text); + Map content = Map.of("content", list); + Map message = Map.of("message", content); + Map dataAsMap = Map.of("output", message); + ModelTensor tensor = new ModelTensor("tensor", new Number[0], new long[0], MLResultDataType.STRING, null, null, dataAsMap); + ModelTensorOutput mlOutput = new ModelTensorOutput(List.of(new ModelTensors(List.of(tensor)))); + ActionFuture future = mock(ActionFuture.class); + when(future.actionGet(anyLong())).thenReturn(mlOutput); + when(mlClient.predict(any(), any())).thenReturn(future); + ChatCompletionInput input = new ChatCompletionInput( + "bedrock-converse/model", + "question", + Collections.emptyList(), + Collections.emptyList(), + 0, + "prompt", + "instructions", + Llm.ModelProvider.BEDROCK_CONVERSE, + null, null ); doAnswer(invocation -> { @@ -208,6 +259,7 @@ public void testChatCompletionApiForCohere() throws Exception { "prompt", "instructions", Llm.ModelProvider.COHERE, + null, null ); doAnswer(invocation -> { @@ -253,6 +305,7 @@ public void testChatCompletionApiForCohereWithError() throws Exception { "prompt", "instructions", Llm.ModelProvider.COHERE, + null, null ); doAnswer(invocation -> { @@ -300,7 +353,8 @@ public void testChatCompletionApiForFoo() throws Exception { "prompt", "instructions", null, - llmRespondField + llmRespondField, + null ); doAnswer(invocation -> { ((ActionListener) invocation.getArguments()[2]).onResponse(mlOutput); @@ -347,7 +401,8 @@ public void testChatCompletionApiForFooWithError() throws Exception { "prompt", "instructions", null, - llmRespondField + llmRespondField, + null ); doAnswer(invocation -> { ((ActionListener) invocation.getArguments()[2]).onResponse(mlOutput); @@ -395,7 +450,8 @@ public void testChatCompletionApiForFooWithErrorUnknownMessageField() throws Exc "prompt", "instructions", null, - llmRespondField + llmRespondField, + null ); doAnswer(invocation -> { ((ActionListener) invocation.getArguments()[2]).onResponse(mlOutput); @@ -443,7 +499,8 @@ public void testChatCompletionApiForFooWithErrorUnknownErrorField() throws Excep "prompt", "instructions", null, - llmRespondField + llmRespondField, + null ); doAnswer(invocation -> { ((ActionListener) invocation.getArguments()[2]).onResponse(mlOutput); @@ -489,6 +546,7 @@ public void testChatCompletionThrowingError() throws Exception { "prompt", "instructions", Llm.ModelProvider.OPENAI, + null, null ); @@ -536,6 +594,7 @@ public void testChatCompletionBedrockThrowingError() throws Exception { "prompt", "instructions", Llm.ModelProvider.BEDROCK, + null, null ); doAnswer(invocation -> { @@ -585,6 +644,7 @@ public void testIllegalArgument1() { "prompt", "instructions", null, + null, null ); connector.doChatCompletion(input, ActionListener.wrap(r -> {}, e -> {})); diff --git a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlockTests.java b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlockTests.java new file mode 100644 index 0000000000..62c6381b55 --- /dev/null +++ b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/llm/MessageBlockTests.java @@ -0,0 +1,103 @@ +/* + * Copyright 2023 Aryn + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.searchpipelines.questionanswering.generative.llm; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.junit.Rule; +import org.junit.rules.ExpectedException; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +public class MessageBlockTests extends OpenSearchTestCase { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + public void testStreamRoundTrip() throws Exception { + MessageBlock.TextBlock tb = new MessageBlock.TextBlock("text"); + MessageBlock.ImageBlock ib = new MessageBlock.ImageBlock("jpeg", "data", null); + MessageBlock.ImageBlock ib2 = new MessageBlock.ImageBlock("jpeg", null, "https://xyz/foo.jpg"); + MessageBlock.DocumentBlock db = new MessageBlock.DocumentBlock("pdf", "doc1", "data"); + List blocks = List.of(tb, ib, ib2, db); + MessageBlock mb = new MessageBlock(); + mb.setRole("user"); + mb.setBlockList(blocks); + BytesStreamOutput bso = new BytesStreamOutput(); + mb.writeTo(bso); + MessageBlock read = new MessageBlock(bso.bytes().streamInput()); + assertEquals(mb, read); + } + + public void testFromXContentParseError() throws IOException { + exceptionRule.expect(XContentParseException.class); + + MessageBlock.TextBlock tb = new MessageBlock.TextBlock("text"); + MessageBlock.ImageBlock ib = new MessageBlock.ImageBlock("jpeg", "data", null); + // MessageBlock.ImageBlock ib2 = new MessageBlock.ImageBlock("jpeg", null, "https://xyz/foo.jpg"); + MessageBlock.ImageBlock ib2 = new MessageBlock.ImageBlock(Map.of("format", "png", "data", "xyz")); + MessageBlock.DocumentBlock db = new MessageBlock.DocumentBlock("pdf", "doc1", "data"); + List blocks = List.of(tb, ib, ib2, db); + MessageBlock mb = new MessageBlock(); + mb.setRole("user"); + mb.setBlockList(blocks); + try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { + mb.toXContent(builder, ToXContent.EMPTY_PARAMS); + try (XContentBuilder shuffled = shuffleXContent(builder); XContentParser parser = createParser(shuffled)) { + // read = TaskResult.PARSER.apply(parser, null); + MessageBlock.fromXContent(parser); + } + } finally { + // throw new IOException("Error processing [" + mb + "]", e); + } + } + + public void testInvalidImageBlock1() { + exceptionRule.expect(IllegalArgumentException.class); + MessageBlock.ImageBlock ib = new MessageBlock.ImageBlock(Map.of("format", "png")); + } + + public void testInvalidImageBlock2() { + exceptionRule.expect(IllegalArgumentException.class); + MessageBlock.ImageBlock ib = new MessageBlock.ImageBlock("jpeg", null, null); + } + + public void testInvalidDocumentBlock1() { + exceptionRule.expect(NullPointerException.class); + MessageBlock.DocumentBlock db = new MessageBlock.DocumentBlock(null, null, null); + } + + public void testInvalidDocumentBlock2() { + exceptionRule.expect(IllegalStateException.class); + MessageBlock.DocumentBlock db = new MessageBlock.DocumentBlock(Map.of()); + } + + public void testDocumentBlockCtor1() { + MessageBlock.DocumentBlock db = new MessageBlock.DocumentBlock(Map.of("format", "pdf", "name", "doc", "data", "xyz")); + assertEquals(db.format, "pdf"); + assertEquals(db.name, "doc"); + assertEquals(db.data, "xyz"); + } +} diff --git a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtilTests.java b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtilTests.java index a3aedf4e5d..0d82a18a15 100644 --- a/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtilTests.java +++ b/search-processors/src/test/java/org/opensearch/searchpipelines/questionanswering/generative/prompt/PromptUtilTests.java @@ -26,12 +26,19 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.junit.Rule; +import org.junit.rules.ExpectedException; import org.opensearch.ml.common.conversation.ConversationalIndexConstants; import org.opensearch.ml.common.conversation.Interaction; +import org.opensearch.searchpipelines.questionanswering.generative.llm.Llm; +import org.opensearch.searchpipelines.questionanswering.generative.llm.MessageBlock; import org.opensearch.test.OpenSearchTestCase; public class PromptUtilTests extends OpenSearchTestCase { + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + public void testPromptUtilStaticMethods() { assertNull(PromptUtil.getQuestionRephrasingPrompt("question", Collections.emptyList())); } @@ -72,7 +79,50 @@ public void testBuildMessageParameter() { ); contexts.add("context 1"); contexts.add("context 2"); - String parameter = PromptUtil.buildMessageParameter(systemPrompt, userInstructions, question, chatHistory, contexts); + String parameter = PromptUtil + .buildMessageParameter(Llm.ModelProvider.BEDROCK_CONVERSE, systemPrompt, userInstructions, question, chatHistory, contexts); + Map parameters = Map.of("model", "foo", "messages", parameter); + assertTrue(isJson(parameter)); + } + + public void testBuildMessageParameterForOpenAI() { + String systemPrompt = "You are the best."; + String userInstructions = null; + String question = "Who am I"; + List contexts = new ArrayList<>(); + List chatHistory = List + .of( + Interaction + .fromMap( + "convo1", + Map + .of( + ConversationalIndexConstants.INTERACTIONS_CREATE_TIME_FIELD, + Instant.now().toString(), + ConversationalIndexConstants.INTERACTIONS_INPUT_FIELD, + "message 1", + ConversationalIndexConstants.INTERACTIONS_RESPONSE_FIELD, + "answer1" + ) + ), + Interaction + .fromMap( + "convo1", + Map + .of( + ConversationalIndexConstants.INTERACTIONS_CREATE_TIME_FIELD, + Instant.now().toString(), + ConversationalIndexConstants.INTERACTIONS_INPUT_FIELD, + "message 2", + ConversationalIndexConstants.INTERACTIONS_RESPONSE_FIELD, + "answer2" + ) + ) + ); + contexts.add("context 1"); + contexts.add("context 2"); + String parameter = PromptUtil + .buildMessageParameter(Llm.ModelProvider.OPENAI, systemPrompt, userInstructions, question, chatHistory, contexts); Map parameters = Map.of("model", "foo", "messages", parameter); assertTrue(isJson(parameter)); } @@ -117,6 +167,139 @@ public void testBuildBedrockInputParameter() { assertTrue(parameter.contains(systemPrompt)); } + public void testBuildBedrockConverseInputParameter() { + String systemPrompt = "You are the best."; + String userInstructions = null; + String question = "Who am I"; + List contexts = new ArrayList<>(); + List chatHistory = List + .of( + Interaction + .fromMap( + "convo1", + Map + .of( + ConversationalIndexConstants.INTERACTIONS_CREATE_TIME_FIELD, + Instant.now().toString(), + ConversationalIndexConstants.INTERACTIONS_INPUT_FIELD, + "message 1", + ConversationalIndexConstants.INTERACTIONS_RESPONSE_FIELD, + "answer1" + ) + ), + Interaction + .fromMap( + "convo1", + Map + .of( + ConversationalIndexConstants.INTERACTIONS_CREATE_TIME_FIELD, + Instant.now().toString(), + ConversationalIndexConstants.INTERACTIONS_INPUT_FIELD, + "message 2", + ConversationalIndexConstants.INTERACTIONS_RESPONSE_FIELD, + "answer2" + ) + ) + ); + contexts.add("context 1"); + contexts.add("context 2"); + MessageBlock.TextBlock tb = new MessageBlock.TextBlock("text"); + MessageBlock.ImageBlock ib = new MessageBlock.ImageBlock("jpeg", "data", null); + MessageBlock.DocumentBlock db = new MessageBlock.DocumentBlock("pdf", "file1", "data"); + List blocks = List.of(tb, ib, db); + MessageBlock mb = new MessageBlock(); + mb.setBlockList(blocks); + List llmMessages = List.of(mb); + String parameter = PromptUtil + .buildMessageParameter( + Llm.ModelProvider.BEDROCK_CONVERSE, + systemPrompt, + userInstructions, + question, + chatHistory, + contexts, + llmMessages + ); + assertTrue(parameter.contains(systemPrompt)); + } + + public void testBuildOpenAIInputParameter() { + String systemPrompt = "You are the best."; + String userInstructions = null; + String question = "Who am I"; + List contexts = new ArrayList<>(); + List chatHistory = List + .of( + Interaction + .fromMap( + "convo1", + Map + .of( + ConversationalIndexConstants.INTERACTIONS_CREATE_TIME_FIELD, + Instant.now().toString(), + ConversationalIndexConstants.INTERACTIONS_INPUT_FIELD, + "message 1", + ConversationalIndexConstants.INTERACTIONS_RESPONSE_FIELD, + "answer1" + ) + ), + Interaction + .fromMap( + "convo1", + Map + .of( + ConversationalIndexConstants.INTERACTIONS_CREATE_TIME_FIELD, + Instant.now().toString(), + ConversationalIndexConstants.INTERACTIONS_INPUT_FIELD, + "message 2", + ConversationalIndexConstants.INTERACTIONS_RESPONSE_FIELD, + "answer2" + ) + ) + ); + contexts.add("context 1"); + contexts.add("context 2"); + MessageBlock.TextBlock tb = new MessageBlock.TextBlock("text"); + MessageBlock.ImageBlock ib = new MessageBlock.ImageBlock("jpeg", "data", null); + MessageBlock.ImageBlock ib2 = new MessageBlock.ImageBlock("jpeg", null, "https://xyz/foo.jpg"); + List blocks = List.of(tb, ib, ib2); + MessageBlock mb = new MessageBlock(); + mb.setBlockList(blocks); + List llmMessages = List.of(mb); + String parameter = PromptUtil + .buildMessageParameter(Llm.ModelProvider.OPENAI, systemPrompt, userInstructions, question, chatHistory, contexts, llmMessages); + assertTrue(parameter.contains(systemPrompt)); + } + + public void testGetPromptTemplate() { + String systemPrompt = "you are a helpful assistant."; + String userInstructions = "lay out your answer as a sequence of steps."; + String actual = PromptUtil.getPromptTemplate(systemPrompt, userInstructions); + assertTrue(actual.contains(systemPrompt)); + assertTrue(actual.contains(userInstructions)); + } + + public void testMessageCtor() { + PromptUtil.Message message = new PromptUtil.Message(PromptUtil.ChatRole.USER, new PromptUtil.OpenAIContent()); + assertEquals(message.getChatRole(), PromptUtil.ChatRole.USER); + } + + public void testBedrockContentCtor() { + PromptUtil.Content content = new PromptUtil.BedrockContent("text", "foo"); + assertTrue(content.toJson().toString().contains("foo")); + } + + public void testMessageArrayBuilderCtor1() { + exceptionRule.expect(IllegalArgumentException.class); + PromptUtil.MessageArrayBuilder builder = new PromptUtil.MessageArrayBuilder(Llm.ModelProvider.COHERE); + } + + public void testMessageArrayBuilderInvalidUsage1() { + exceptionRule.expect(RuntimeException.class); + PromptUtil.MessageArrayBuilder builder = new PromptUtil.MessageArrayBuilder(Llm.ModelProvider.OPENAI); + builder.addTextContent("boom"); + } + private boolean isJson(String Json) { try { new JSONObject(Json); From 3c118e5378593f32e68c0b5021cce75bdda1bcb2 Mon Sep 17 00:00:00 2001 From: Dhrubo Saha Date: Fri, 6 Sep 2024 11:10:11 -0700 Subject: [PATCH 14/23] upgrading bounty castle version (#2903) Signed-off-by: Dhrubo Saha --- ml-algorithms/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ml-algorithms/build.gradle b/ml-algorithms/build.gradle index 5b7d146da7..7285623e25 100644 --- a/ml-algorithms/build.gradle +++ b/ml-algorithms/build.gradle @@ -65,7 +65,10 @@ dependencies { implementation platform('software.amazon.awssdk:bom:2.25.40') api 'software.amazon.awssdk:auth:2.25.40' implementation 'software.amazon.awssdk:apache-client' - implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1' + implementation ('com.amazonaws:aws-encryption-sdk-java:2.4.1') { + exclude group: 'org.bouncycastle', module: 'bcprov-ext-jdk18on' + } + implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1' implementation group: 'software.amazon.awssdk', name: 'aws-core', version: '2.25.40' implementation group: 'software.amazon.awssdk', name: 's3', version: '2.25.40' implementation group: 'software.amazon.awssdk', name: 'regions', version: '2.25.40' From a9c6a6e3dc713459994a314a3e2ba83442164b04 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Sun, 8 Sep 2024 21:08:51 -0700 Subject: [PATCH 15/23] Add back RAG IT tests that use Converse API. (#2916) * Add back RAG IT tests that use Converse API. Signed-off-by: Austin Lee * Fix spotless. Signed-off-by: Austin Lee --------- Signed-off-by: Austin Lee --- .../org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java index e8154f4c2d..8fb58c1b9a 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLRAGSearchProcessorIT.java @@ -34,7 +34,6 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Before; -import org.junit.Ignore; import org.opensearch.client.Response; import org.opensearch.core.rest.RestStatus; import org.opensearch.ml.common.MLTaskState; @@ -582,7 +581,6 @@ public void testBM25WithOpenAI() throws Exception { assertNotNull(answer); } - @Ignore public void testBM25WithOpenAIWithImage() throws Exception { // Skip test if key is null if (OPENAI_KEY == null) { @@ -722,7 +720,6 @@ public void testBM25WithBedrock() throws Exception { assertNotNull(answer); } - @Ignore public void testBM25WithBedrockConverse() throws Exception { // Skip test if key is null if (AWS_ACCESS_KEY_ID == null) { @@ -775,7 +772,6 @@ public void testBM25WithBedrockConverse() throws Exception { assertNotNull(answer); } - @Ignore public void testBM25WithBedrockConverseUsingLlmMessages() throws Exception { // Skip test if key is null if (AWS_ACCESS_KEY_ID == null) { @@ -836,7 +832,6 @@ public void testBM25WithBedrockConverseUsingLlmMessages() throws Exception { assertNotNull(answer); } - @Ignore public void testBM25WithBedrockConverseUsingLlmMessagesForDocumentChat() throws Exception { // Skip test if key is null if (AWS_ACCESS_KEY_ID == null) { @@ -953,7 +948,6 @@ public void testBM25WithOpenAIWithConversation() throws Exception { assertNotNull(interactionId); } - @Ignore public void testBM25WithOpenAIWithConversationAndImage() throws Exception { // Skip test if key is null if (OPENAI_KEY == null) { From 55d28e0a20ea2e5d12aa7079c427f7207fb75366 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Mon, 9 Sep 2024 10:53:45 -0700 Subject: [PATCH 16/23] Support null exception in ErrorMessage (#2900) * null exception Signed-off-by: Jing Zhang * address comments Signed-off-by: Jing Zhang --------- Signed-off-by: Jing Zhang --- .../java/org/opensearch/ml/utils/error/ErrorMessage.java | 6 +++++- .../org/opensearch/ml/utils/error/ErrorMessageTests.java | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/java/org/opensearch/ml/utils/error/ErrorMessage.java b/plugin/src/main/java/org/opensearch/ml/utils/error/ErrorMessage.java index 48307c50f2..0cb095cb06 100644 --- a/plugin/src/main/java/org/opensearch/ml/utils/error/ErrorMessage.java +++ b/plugin/src/main/java/org/opensearch/ml/utils/error/ErrorMessage.java @@ -44,7 +44,7 @@ public ErrorMessage(Throwable exception, int status) { } private String fetchType() { - return exception.getClass().getSimpleName(); + return exception == null ? "Unknown Exception" : exception.getClass().getSimpleName(); } protected String fetchReason() { @@ -52,6 +52,10 @@ protected String fetchReason() { } protected String fetchDetails() { + if (exception == null) { + return "No Exception Details"; + } + final String msg; // Prevent the method from exposing internal information such as internal ip address etc. that is a security concern. if (hasInternalInformation(exception)) { diff --git a/plugin/src/test/java/org/opensearch/ml/utils/error/ErrorMessageTests.java b/plugin/src/test/java/org/opensearch/ml/utils/error/ErrorMessageTests.java index 9e6e8586fe..14419a1ce9 100644 --- a/plugin/src/test/java/org/opensearch/ml/utils/error/ErrorMessageTests.java +++ b/plugin/src/test/java/org/opensearch/ml/utils/error/ErrorMessageTests.java @@ -144,4 +144,12 @@ public void getDetails() { assertEquals(errorMessage.getDetails(), "illegal state"); } + + @Test + public void ConstructNullException() { + ErrorMessage errorMessage = new ErrorMessage(null, SERVICE_UNAVAILABLE.getStatus()); + + assertEquals(errorMessage.getType(), "Unknown Exception"); + assertEquals(errorMessage.getDetails(), "No Exception Details"); + } } From a4a7c6bb980aaa44ed2783653db1480cbc4531c0 Mon Sep 17 00:00:00 2001 From: Xun Zhang Date: Tue, 10 Sep 2024 10:43:59 -0700 Subject: [PATCH 17/23] update the field mapping for batch ingest (#2921) * update the field mapping for batch ingest Signed-off-by: Xun Zhang --------- Signed-off-by: Xun Zhang --- .../batch/MLBatchIngestionInput.java | 26 +++- .../ml/engine/ingest/AbstractIngestion.java | 100 +++++++-------- .../engine/ingest/AbstractIngestionTests.java | 121 ++++++++++++------ .../engine/ingest/S3DataIngestionTests.java | 9 +- .../TransportBatchIngestionActionTests.java | 16 +-- .../rest/RestMLBatchIngestionActionTests.java | 4 +- ...tMLInferenceSearchResponseProcessorIT.java | 1 - .../org/opensearch/ml/utils/TestHelper.java | 10 +- 8 files changed, 166 insertions(+), 121 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java index e7050f0bd2..fbdc895efc 100644 --- a/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java +++ b/common/src/main/java/org/opensearch/ml/common/transport/batch/MLBatchIngestionInput.java @@ -28,13 +28,17 @@ public class MLBatchIngestionInput implements ToXContentObject, Writeable { public static final String INDEX_NAME_FIELD = "index_name"; public static final String FIELD_MAP_FIELD = "field_map"; - public static final String DATA_SOURCE_FIELD = "data_source"; + public static final String INGEST_FIELDS = "ingest_fields"; public static final String CONNECTOR_CREDENTIAL_FIELD = "credential"; + public static final String DATA_SOURCE_FIELD = "data_source"; + @Getter private String indexName; @Getter private Map fieldMapping; @Getter + private String[] ingestFields; + @Getter private Map dataSources; @Getter private Map credential; @@ -43,6 +47,7 @@ public class MLBatchIngestionInput implements ToXContentObject, Writeable { public MLBatchIngestionInput( String indexName, Map fieldMapping, + String[] ingestFields, Map dataSources, Map credential ) { @@ -58,6 +63,7 @@ public MLBatchIngestionInput( } this.indexName = indexName; this.fieldMapping = fieldMapping; + this.ingestFields = ingestFields; this.dataSources = dataSources; this.credential = credential; } @@ -65,6 +71,7 @@ public MLBatchIngestionInput( public static MLBatchIngestionInput parse(XContentParser parser) throws IOException { String indexName = null; Map fieldMapping = null; + String[] ingestFields = null; Map dataSources = null; Map credential = new HashMap<>(); @@ -80,6 +87,9 @@ public static MLBatchIngestionInput parse(XContentParser parser) throws IOExcept case FIELD_MAP_FIELD: fieldMapping = parser.map(); break; + case INGEST_FIELDS: + ingestFields = parser.list().toArray(new String[0]); + break; case CONNECTOR_CREDENTIAL_FIELD: credential = parser.mapStrings(); break; @@ -91,7 +101,7 @@ public static MLBatchIngestionInput parse(XContentParser parser) throws IOExcept break; } } - return new MLBatchIngestionInput(indexName, fieldMapping, dataSources, credential); + return new MLBatchIngestionInput(indexName, fieldMapping, ingestFields, dataSources, credential); } @Override @@ -103,6 +113,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (fieldMapping != null) { builder.field(FIELD_MAP_FIELD, fieldMapping); } + if (ingestFields != null) { + builder.field(INGEST_FIELDS, ingestFields); + } if (credential != null) { builder.field(CONNECTOR_CREDENTIAL_FIELD, credential); } @@ -122,6 +135,12 @@ public void writeTo(StreamOutput output) throws IOException { } else { output.writeBoolean(false); } + if (ingestFields != null) { + output.writeBoolean(true); + output.writeStringArray(ingestFields); + } else { + output.writeBoolean(false); + } if (credential != null) { output.writeBoolean(true); output.writeMap(credential, StreamOutput::writeString, StreamOutput::writeString); @@ -141,6 +160,9 @@ public MLBatchIngestionInput(StreamInput input) throws IOException { if (input.readBoolean()) { fieldMapping = input.readMap(s -> s.readString(), s -> s.readGenericValue()); } + if (input.readBoolean()) { + ingestFields = input.readStringArray(); + } if (input.readBoolean()) { credential = input.readMap(s -> s.readString(), s -> s.readString()); } diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java index be61f09e28..e0b075f567 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java @@ -8,11 +8,11 @@ import static org.opensearch.ml.common.utils.StringUtils.getJsonPath; import static org.opensearch.ml.common.utils.StringUtils.obtainFieldNameFromJsonPath; -import java.util.Collection; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -34,12 +34,6 @@ @Log4j2 public class AbstractIngestion implements Ingestable { - public static final String OUTPUT = "output"; - public static final String INPUT = "input"; - public static final String OUTPUT_FIELD_NAMES = "output_names"; - public static final String INPUT_FIELD_NAMES = "input_names"; - public static final String INGEST_FIELDS = "ingest_fields"; - public static final String ID_FIELD = "id_field"; private final Client client; @@ -85,12 +79,11 @@ protected double calculateSuccessRate(List successRates) { * Filters fields in the map where the value contains the specified source index as a prefix. * * @param mlBatchIngestionInput The MLBatchIngestionInput. - * @param index The source index to filter by. - * @return A new map with only the entries that match the specified source index. + * @param indexInFieldMap The source index to filter by. + * @return A new map with only the entries that match the specified source index and correctly mapped to JsonPath. */ - protected Map filterFieldMapping(MLBatchIngestionInput mlBatchIngestionInput, int index) { + protected Map filterFieldMapping(MLBatchIngestionInput mlBatchIngestionInput, int indexInFieldMap) { Map fieldMap = mlBatchIngestionInput.getFieldMapping(); - int indexInFieldMap = index + 1; String prefix = "source[" + indexInFieldMap + "]"; Map filteredFieldMap = fieldMap.entrySet().stream().filter(entry -> { @@ -104,19 +97,29 @@ protected Map filterFieldMapping(MLBatchIngestionInput mlBatchIn }).collect(Collectors.toMap(Map.Entry::getKey, entry -> { Object value = entry.getValue(); if (value instanceof String) { - return value; + return getJsonPath((String) value); } else if (value instanceof List) { - return ((List) value).stream().filter(val -> val.contains(prefix)).collect(Collectors.toList()); + return ((List) value) + .stream() + .filter(val -> val.contains(prefix)) + .map(StringUtils::getJsonPath) + .collect(Collectors.toList()); } return null; })); - if (filteredFieldMap.containsKey(OUTPUT)) { - filteredFieldMap.put(OUTPUT_FIELD_NAMES, fieldMap.get(OUTPUT_FIELD_NAMES)); - } - if (filteredFieldMap.containsKey(INPUT)) { - filteredFieldMap.put(INPUT_FIELD_NAMES, fieldMap.get(INPUT_FIELD_NAMES)); + String[] ingestFields = mlBatchIngestionInput.getIngestFields(); + if (ingestFields != null) { + Arrays + .stream(ingestFields) + .filter(Objects::nonNull) + .filter(val -> val.contains(prefix)) + .map(StringUtils::getJsonPath) + .forEach(jsonPath -> { + filteredFieldMap.put(obtainFieldNameFromJsonPath(jsonPath), jsonPath); + }); } + return filteredFieldMap; } @@ -128,42 +131,21 @@ protected Map filterFieldMapping(MLBatchIngestionInput mlBatchIn * @return A new map that contains all the fields and data for ingestion. */ protected Map processFieldMapping(String jsonStr, Map fieldMapping) { - String inputJsonPath = fieldMapping.containsKey(INPUT) ? getJsonPath((String) fieldMapping.get(INPUT)) : null; - List remoteModelInput = inputJsonPath != null ? (List) JsonPath.read(jsonStr, inputJsonPath) : null; - List inputFieldNames = inputJsonPath != null ? (List) fieldMapping.get(INPUT_FIELD_NAMES) : null; - - String outputJsonPath = fieldMapping.containsKey(OUTPUT) ? getJsonPath((String) fieldMapping.get(OUTPUT)) : null; - List remoteModelOutput = outputJsonPath != null ? (List) JsonPath.read(jsonStr, outputJsonPath) : null; - List outputFieldNames = outputJsonPath != null ? (List) fieldMapping.get(OUTPUT_FIELD_NAMES) : null; - - List ingestFieldsJsonPath = Optional - .ofNullable((List) fieldMapping.get(INGEST_FIELDS)) - .stream() - .flatMap(Collection::stream) - .map(StringUtils::getJsonPath) - .collect(Collectors.toList()); - Map jsonMap = new HashMap<>(); - - populateJsonMap(jsonMap, inputFieldNames, remoteModelInput); - populateJsonMap(jsonMap, outputFieldNames, remoteModelOutput); - - for (String fieldPath : ingestFieldsJsonPath) { - jsonMap.put(obtainFieldNameFromJsonPath(fieldPath), JsonPath.read(jsonStr, fieldPath)); + if (fieldMapping == null || fieldMapping.isEmpty()) { + return jsonMap; } - if (fieldMapping.containsKey(ID_FIELD)) { - List docIdJsonPath = Optional - .ofNullable((List) fieldMapping.get(ID_FIELD)) - .stream() - .flatMap(Collection::stream) - .map(StringUtils::getJsonPath) - .collect(Collectors.toList()); - if (docIdJsonPath.size() != 1) { - throw new IllegalArgumentException("The Id field must contains only 1 jsonPath for each source"); + fieldMapping.entrySet().stream().forEach(entry -> { + Object value = entry.getValue(); + if (value instanceof String) { + String jsonPath = (String) value; + jsonMap.put(entry.getKey(), JsonPath.read(jsonStr, jsonPath)); + } else if (value instanceof List) { + ((List) value).stream().forEach(jsonPath -> { jsonMap.put(entry.getKey(), JsonPath.read(jsonStr, jsonPath)); }); } - jsonMap.put("_id", JsonPath.read(jsonStr, docIdJsonPath.get(0))); - } + }); + return jsonMap; } @@ -180,12 +162,11 @@ protected void batchIngest( ? mlBatchIngestionInput.getFieldMapping() : filterFieldMapping(mlBatchIngestionInput, sourceIndex); Map jsonMap = processFieldMapping(jsonStr, filteredMapping); - if (isSoleSource || sourceIndex == 0) { + if (jsonMap.isEmpty()) { + return; + } + if (isSoleSource && !jsonMap.containsKey("_id")) { IndexRequest indexRequest = new IndexRequest(mlBatchIngestionInput.getIndexName()); - if (jsonMap.containsKey("_id")) { - String id = (String) jsonMap.remove("_id"); - indexRequest.id(id); - } indexRequest.source(jsonMap); bulkRequest.add(indexRequest); } else { @@ -198,6 +179,13 @@ protected void batchIngest( bulkRequest.add(updateRequest); } }); + if (bulkRequest.numberOfActions() == 0) { + bulkResponseListener + .onFailure( + new IllegalArgumentException("the bulk ingestion is empty: please check your field mapping to match your sources") + ); + return; + } client.bulk(bulkRequest, bulkResponseListener); } diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java index d2f66dacbc..a4c155ba77 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java @@ -13,14 +13,9 @@ import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.ID_FIELD; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INGEST_FIELDS; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT_FIELD_NAMES; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT_FIELD_NAMES; import java.util.Arrays; import java.util.Collections; @@ -34,6 +29,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.opensearch.action.bulk.BulkRequest; @@ -52,6 +48,7 @@ public class AbstractIngestionTests { S3DataIngestion s3DataIngestion = new S3DataIngestion(client); Map fieldMap; + String[] ingestFields; @Before public void setUp() { @@ -59,11 +56,12 @@ public void setUp() { s3DataIngestion = new S3DataIngestion(client); fieldMap = new HashMap<>(); - fieldMap.put(INPUT, "source[1].$.content"); - fieldMap.put(OUTPUT, "source[1].$.SageMakerOutput"); - fieldMap.put(INPUT_FIELD_NAMES, Arrays.asList("chapter", "title")); - fieldMap.put(OUTPUT_FIELD_NAMES, Arrays.asList("chapter_embedding", "title_embedding")); - fieldMap.put(INGEST_FIELDS, Arrays.asList("source[1].$.id")); + fieldMap.put("chapter", "$.content[0]"); + fieldMap.put("title", "$.content[1]"); + fieldMap.put("chapter_embedding", "$.SageMakerOutput[0]"); + fieldMap.put("title_embedding", "$.SageMakerOutput[1]"); + + ingestFields = new String[] { "$.id" }; } @Test @@ -161,38 +159,53 @@ public void testCalculateSuccessRate_SingleValue() { } @Test - public void testFilterFieldMapping_ValidInput_MatchingPrefix() { + public void testFilterFieldMapping_ValidInput_EmptyPrefix() { // Arrange - MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput( + "indexName", + fieldMap, + ingestFields, + new HashMap<>(), + new HashMap<>() + ); Map result = s3DataIngestion.filterFieldMapping(mlBatchIngestionInput, 0); // Assert - assertEquals(5, result.size()); - assertEquals("source[1].$.content", result.get(INPUT)); - assertEquals("source[1].$.SageMakerOutput", result.get(OUTPUT)); - assertEquals(Arrays.asList("chapter", "title"), result.get(INPUT_FIELD_NAMES)); - assertEquals(Arrays.asList("chapter_embedding", "title_embedding"), result.get(OUTPUT_FIELD_NAMES)); - assertEquals(Arrays.asList("source[1].$.id"), result.get(INGEST_FIELDS)); + assertEquals(0, result.size()); + assertEquals(true, result.isEmpty()); } @Test - public void testFilterFieldMapping_NoMatchingPrefix() { + public void testFilterFieldMapping_MatchingPrefix() { // Arrange Map fieldMap = new HashMap<>(); - fieldMap.put("field1", "source[3].$.response.body.data[*].embedding"); - fieldMap.put("field2", "source[4].$.body.input"); - - MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + fieldMap.put("question", "source[1].$.body.input[0]"); + fieldMap.put("question_embedding", "source[0].$.response.body.data[0].embedding"); + fieldMap.put("answer", "source[1].$.body.input[1]"); + fieldMap.put("answer_embedding", "source[0].$.response.body.data[1].embedding"); + fieldMap.put("_id", Arrays.asList("source[0].$.custom_id", "source[1].$.custom_id")); + + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput( + "indexName", + fieldMap, + ingestFields, + new HashMap<>(), + new HashMap<>() + ); // Act Map result = s3DataIngestion.filterFieldMapping(mlBatchIngestionInput, 0); // Assert - assertTrue(result.isEmpty()); + assertEquals(3, result.size()); + + assertEquals("$.response.body.data[0].embedding", result.get("question_embedding")); + assertEquals("$.response.body.data[1].embedding", result.get("answer_embedding")); + assertEquals(Arrays.asList("$.custom_id"), result.get("_id")); } @Test - public void testProcessFieldMapping_ValidInput() { + public void testProcessFieldMapping_FromSM() { String jsonStr = "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}"; // Arrange @@ -203,21 +216,34 @@ public void testProcessFieldMapping_ValidInput() { // Assert assertEquals("this is chapter 1", processedFieldMapping.get("chapter")); assertEquals("harry potter", processedFieldMapping.get("title")); - assertEquals(1, processedFieldMapping.get("id")); } @Test - public void testProcessFieldMapping_NoIdFieldInput() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("The Id field must contains only 1 jsonPath for each source"); + public void testProcessFieldMapping_FromOpenAI() { + String jsonStr = + "{\"id\": \"batch_req_pgNqCfERGHOcMwAHGUWSO0nV\", \"custom_id\": \"request-1\", \"response\": {\"status_code\": 200, \"request_id\": \"fca3d548770f1f299d067c64c11a14fd\", \"body\": {\"object\": \"list\", \"data\": [{\"object\": \"embedding\", \"index\": 0, \"embedding\": [0.0044326545, -0.029703418]}, {\"object\": \"embedding\", \"index\": 1, \"embedding\": [0.002297497, -0.009297881]}], \"model\": \"text-embedding-ada-002\", \"usage\": {\"prompt_tokens\": 15, \"total_tokens\": 15}}}, \"error\": null}"; + // Arrange + Map fieldMap = new HashMap<>(); + fieldMap.put("question_embedding", "$.response.body.data[0].embedding"); + fieldMap.put("answer_embedding", "$.response.body.data[1].embedding"); + fieldMap.put("_id", Arrays.asList("$.custom_id")); + + // Act + Map processedFieldMapping = s3DataIngestion.processFieldMapping(jsonStr, fieldMap); + // Assert + assertEquals("request-1", processedFieldMapping.get("_id")); + } + + @Test + public void testProcessFieldMapping_EmptyFieldInput() { String jsonStr = "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}"; // Arrange - fieldMap.put(ID_FIELD, null); // Act - s3DataIngestion.processFieldMapping(jsonStr, fieldMap); + Map result = s3DataIngestion.processFieldMapping(jsonStr, new HashMap<>()); + assertEquals(true, result.isEmpty()); } @Test @@ -232,7 +258,13 @@ public void testBatchIngestSuccess_SoleSource() { .asList( "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}" ); - MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput( + "indexName", + fieldMap, + ingestFields, + new HashMap<>(), + new HashMap<>() + ); ActionListener bulkResponseListener = mock(ActionListener.class); s3DataIngestion.batchIngest(sourceLines, mlBatchIngestionInput, bulkResponseListener, 0, true); @@ -241,10 +273,7 @@ public void testBatchIngestSuccess_SoleSource() { } @Test - public void testBatchIngestSuccess_NoIdError() { - exceptionRule.expect(IllegalArgumentException.class); - exceptionRule.expectMessage("The id filed must be provided to match documents for multiple sources"); - + public void testBatchIngestSuccess_returnForNullJasonMap() { doAnswer(invocation -> { ActionListener bulkResponseListener = invocation.getArgument(1); bulkResponseListener.onResponse(mock(BulkResponse.class)); @@ -255,8 +284,22 @@ public void testBatchIngestSuccess_NoIdError() { .asList( "{\"SageMakerOutput\":[[-0.017166402, 0.055771016],[-0.004301484,-0.042826906]],\"content\":[\"this is chapter 1\",\"harry potter\"],\"id\":1}" ); - MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput("indexName", fieldMap, new HashMap<>(), new HashMap<>()); + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput( + "indexName", + fieldMap, + ingestFields, + new HashMap<>(), + new HashMap<>() + ); ActionListener bulkResponseListener = mock(ActionListener.class); - s3DataIngestion.batchIngest(sourceLines, mlBatchIngestionInput, bulkResponseListener, 1, false); + s3DataIngestion.batchIngest(sourceLines, mlBatchIngestionInput, bulkResponseListener, 0, false); + + verify(client, never()).bulk(isA(BulkRequest.class), isA(ActionListener.class)); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(bulkResponseListener).onFailure(argumentCaptor.capture()); + assert (argumentCaptor + .getValue() + .getMessage() + .equals("the bulk ingestion is empty: please check your field mapping to match your sources")); } } diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java index 4bf2ffd58f..a7bea9ab55 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/S3DataIngestionTests.java @@ -5,9 +5,6 @@ package org.opensearch.ml.engine.ingest; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INGEST_FIELDS; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT_FIELD_NAMES; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT_FIELD_NAMES; import static org.opensearch.ml.engine.ingest.S3DataIngestion.SOURCE; import java.util.Arrays; @@ -39,11 +36,7 @@ public void setUp() { s3DataIngestion = new S3DataIngestion(client); Map fieldMap = new HashMap<>(); - fieldMap.put("input", "$.content"); - fieldMap.put("output", "$.SageMakerOutput"); - fieldMap.put(INPUT_FIELD_NAMES, Arrays.asList("chapter", "title")); - fieldMap.put(OUTPUT_FIELD_NAMES, Arrays.asList("chapter_embedding", "title_embedding")); - fieldMap.put(INGEST_FIELDS, Arrays.asList("$.id")); + fieldMap.put("chapter", "$.content[0]"); Map credential = Map .of("region", "us-east-1", "access_key", "some accesskey", "secret_key", "some secret", "session_token", "some token"); diff --git a/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java index 7b3766dadf..2916359110 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java @@ -15,9 +15,6 @@ import static org.opensearch.ml.common.MLTask.STATE_FIELD; import static org.opensearch.ml.common.MLTaskState.COMPLETED; import static org.opensearch.ml.common.MLTaskState.FAILED; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INGEST_FIELDS; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.INPUT_FIELD_NAMES; -import static org.opensearch.ml.engine.ingest.AbstractIngestion.OUTPUT_FIELD_NAMES; import static org.opensearch.ml.engine.ingest.S3DataIngestion.SOURCE; import static org.opensearch.ml.task.MLTaskManager.TASK_SEMAPHORE_TIMEOUT; @@ -68,6 +65,7 @@ public class TransportBatchIngestionActionTests extends OpenSearchTestCase { private TransportBatchIngestionAction batchAction; private MLBatchIngestionInput batchInput; + private String[] ingestFields; @Before public void setup() { @@ -75,11 +73,12 @@ public void setup() { batchAction = new TransportBatchIngestionAction(transportService, actionFilters, client, mlTaskManager, threadPool); Map fieldMap = new HashMap<>(); - fieldMap.put("input", "$.content"); - fieldMap.put("output", "$.SageMakerOutput"); - fieldMap.put(INPUT_FIELD_NAMES, Arrays.asList("chapter", "title")); - fieldMap.put(OUTPUT_FIELD_NAMES, Arrays.asList("chapter_embedding", "title_embedding")); - fieldMap.put(INGEST_FIELDS, Arrays.asList("$.id")); + fieldMap.put("chapter", "$.content[0]"); + fieldMap.put("title", "$.content[1]"); + fieldMap.put("chapter_embedding", "$.SageMakerOutput[0]"); + fieldMap.put("title_embedding", "$.SageMakerOutput[1]"); + + ingestFields = new String[] { "$.id" }; Map credential = Map .of("region", "us-east-1", "access_key", "some accesskey", "secret_key", "some secret", "session_token", "some token"); @@ -91,6 +90,7 @@ public void setup() { .builder() .indexName("testIndex") .fieldMapping(fieldMap) + .ingestFields(ingestFields) .credential(credential) .dataSources(dataSource) .build(); diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java index 9b6a00c8d7..98c7795dd9 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLBatchIngestionActionTests.java @@ -97,7 +97,7 @@ public void testGetRequest() throws IOException { MLBatchIngestionInput mlBatchIngestionInput = mlBatchIngestionRequest.getMlBatchIngestionInput(); assertEquals("test batch index", mlBatchIngestionInput.getIndexName()); - assertEquals("$.content", mlBatchIngestionInput.getFieldMapping().get("input")); + assertEquals("$.content[0]", mlBatchIngestionInput.getFieldMapping().get("chapter")); assertNotNull(mlBatchIngestionInput.getDataSources().get("source")); assertNotNull(mlBatchIngestionInput.getCredential()); } @@ -110,7 +110,7 @@ public void testPrepareRequest() throws Exception { verify(client, times(1)).execute(eq(MLBatchIngestionAction.INSTANCE), argumentCaptor.capture(), any()); MLBatchIngestionInput mlBatchIngestionInput = argumentCaptor.getValue().getMlBatchIngestionInput(); assertEquals("test batch index", mlBatchIngestionInput.getIndexName()); - assertEquals("$.content", mlBatchIngestionInput.getFieldMapping().get("input")); + assertEquals("$.content[0]", mlBatchIngestionInput.getFieldMapping().get("chapter")); assertNotNull(mlBatchIngestionInput.getDataSources().get("source")); assertNotNull(mlBatchIngestionInput.getCredential()); } diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java index 9c82547623..d93c0aeba7 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java @@ -244,7 +244,6 @@ public void testMLInferenceProcessorRemoteModelCustomPrompt() throws Exception { createSearchPipelineProcessor(createPipelineRequestBody, pipelineName); Map response = searchWithPipeline(client(), index_name, pipelineName, query); - System.out.println(response); Assert.assertNotNull(JsonPath.parse(response).read("$.hits.hits[0]._source.llm_response")); Assert.assertNotNull(JsonPath.parse(response).read("$.hits.hits[1]._source.llm_response")); } diff --git a/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java b/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java index 26f0afa8fb..11c0fef7ed 100644 --- a/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java +++ b/plugin/src/test/java/org/opensearch/ml/utils/TestHelper.java @@ -527,12 +527,12 @@ public static RestRequest getBatchIngestionRestRequest() { final String requestContent = "{\n" + " \"index_name\": \"test batch index\",\n" + " \"field_map\": {\n" - + " \"input\": \"$.content\",\n" - + " \"output\": \"$.SageMakerOutput\",\n" - + " \"input_names\": [\"chapter\", \"title\"],\n" - + " \"output_names\": [\"chapter_embedding\", \"title_embedding\"],\n" - + " \"ingest_fields\": [\"$.id\"]\n" + + " \"chapter\": \"$.content[0]\",\n" + + " \"title\": \"$.content[1]\",\n" + + " \"chapter_embedding\": \"$.SageMakerOutput[0]\",\n" + + " \"title_embedding\": \"$.SageMakerOutput[1]\"\n" + " },\n" + + " \"ingest_fields\": [\"$.id\"],\n" + " \"credential\": {\n" + " \"region\": \"xxxxxxxx\"\n" + " },\n" From 93d0429db5f7d32305362e985d0f9ffa908d5275 Mon Sep 17 00:00:00 2001 From: Mingshi Liu Date: Tue, 10 Sep 2024 12:04:43 -0700 Subject: [PATCH 18/23] add skip when aws key is null (#2926) Signed-off-by: Mingshi Liu --- .../ml/rest/RestMLInferenceSearchResponseProcessorIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java index d93c0aeba7..2efa0a442c 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLInferenceSearchResponseProcessorIT.java @@ -257,6 +257,10 @@ public void testMLInferenceProcessorRemoteModelCustomPrompt() throws Exception { * @throws Exception if any error occurs during the test */ public void testMLInferenceProcessorRemoteModelStringField() throws Exception { + // Skip test if key is null + if (AWS_ACCESS_KEY_ID == null) { + return; + } String createPipelineRequestBody = "{\n" + " \"response_processors\": [\n" + " {\n" From 30228ea458375fabe0287430b01ef9355b832045 Mon Sep 17 00:00:00 2001 From: Xun Zhang Date: Thu, 12 Sep 2024 14:11:07 -0700 Subject: [PATCH 19/23] =?UTF-8?q?fix=20field=20mapping,=20add=20more=20err?= =?UTF-8?q?or=20handling=20and=20remove=20checking=20jobId=20=E2=80=A6=20(?= =?UTF-8?q?#2933)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix field mapping, add more error handling and remove checking jobId filed in batch job response Signed-off-by: Xun Zhang * add beckrock url in the allowed list and more UTs Signed-off-by: Xun Zhang --------- Signed-off-by: Xun Zhang --- .../ml/engine/ingest/AbstractIngestion.java | 53 ++++++++++++++++++- .../engine/ingest/AbstractIngestionTests.java | 29 ++++++++++ .../batch/TransportBatchIngestionAction.java | 36 +++++++++++-- .../ml/plugin/MachineLearningPlugin.java | 12 ++++- .../ml/settings/MLCommonsSettings.java | 3 +- .../ml/task/MLPredictTaskRunner.java | 4 +- .../TransportBatchIngestionActionTests.java | 44 +++++++++++++++ .../ml/task/MLPredictTaskRunnerTests.java | 7 ++- 8 files changed, 174 insertions(+), 14 deletions(-) diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java index e0b075f567..ce6fa19145 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/ingest/AbstractIngestion.java @@ -75,6 +75,55 @@ protected double calculateSuccessRate(List successRates) { ); } + /** + * Filters fields in the map where the value contains the specified source index as a prefix. + * When there is only one source file, users can skip the source[] prefix + * + * @param mlBatchIngestionInput The MLBatchIngestionInput. + * @return A new map of for all fields to be ingested. + */ + protected Map filterFieldMappingSoleSource(MLBatchIngestionInput mlBatchIngestionInput) { + Map fieldMap = mlBatchIngestionInput.getFieldMapping(); + String prefix = "source[0]"; + + Map filteredFieldMap = fieldMap.entrySet().stream().filter(entry -> { + Object value = entry.getValue(); + if (value instanceof String) { + String jsonPath = ((String) value); + return jsonPath.contains(prefix) || !jsonPath.startsWith("source"); + } else if (value instanceof List) { + return ((List) value).stream().anyMatch(val -> (val.contains(prefix) || !val.startsWith("source"))); + } + return false; + }).collect(Collectors.toMap(Map.Entry::getKey, entry -> { + Object value = entry.getValue(); + if (value instanceof String) { + return getJsonPath((String) value); + } else if (value instanceof List) { + return ((List) value) + .stream() + .filter(val -> (val.contains(prefix) || !val.startsWith("source"))) + .map(StringUtils::getJsonPath) + .collect(Collectors.toList()); + } + return null; + })); + + String[] ingestFields = mlBatchIngestionInput.getIngestFields(); + if (ingestFields != null) { + Arrays + .stream(ingestFields) + .filter(Objects::nonNull) + .filter(val -> (val.contains(prefix) || !val.startsWith("source"))) + .map(StringUtils::getJsonPath) + .forEach(jsonPath -> { + filteredFieldMap.put(obtainFieldNameFromJsonPath(jsonPath), jsonPath); + }); + } + + return filteredFieldMap; + } + /** * Filters fields in the map where the value contains the specified source index as a prefix. * @@ -159,7 +208,7 @@ protected void batchIngest( BulkRequest bulkRequest = new BulkRequest(); sourceLines.stream().forEach(jsonStr -> { Map filteredMapping = isSoleSource - ? mlBatchIngestionInput.getFieldMapping() + ? filterFieldMappingSoleSource(mlBatchIngestionInput) : filterFieldMapping(mlBatchIngestionInput, sourceIndex); Map jsonMap = processFieldMapping(jsonStr, filteredMapping); if (jsonMap.isEmpty()) { @@ -174,7 +223,7 @@ protected void batchIngest( if (!jsonMap.containsKey("_id")) { throw new IllegalArgumentException("The id filed must be provided to match documents for multiple sources"); } - String id = (String) jsonMap.remove("_id"); + String id = String.valueOf(jsonMap.remove("_id")); UpdateRequest updateRequest = new UpdateRequest(mlBatchIngestionInput.getIndexName(), id).doc(jsonMap).upsert(jsonMap); bulkRequest.add(updateRequest); } diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java index a4c155ba77..1f1653b31c 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/ingest/AbstractIngestionTests.java @@ -204,6 +204,35 @@ public void testFilterFieldMapping_MatchingPrefix() { assertEquals(Arrays.asList("$.custom_id"), result.get("_id")); } + @Test + public void testFilterFieldMappingSoleSource_MatchingPrefix() { + // Arrange + Map fieldMap = new HashMap<>(); + fieldMap.put("question", "source[0].$.body.input[0]"); + fieldMap.put("question_embedding", "source[0].$.response.body.data[0].embedding"); + fieldMap.put("answer", "source[0].$.body.input[1]"); + fieldMap.put("answer_embedding", "$.response.body.data[1].embedding"); + fieldMap.put("_id", Arrays.asList("$.custom_id", "source[1].$.custom_id")); + + MLBatchIngestionInput mlBatchIngestionInput = new MLBatchIngestionInput( + "indexName", + fieldMap, + ingestFields, + new HashMap<>(), + new HashMap<>() + ); + + // Act + Map result = s3DataIngestion.filterFieldMappingSoleSource(mlBatchIngestionInput); + + // Assert + assertEquals(6, result.size()); + + assertEquals("$.body.input[0]", result.get("question")); + assertEquals("$.response.body.data[0].embedding", result.get("question_embedding")); + assertEquals(Arrays.asList("$.custom_id"), result.get("_id")); + } + @Test public void testProcessFieldMapping_FromSM() { String jsonStr = diff --git a/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java b/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java index cf03d0f11a..6fd03b7b52 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/batch/TransportBatchIngestionAction.java @@ -9,7 +9,7 @@ import static org.opensearch.ml.common.MLTask.STATE_FIELD; import static org.opensearch.ml.common.MLTaskState.COMPLETED; import static org.opensearch.ml.common.MLTaskState.FAILED; -import static org.opensearch.ml.plugin.MachineLearningPlugin.TRAIN_THREAD_POOL; +import static org.opensearch.ml.plugin.MachineLearningPlugin.INGEST_THREAD_POOL; import static org.opensearch.ml.task.MLTaskManager.TASK_SEMAPHORE_TIMEOUT; import java.time.Instant; @@ -41,6 +41,8 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import com.jayway.jsonpath.PathNotFoundException; + import lombok.extern.log4j.Log4j2; @Log4j2 @@ -92,9 +94,11 @@ protected void doExecute(Task task, ActionRequest request, ActionListener { - double successRate = ingestable.ingest(mlBatchIngestionInput); - handleSuccessRate(successRate, taskId); + threadPool.executor(INGEST_THREAD_POOL).execute(() -> { + executeWithErrorHandling(() -> { + double successRate = ingestable.ingest(mlBatchIngestionInput); + handleSuccessRate(successRate, taskId); + }, taskId); }); } catch (Exception ex) { log.error("Failed in batch ingestion", ex); @@ -125,6 +129,30 @@ protected void doExecute(Task task, ActionRequest request, ActionListener> getExecutorBuilders(Settings settings) { ML_THREAD_POOL_PREFIX + REMOTE_PREDICT_THREAD_POOL, false ); + FixedExecutorBuilder batchIngestThreadPool = new FixedExecutorBuilder( + settings, + INGEST_THREAD_POOL, + OpenSearchExecutors.allocatedProcessors(settings) * 4, + 30, + ML_THREAD_POOL_PREFIX + INGEST_THREAD_POOL, + false + ); return ImmutableList .of( @@ -894,7 +903,8 @@ public List> getExecutorBuilders(Settings settings) { executeThreadPool, trainThreadPool, predictThreadPool, - remotePredictThreadPool + remotePredictThreadPool, + batchIngestThreadPool ); } diff --git a/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java b/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java index 6daffd30fd..339116226d 100644 --- a/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java +++ b/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java @@ -146,7 +146,8 @@ private MLCommonsSettings() {} "^https://api\\.openai\\.com/.*$", "^https://api\\.cohere\\.ai/.*$", "^https://bedrock-runtime\\..*[a-z0-9-]\\.amazonaws\\.com/.*$", - "^https://bedrock-agent-runtime\\..*[a-z0-9-]\\.amazonaws\\.com/.*$" + "^https://bedrock-agent-runtime\\..*[a-z0-9-]\\.amazonaws\\.com/.*$", + "^https://bedrock\\..*[a-z0-9-]\\.amazonaws\\.com/.*$" ), Function.identity(), Setting.Property.NodeScope, diff --git a/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java b/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java index 3b2e70d4b8..525ae12a88 100644 --- a/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java +++ b/plugin/src/main/java/org/opensearch/ml/task/MLPredictTaskRunner.java @@ -358,13 +358,13 @@ private void runPredict( && tensorOutput.getMlModelOutputs() != null && !tensorOutput.getMlModelOutputs().isEmpty()) { ModelTensors modelOutput = tensorOutput.getMlModelOutputs().get(0); + Integer statusCode = modelOutput.getStatusCode(); if (modelOutput.getMlModelTensors() != null && !modelOutput.getMlModelTensors().isEmpty()) { Map dataAsMap = (Map) modelOutput .getMlModelTensors() .get(0) .getDataAsMap(); - if (dataAsMap != null - && (dataAsMap.containsKey("TransformJobArn") || dataAsMap.containsKey("id"))) { + if (dataAsMap != null && statusCode != null && statusCode >= 200 && statusCode < 300) { remoteJob.putAll(dataAsMap); mlTask.setRemoteJob(remoteJob); mlTask.setTaskId(null); diff --git a/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java index 2916359110..092edfe951 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/batch/TransportBatchIngestionActionTests.java @@ -6,9 +6,14 @@ package org.opensearch.ml.action.batch; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.ml.common.MLTask.ERROR_FIELD; @@ -16,12 +21,14 @@ import static org.opensearch.ml.common.MLTaskState.COMPLETED; import static org.opensearch.ml.common.MLTaskState.FAILED; import static org.opensearch.ml.engine.ingest.S3DataIngestion.SOURCE; +import static org.opensearch.ml.plugin.MachineLearningPlugin.INGEST_THREAD_POOL; import static org.opensearch.ml.task.MLTaskManager.TASK_SEMAPHORE_TIMEOUT; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutorService; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -45,6 +52,8 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import com.jayway.jsonpath.PathNotFoundException; + public class TransportBatchIngestionActionTests extends OpenSearchTestCase { @Mock private Client client; @@ -62,6 +71,8 @@ public class TransportBatchIngestionActionTests extends OpenSearchTestCase { ActionListener actionListener; @Mock ThreadPool threadPool; + @Mock + ExecutorService executorService; private TransportBatchIngestionAction batchAction; private MLBatchIngestionInput batchInput; @@ -105,9 +116,42 @@ public void test_doExecute_success() { listener.onResponse(indexResponse); return null; }).when(mlTaskManager).createMLTask(isA(MLTask.class), isA(ActionListener.class)); + doReturn(executorService).when(threadPool).executor(INGEST_THREAD_POOL); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + batchAction.doExecute(task, mlBatchIngestionRequest, actionListener); verify(actionListener).onResponse(any(MLBatchIngestionResponse.class)); + verify(threadPool).executor(INGEST_THREAD_POOL); + } + + public void test_doExecute_ExecuteWithNoErrorHandling() { + batchAction.executeWithErrorHandling(() -> {}, "taskId"); + + verify(mlTaskManager, never()).updateMLTask(anyString(), isA(Map.class), anyLong(), anyBoolean()); + } + + public void test_doExecute_ExecuteWithPathNotFoundException() { + batchAction.executeWithErrorHandling(() -> { throw new PathNotFoundException("jsonPath not found!"); }, "taskId"); + + verify(mlTaskManager) + .updateMLTask("taskId", Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "jsonPath not found!"), TASK_SEMAPHORE_TIMEOUT, true); + } + + public void test_doExecute_RuntimeException() { + batchAction.executeWithErrorHandling(() -> { throw new RuntimeException("runtime exception in the ingestion!"); }, "taskId"); + + verify(mlTaskManager) + .updateMLTask( + "taskId", + Map.of(STATE_FIELD, FAILED, ERROR_FIELD, "runtime exception in the ingestion!"), + TASK_SEMAPHORE_TIMEOUT, + true + ); } public void test_doExecute_handleSuccessRate100() { diff --git a/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java b/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java index 064008a9c4..223f2ce5a5 100644 --- a/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java +++ b/plugin/src/test/java/org/opensearch/ml/task/MLPredictTaskRunnerTests.java @@ -447,10 +447,9 @@ public void testValidateBatchPredictionSuccess() throws IOException { "output", "{\"properties\":{\"inference_results\":{\"description\":\"This is a test description field\"," + "\"type\":\"array\"}}}" ); - ModelTensorOutput modelTensorOutput = ModelTensorOutput - .builder() - .mlModelOutputs(List.of(ModelTensors.builder().mlModelTensors(List.of(modelTensor)).build())) - .build(); + ModelTensors modelTensors = ModelTensors.builder().statusCode(200).mlModelTensors(List.of(modelTensor)).statusCode(200).build(); + modelTensors.setStatusCode(200); + ModelTensorOutput modelTensorOutput = ModelTensorOutput.builder().mlModelOutputs(List.of(modelTensors)).build(); doAnswer(invocation -> { ActionListener actionListener = invocation.getArgument(1); actionListener.onResponse(MLTaskResponse.builder().output(modelTensorOutput).build()); From bbaea1c3f5b141512ed81562893e5ddfb192996b Mon Sep 17 00:00:00 2001 From: Yaliang Wu Date: Thu, 12 Sep 2024 15:46:30 -0700 Subject: [PATCH 20/23] fix get batch task bug (#2937) Signed-off-by: Yaliang Wu --- .../algorithms/remote/AwsConnectorExecutor.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java index 2ebc7ce563..e0bcd1bc73 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/remote/AwsConnectorExecutor.java @@ -6,11 +6,13 @@ package org.opensearch.ml.engine.algorithms.remote; import static org.opensearch.ml.common.connector.ConnectorProtocols.AWS_SIGV4; +import static software.amazon.awssdk.http.SdkHttpMethod.GET; import static software.amazon.awssdk.http.SdkHttpMethod.POST; import java.security.AccessController; import java.security.PrivilegedExceptionAction; import java.time.Duration; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -86,7 +88,18 @@ public void invokeRemoteService( ActionListener> actionListener ) { try { - SdkHttpFullRequest request = ConnectorUtils.buildSdkRequest(action, connector, parameters, payload, POST); + SdkHttpFullRequest request; + switch (connector.getActionHttpMethod(action).toUpperCase(Locale.ROOT)) { + case "POST": + log.debug("original payload to remote model: " + payload); + request = ConnectorUtils.buildSdkRequest(action, connector, parameters, payload, POST); + break; + case "GET": + request = ConnectorUtils.buildSdkRequest(action, connector, parameters, null, GET); + break; + default: + throw new IllegalArgumentException("unsupported http method"); + } AsyncExecuteRequest executeRequest = AsyncExecuteRequest .builder() .request(signRequest(request)) From 853efd6be34f981740dec99dd649754eb085de83 Mon Sep 17 00:00:00 2001 From: Mingshi Liu Date: Fri, 13 Sep 2024 11:15:32 -0700 Subject: [PATCH 21/23] fix full_response false and no mapping exceptions (#2944) Signed-off-by: Mingshi Liu --- .../MLInferenceSearchResponseProcessor.java | 2 +- ...InferenceSearchResponseProcessorTests.java | 213 ++++++++++++++++-- 2 files changed, 194 insertions(+), 21 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java b/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java index 38e62528f3..2164877b9f 100644 --- a/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java +++ b/plugin/src/main/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessor.java @@ -634,7 +634,7 @@ private static Map getDefaultOutputMapping(Integer mappingIndex, Map outputMapping; if (processOutputMap == null || processOutputMap.size() == 0) { outputMapping = new HashMap<>(); - outputMapping.put(DEFAULT_OUTPUT_FIELD_NAME, "$." + DEFAULT_OUTPUT_FIELD_NAME); + outputMapping.put(DEFAULT_OUTPUT_FIELD_NAME, null); } else { outputMapping = processOutputMap.get(mappingIndex); } diff --git a/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java b/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java index 62b397f84b..8f04cab9d4 100644 --- a/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java +++ b/plugin/src/test/java/org/opensearch/ml/processor/MLInferenceSearchResponseProcessorTests.java @@ -351,6 +351,158 @@ public void onFailure(Exception e) { verify(client, times(1)).execute(any(), any(), any()); } + /** + * Tests create processor with one_to_one is false + * with custom prompt + * with many to one prediction, 5 documents in hits are calling 1 prediction tasks + * with full response path false and no output mapping is provided + * @throws Exception if an error occurs during the test + */ + public void testProcessResponseManyToOneWithCustomPromptFullResponsePathFalse() throws Exception { + + String documentField = "text"; + String modelInputField = "context"; + List> inputMap = new ArrayList<>(); + Map input = new HashMap<>(); + input.put(modelInputField, documentField); + inputMap.add(input); + + Map modelConfig = new HashMap<>(); + modelConfig + .put( + "prompt", + "\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: ${parameters.context}. \\n\\n Human: please summarize the documents \\n\\n Assistant:" + ); + MLInferenceSearchResponseProcessor responseProcessor = new MLInferenceSearchResponseProcessor( + "model1", + inputMap, + null, + modelConfig, + DEFAULT_MAX_PREDICTION_TASKS, + PROCESSOR_TAG, + DESCRIPTION, + false, + "remote", + false, + false, + false, + "{ \"parameters\": ${ml_inference.parameters} }", + client, + TEST_XCONTENT_REGISTRY_FOR_QUERY, + false + ); + + SearchRequest request = getSearchRequest(); + String fieldName = "text"; + SearchResponse response = getSearchResponse(5, true, fieldName); + Map predictionResult = ImmutableMap.of("response", "here is a summary of the documents"); + + ModelTensor modelTensor = ModelTensor.builder().dataAsMap(predictionResult).build(); + ModelTensors modelTensors = ModelTensors.builder().mlModelTensors(Arrays.asList(modelTensor)).build(); + ModelTensorOutput mlModelTensorOutput = ModelTensorOutput.builder().mlModelOutputs(Arrays.asList(modelTensors)).build(); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + actionListener.onResponse(MLTaskResponse.builder().output(mlModelTensorOutput).build()); + return null; + }).when(client).execute(any(), any(), any()); + + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(SearchResponse newSearchResponse) { + assertEquals(newSearchResponse.getHits().getHits().length, 5); + assertEquals(newSearchResponse.getHits().getHits()[0].getSourceAsMap().get("inference_results"), predictionResult); + assertEquals(newSearchResponse.getHits().getHits()[1].getSourceAsMap().get("inference_results"), predictionResult); + assertEquals(newSearchResponse.getHits().getHits()[2].getSourceAsMap().get("inference_results"), predictionResult); + assertEquals(newSearchResponse.getHits().getHits()[3].getSourceAsMap().get("inference_results"), predictionResult); + } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } + + }; + responseProcessor.processResponseAsync(request, response, responseContext, listener); + verify(client, times(1)).execute(any(), any(), any()); + } + + /** + * Tests create processor with one_to_one is false + * with custom prompt + * with many to one prediction, 5 documents in hits are calling 1 prediction tasks + * with full response path true and no output mapping is provided + * @throws Exception if an error occurs during the test + */ + public void testProcessResponseManyToOneWithCustomPromptFullResponsePathTrue() throws Exception { + + String documentField = "text"; + String modelInputField = "context"; + List> inputMap = new ArrayList<>(); + Map input = new HashMap<>(); + input.put(modelInputField, documentField); + inputMap.add(input); + + Map modelConfig = new HashMap<>(); + modelConfig + .put( + "prompt", + "\\n\\nHuman: You are a professional data analyst. You will always answer question based on the given context first. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say I don't know. Context: ${parameters.context}. \\n\\n Human: please summarize the documents \\n\\n Assistant:" + ); + MLInferenceSearchResponseProcessor responseProcessor = new MLInferenceSearchResponseProcessor( + "model1", + inputMap, + null, + modelConfig, + DEFAULT_MAX_PREDICTION_TASKS, + PROCESSOR_TAG, + DESCRIPTION, + false, + "remote", + true, + false, + false, + "{ \"parameters\": ${ml_inference.parameters} }", + client, + TEST_XCONTENT_REGISTRY_FOR_QUERY, + false + ); + + SearchRequest request = getSearchRequest(); + String fieldName = "text"; + SearchResponse response = getSearchResponse(5, true, fieldName); + Map predictionResult = ImmutableMap.of("response", "here is a summary of the documents"); + ModelTensor modelTensor = ModelTensor.builder().dataAsMap(predictionResult).build(); + ModelTensors modelTensors = ModelTensors.builder().mlModelTensors(Arrays.asList(modelTensor)).build(); + ModelTensorOutput mlModelTensorOutput = ModelTensorOutput.builder().mlModelOutputs(Arrays.asList(modelTensors)).build(); + Map fullPredictionResult = generateInferenceResult("here is a summary of the documents"); + + doAnswer(invocation -> { + ActionListener actionListener = invocation.getArgument(2); + actionListener.onResponse(MLTaskResponse.builder().output(mlModelTensorOutput).build()); + return null; + }).when(client).execute(any(), any(), any()); + + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(SearchResponse newSearchResponse) { + assertEquals(newSearchResponse.getHits().getHits().length, 5); + assertEquals(newSearchResponse.getHits().getHits()[0].getSourceAsMap().get("inference_results"), fullPredictionResult); + assertEquals(newSearchResponse.getHits().getHits()[1].getSourceAsMap().get("inference_results"), fullPredictionResult); + assertEquals(newSearchResponse.getHits().getHits()[2].getSourceAsMap().get("inference_results"), fullPredictionResult); + assertEquals(newSearchResponse.getHits().getHits()[3].getSourceAsMap().get("inference_results"), fullPredictionResult); + } + + @Override + public void onFailure(Exception e) { + throw new RuntimeException(e); + } + + }; + responseProcessor.processResponseAsync(request, response, responseContext, listener); + verify(client, times(1)).execute(any(), any(), any()); + } + /** * Tests create processor with one_to_one is true * with no mapping provided @@ -401,23 +553,23 @@ public void onResponse(SearchResponse newSearchResponse) { assertEquals(newSearchResponse.getHits().getHits().length, 5); assertEquals( newSearchResponse.getHits().getHits()[0].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[1].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[2].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[3].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[4].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); } @@ -482,23 +634,23 @@ public void onResponse(SearchResponse newSearchResponse) { assertEquals(newSearchResponse.getHits().getHits().length, 5); assertEquals( newSearchResponse.getHits().getHits()[0].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[1].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[2].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[3].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[4].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); } @@ -1893,23 +2045,23 @@ public void onResponse(SearchResponse newSearchResponse) { assertEquals(newSearchResponse.getHits().getHits().length, 5); assertEquals( newSearchResponse.getHits().getHits()[0].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[1].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[2].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[3].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[4].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); } @@ -1973,23 +2125,23 @@ public void onResponse(SearchResponse newSearchResponse) { assertEquals(newSearchResponse.getHits().getHits().length, 5); assertEquals( newSearchResponse.getHits().getHits()[0].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[1].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[2].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[3].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); assertEquals( newSearchResponse.getHits().getHits()[4].getSourceAsMap().get(DEFAULT_OUTPUT_FIELD_NAME).toString(), - "[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]" + "{inference_results=[{output=[{dataAsMap={response=[0.0, 1.0, 2.0, 3.0, 4.0]}}]}]}" ); } @@ -3054,6 +3206,27 @@ private static SearchRequest getSearchRequest() { return request; } + private static Map generateInferenceResult(String response) { + Map inferenceResult = new HashMap<>(); + List> inferenceResults = new ArrayList<>(); + + Map outputMap = new HashMap<>(); + List> outputs = new ArrayList<>(); + + Map responseOutput = new HashMap<>(); + Map dataAsMap = new HashMap<>(); + dataAsMap.put("response", response); + responseOutput.put("dataAsMap", dataAsMap); + + outputs.add(responseOutput); + outputMap.put("output", outputs); + + inferenceResults.add(outputMap); + inferenceResult.put("inference_results", inferenceResults); + + return inferenceResult; + } + /** * Helper method to create an instance of the MLInferenceSearchResponseProcessor with the specified parameters in * single pair of input and output mapping. From 091f5dfb673b79fee26315a438de6cd2fa437d49 Mon Sep 17 00:00:00 2001 From: Yaliang Wu Date: Fri, 13 Sep 2024 13:39:26 -0700 Subject: [PATCH 22/23] fix ML task index mapping (#2949) Signed-off-by: Yaliang Wu --- .../main/java/org/opensearch/ml/common/CommonValue.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/org/opensearch/ml/common/CommonValue.java b/common/src/main/java/org/opensearch/ml/common/CommonValue.java index edf76dc35e..6a48cd5081 100644 --- a/common/src/main/java/org/opensearch/ml/common/CommonValue.java +++ b/common/src/main/java/org/opensearch/ml/common/CommonValue.java @@ -391,11 +391,10 @@ public class CommonValue { + " \"" + MLTask.IS_ASYNC_TASK_FIELD + "\" : {\"type\" : \"boolean\"}, \n" - + USER_FIELD_MAPPING - + " }\n" - + "}" + + " \"" + MLTask.REMOTE_JOB_FIELD - + "\" : {\"type\": \"flat_object\"}\n" + + "\" : {\"type\": \"flat_object\"}, \n" + + USER_FIELD_MAPPING + " }\n" + "}"; From 0d2693180bbd76aab7048490ce9eb4dc5eeb3fe1 Mon Sep 17 00:00:00 2001 From: Yaliang Wu Date: Mon, 16 Sep 2024 08:33:13 -0700 Subject: [PATCH 23/23] add bedrock batch job post process function; enhance remote job status parsing (#2955) Signed-off-by: Yaliang Wu --- .../org/opensearch/ml/common/MLTaskState.java | 4 +- .../org/opensearch/ml/common/MLTaskType.java | 4 +- .../connector/MLPostProcessFunction.java | 5 + ...BedrockBatchJobArnPostProcessFunction.java | 40 +++++++ .../task/MLCancelBatchJobAction.java | 2 +- ...ockBatchJobArnPostProcessFunctionTest.java | 58 ++++++++++ .../action/tasks/GetTaskTransportAction.java | 101 +++++++++++++++--- .../ml/plugin/MachineLearningPlugin.java | 7 +- .../ml/rest/RestMLCancelBatchJobAction.java | 12 +-- .../ml/settings/MLCommonsSettings.java | 43 ++++++++ .../tasks/GetTaskTransportActionTests.java | 94 +++++++++++++++- .../rest/RestMLCancelBatchJobActionTests.java | 4 +- 12 files changed, 344 insertions(+), 30 deletions(-) create mode 100644 common/src/main/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunction.java create mode 100644 common/src/test/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunctionTest.java diff --git a/common/src/main/java/org/opensearch/ml/common/MLTaskState.java b/common/src/main/java/org/opensearch/ml/common/MLTaskState.java index 77336be901..dfd7b835d4 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLTaskState.java +++ b/common/src/main/java/org/opensearch/ml/common/MLTaskState.java @@ -28,5 +28,7 @@ public enum MLTaskState { COMPLETED, FAILED, CANCELLED, - COMPLETED_WITH_ERROR + COMPLETED_WITH_ERROR, + CANCELLING, + EXPIRED } diff --git a/common/src/main/java/org/opensearch/ml/common/MLTaskType.java b/common/src/main/java/org/opensearch/ml/common/MLTaskType.java index 179bf152cd..aafff5b50e 100644 --- a/common/src/main/java/org/opensearch/ml/common/MLTaskType.java +++ b/common/src/main/java/org/opensearch/ml/common/MLTaskType.java @@ -8,7 +8,6 @@ public enum MLTaskType { TRAINING, PREDICTION, - BATCH_PREDICTION, TRAINING_AND_PREDICTION, EXECUTION, @Deprecated @@ -17,5 +16,6 @@ public enum MLTaskType { LOAD_MODEL, REGISTER_MODEL, DEPLOY_MODEL, - BATCH_INGEST + BATCH_INGEST, + BATCH_PREDICTION } diff --git a/common/src/main/java/org/opensearch/ml/common/connector/MLPostProcessFunction.java b/common/src/main/java/org/opensearch/ml/common/connector/MLPostProcessFunction.java index 5ba465b15a..abe56cde0e 100644 --- a/common/src/main/java/org/opensearch/ml/common/connector/MLPostProcessFunction.java +++ b/common/src/main/java/org/opensearch/ml/common/connector/MLPostProcessFunction.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.function.Function; +import org.opensearch.ml.common.connector.functions.postprocess.BedrockBatchJobArnPostProcessFunction; import org.opensearch.ml.common.connector.functions.postprocess.BedrockEmbeddingPostProcessFunction; import org.opensearch.ml.common.connector.functions.postprocess.CohereRerankPostProcessFunction; import org.opensearch.ml.common.connector.functions.postprocess.EmbeddingPostProcessFunction; @@ -20,6 +21,7 @@ public class MLPostProcessFunction { public static final String COHERE_EMBEDDING = "connector.post_process.cohere.embedding"; public static final String OPENAI_EMBEDDING = "connector.post_process.openai.embedding"; public static final String BEDROCK_EMBEDDING = "connector.post_process.bedrock.embedding"; + public static final String BEDROCK_BATCH_JOB_ARN = "connector.post_process.bedrock.batch_job_arn"; public static final String COHERE_RERANK = "connector.post_process.cohere.rerank"; public static final String DEFAULT_EMBEDDING = "connector.post_process.default.embedding"; public static final String DEFAULT_RERANK = "connector.post_process.default.rerank"; @@ -31,17 +33,20 @@ public class MLPostProcessFunction { static { EmbeddingPostProcessFunction embeddingPostProcessFunction = new EmbeddingPostProcessFunction(); BedrockEmbeddingPostProcessFunction bedrockEmbeddingPostProcessFunction = new BedrockEmbeddingPostProcessFunction(); + BedrockBatchJobArnPostProcessFunction batchJobArnPostProcessFunction = new BedrockBatchJobArnPostProcessFunction(); CohereRerankPostProcessFunction cohereRerankPostProcessFunction = new CohereRerankPostProcessFunction(); JSON_PATH_EXPRESSION.put(OPENAI_EMBEDDING, "$.data[*].embedding"); JSON_PATH_EXPRESSION.put(COHERE_EMBEDDING, "$.embeddings"); JSON_PATH_EXPRESSION.put(DEFAULT_EMBEDDING, "$[*]"); JSON_PATH_EXPRESSION.put(BEDROCK_EMBEDDING, "$.embedding"); + JSON_PATH_EXPRESSION.put(BEDROCK_BATCH_JOB_ARN, "$"); JSON_PATH_EXPRESSION.put(COHERE_RERANK, "$.results"); JSON_PATH_EXPRESSION.put(DEFAULT_RERANK, "$[*]"); POST_PROCESS_FUNCTIONS.put(OPENAI_EMBEDDING, embeddingPostProcessFunction); POST_PROCESS_FUNCTIONS.put(COHERE_EMBEDDING, embeddingPostProcessFunction); POST_PROCESS_FUNCTIONS.put(DEFAULT_EMBEDDING, embeddingPostProcessFunction); POST_PROCESS_FUNCTIONS.put(BEDROCK_EMBEDDING, bedrockEmbeddingPostProcessFunction); + POST_PROCESS_FUNCTIONS.put(BEDROCK_BATCH_JOB_ARN, batchJobArnPostProcessFunction); POST_PROCESS_FUNCTIONS.put(COHERE_RERANK, cohereRerankPostProcessFunction); POST_PROCESS_FUNCTIONS.put(DEFAULT_RERANK, cohereRerankPostProcessFunction); } diff --git a/common/src/main/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunction.java b/common/src/main/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunction.java new file mode 100644 index 0000000000..e69829855e --- /dev/null +++ b/common/src/main/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunction.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.connector.functions.postprocess; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.opensearch.ml.common.output.model.ModelTensor; + +public class BedrockBatchJobArnPostProcessFunction extends ConnectorPostProcessFunction> { + public static final String JOB_ARN = "jobArn"; + public static final String PROCESSED_JOB_ARN = "processedJobArn"; + + @Override + public void validate(Object input) { + if (!(input instanceof Map)) { + throw new IllegalArgumentException("Post process function input is not a Map."); + } + Map jobInfo = (Map) input; + if (!(jobInfo.containsKey(JOB_ARN))) { + throw new IllegalArgumentException("job arn is missing."); + } + } + + @Override + public List process(Map jobInfo) { + List modelTensors = new ArrayList<>(); + Map processedResult = new HashMap<>(); + processedResult.putAll(jobInfo); + String jobArn = jobInfo.get(JOB_ARN); + processedResult.put(PROCESSED_JOB_ARN, jobArn.replace("/", "%2F")); + modelTensors.add(ModelTensor.builder().name("response").dataAsMap(processedResult).build()); + return modelTensors; + } +} diff --git a/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java index 6ea26c9eb3..5c75e4c8d2 100644 --- a/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java +++ b/common/src/main/java/org/opensearch/ml/common/transport/task/MLCancelBatchJobAction.java @@ -9,7 +9,7 @@ public class MLCancelBatchJobAction extends ActionType { public static final MLCancelBatchJobAction INSTANCE = new MLCancelBatchJobAction(); - public static final String NAME = "cluster:admin/opensearch/ml/tasks/cancel_batch_job"; + public static final String NAME = "cluster:admin/opensearch/ml/tasks/cancel"; private MLCancelBatchJobAction() { super(NAME, MLCancelBatchJobResponse::new); diff --git a/common/src/test/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunctionTest.java b/common/src/test/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunctionTest.java new file mode 100644 index 0000000000..05b5d490cd --- /dev/null +++ b/common/src/test/java/org/opensearch/ml/common/connector/functions/postprocess/BedrockBatchJobArnPostProcessFunctionTest.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.ml.common.connector.functions.postprocess; + +import static org.junit.Assert.assertEquals; +import static org.opensearch.ml.common.connector.functions.postprocess.BedrockBatchJobArnPostProcessFunction.JOB_ARN; +import static org.opensearch.ml.common.connector.functions.postprocess.BedrockBatchJobArnPostProcessFunction.PROCESSED_JOB_ARN; + +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.opensearch.ml.common.output.model.ModelTensor; + +public class BedrockBatchJobArnPostProcessFunctionTest { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + BedrockBatchJobArnPostProcessFunction function; + + @Before + public void setUp() { + function = new BedrockBatchJobArnPostProcessFunction(); + } + + @Test + public void process_WrongInput_NotMap() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("Post process function input is not a Map."); + function.apply("abc"); + } + + @Test + public void process_WrongInput_NotContainJobArn() { + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("job arn is missing."); + function.apply(Map.of("test", "value")); + } + + @Test + public void process_CorrectInput() { + String jobArn = "arn:aws:bedrock:us-east-1:12345678912:model-invocation-job/w1xtlm0ik3e1"; + List result = function.apply(Map.of(JOB_ARN, jobArn)); + assertEquals(1, result.size()); + assertEquals(jobArn, result.get(0).getDataAsMap().get(JOB_ARN)); + assertEquals( + "arn:aws:bedrock:us-east-1:12345678912:model-invocation-job%2Fw1xtlm0ik3e1", + result.get(0).getDataAsMap().get(PROCESSED_JOB_ARN) + ); + } +} diff --git a/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java index 01b4724046..28fbffb3f8 100644 --- a/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java +++ b/plugin/src/main/java/org/opensearch/ml/action/tasks/GetTaskTransportAction.java @@ -11,14 +11,25 @@ import static org.opensearch.ml.common.MLTask.REMOTE_JOB_FIELD; import static org.opensearch.ml.common.MLTask.STATE_FIELD; import static org.opensearch.ml.common.MLTaskState.CANCELLED; +import static org.opensearch.ml.common.MLTaskState.CANCELLING; import static org.opensearch.ml.common.MLTaskState.COMPLETED; +import static org.opensearch.ml.common.MLTaskState.EXPIRED; import static org.opensearch.ml.common.connector.ConnectorAction.ActionType.BATCH_PREDICT_STATUS; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_FIELD; import static org.opensearch.ml.utils.MLExceptionUtils.logException; import static org.opensearch.ml.utils.MLNodeUtils.createXContentParserFromRegistry; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchStatusException; @@ -30,6 +41,8 @@ import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; @@ -80,6 +93,12 @@ public class GetTaskTransportAction extends HandledTransportAction remoteJobStatusFields; + volatile Pattern remoteJobCompletedStatusRegexPattern; + volatile Pattern remoteJobCancelledStatusRegexPattern; + volatile Pattern remoteJobCancellingStatusRegexPattern; + volatile Pattern remoteJobExpiredStatusRegexPattern; + @Inject public GetTaskTransportAction( TransportService transportService, @@ -91,7 +110,8 @@ public GetTaskTransportAction( ConnectorAccessControlHelper connectorAccessControlHelper, EncryptorImpl encryptor, MLTaskManager mlTaskManager, - MLModelManager mlModelManager + MLModelManager mlModelManager, + Settings settings ) { super(MLTaskGetAction.NAME, transportService, actionFilters, MLTaskGetRequest::new); this.client = client; @@ -102,6 +122,44 @@ public GetTaskTransportAction( this.encryptor = encryptor; this.mlTaskManager = mlTaskManager; this.mlModelManager = mlModelManager; + + remoteJobStatusFields = ML_COMMONS_REMOTE_JOB_STATUS_FIELD.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ML_COMMONS_REMOTE_JOB_STATUS_FIELD, it -> remoteJobStatusFields = it); + initializeRegexPattern( + ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX, + settings, + clusterService, + (regex) -> remoteJobCompletedStatusRegexPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE) + ); + initializeRegexPattern( + ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX, + settings, + clusterService, + (regex) -> remoteJobCancelledStatusRegexPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE) + ); + initializeRegexPattern( + ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX, + settings, + clusterService, + (regex) -> remoteJobCancellingStatusRegexPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE) + ); + initializeRegexPattern( + ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX, + settings, + clusterService, + (regex) -> remoteJobExpiredStatusRegexPattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE) + ); + } + + private void initializeRegexPattern( + Setting setting, + Settings settings, + ClusterService clusterService, + Consumer patternInitializer + ) { + String regex = setting.get(settings); + patternInitializer.accept(regex); + clusterService.getClusterSettings().addSettingsUpdateConsumer(setting, it -> patternInitializer.accept(it)); } @Override @@ -210,7 +268,7 @@ private void executeConnector( MLInput mlInput, String taskId, MLTask mlTask, - Map transformJob, + Map remoteJob, ActionListener actionListener ) { if (connectorAccessControlHelper.validateConnectorAccess(client, connector)) { @@ -222,7 +280,7 @@ private void executeConnector( connectorExecutor.setClient(client); connectorExecutor.setXContentRegistry(xContentRegistry); connectorExecutor.executeAction(BATCH_PREDICT_STATUS.name(), mlInput, ActionListener.wrap(taskResponse -> { - processTaskResponse(mlTask, taskId, taskResponse, transformJob, actionListener); + processTaskResponse(mlTask, taskId, taskResponse, remoteJob, actionListener); }, e -> { actionListener.onFailure(e); })); } else { actionListener @@ -230,7 +288,7 @@ private void executeConnector( } } - private void processTaskResponse( + protected void processTaskResponse( MLTask mlTask, String taskId, MLTaskResponse taskResponse, @@ -248,15 +306,11 @@ private void processTaskResponse( Map updatedTask = new HashMap<>(); updatedTask.put(REMOTE_JOB_FIELD, remoteJob); - if ((remoteJob.containsKey("status") && remoteJob.get("status").equals("completed")) - || (remoteJob.containsKey("TransformJobStatus") && remoteJob.get("TransformJobStatus").equals("Completed"))) { - updatedTask.put(STATE_FIELD, COMPLETED); - mlTask.setState(COMPLETED); - - } else if ((remoteJob.containsKey("status") && remoteJob.get("status").equals("cancelled")) - || (remoteJob.containsKey("TransformJobStatus") && remoteJob.get("TransformJobStatus").equals("Stopped"))) { - updatedTask.put(STATE_FIELD, CANCELLED); - mlTask.setState(CANCELLED); + for (String statusField : remoteJobStatusFields) { + String statusValue = String.valueOf(remoteJob.get(statusField)); + if (remoteJob.containsKey(statusField)) { + updateTaskState(updatedTask, mlTask, statusValue); + } } mlTaskManager.updateMLTaskDirectly(taskId, updatedTask, ActionListener.wrap(response -> { actionListener.onResponse(MLTaskGetResponse.builder().mlTask(mlTask).build()); @@ -280,4 +334,25 @@ private void processTaskResponse( log.error("Unable to fetch status for ml task ", e); } } + + private void updateTaskState(Map updatedTask, MLTask mlTask, String statusValue) { + if (matchesPattern(remoteJobCancellingStatusRegexPattern, statusValue)) { + updatedTask.put(STATE_FIELD, CANCELLING); + mlTask.setState(CANCELLING); + } else if (matchesPattern(remoteJobCancelledStatusRegexPattern, statusValue)) { + updatedTask.put(STATE_FIELD, CANCELLED); + mlTask.setState(CANCELLED); + } else if (matchesPattern(remoteJobCompletedStatusRegexPattern, statusValue)) { + updatedTask.put(STATE_FIELD, COMPLETED); + mlTask.setState(COMPLETED); + } else if (matchesPattern(remoteJobExpiredStatusRegexPattern, statusValue)) { + updatedTask.put(STATE_FIELD, EXPIRED); + mlTask.setState(EXPIRED); + } + } + + private boolean matchesPattern(Pattern pattern, String input) { + Matcher matcher = pattern.matcher(input); + return matcher.find(); + } } diff --git a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java index ed4d595897..39aaf05ff9 100644 --- a/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java +++ b/plugin/src/main/java/org/opensearch/ml/plugin/MachineLearningPlugin.java @@ -964,7 +964,12 @@ public List> getSettings() { MLCommonsSettings.ML_COMMONS_RAG_PIPELINE_FEATURE_ENABLED, MLCommonsSettings.ML_COMMONS_AGENT_FRAMEWORK_ENABLED, MLCommonsSettings.ML_COMMONS_MODEL_AUTO_DEPLOY_ENABLE, - MLCommonsSettings.ML_COMMONS_CONNECTOR_PRIVATE_IP_ENABLED + MLCommonsSettings.ML_COMMONS_CONNECTOR_PRIVATE_IP_ENABLED, + MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_FIELD, + MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX, + MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX, + MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX, + MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX ); return settings; } diff --git a/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java b/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java index 33c7314be2..49d2247122 100644 --- a/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java +++ b/plugin/src/main/java/org/opensearch/ml/rest/RestMLCancelBatchJobAction.java @@ -23,8 +23,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +//TODO: Rename class and support cancelling more tasks. Now only support cancelling remote job public class RestMLCancelBatchJobAction extends BaseRestHandler { - private static final String ML_CANCEL_BATCH_ACTION = "ml_cancel_batch_action"; + private static final String ML_CANCEL_TASK_ACTION = "ml_cancel_task_action"; /** * Constructor @@ -33,18 +34,13 @@ public RestMLCancelBatchJobAction() {} @Override public String getName() { - return ML_CANCEL_BATCH_ACTION; + return ML_CANCEL_TASK_ACTION; } @Override public List routes() { return ImmutableList - .of( - new Route( - RestRequest.Method.POST, - String.format(Locale.ROOT, "%s/tasks/{%s}/_cancel_batch", ML_BASE_URI, PARAMETER_TASK_ID) - ) - ); + .of(new Route(RestRequest.Method.POST, String.format(Locale.ROOT, "%s/tasks/{%s}/_cancel", ML_BASE_URI, PARAMETER_TASK_ID))); } @Override diff --git a/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java b/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java index 339116226d..b9d7f2a9fc 100644 --- a/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java +++ b/plugin/src/main/java/org/opensearch/ml/settings/MLCommonsSettings.java @@ -200,4 +200,47 @@ private MLCommonsSettings() {} public static final Setting ML_COMMONS_CONNECTOR_PRIVATE_IP_ENABLED = Setting .boolSetting("plugins.ml_commons.connector.private_ip_enabled", false, Setting.Property.NodeScope, Setting.Property.Dynamic); + + public static final Setting> ML_COMMONS_REMOTE_JOB_STATUS_FIELD = Setting + .listSetting( + "plugins.ml_commons.remote_job.status_field", + ImmutableList + .of( + "status", // openai, bedrock, cohere + "Status", + "TransformJobStatus" // sagemaker + ), + Function.identity(), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + public static final Setting ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX = Setting + .simpleString( + "plugins.ml_commons.remote_job.status_regex.completed", + "(complete|completed)", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX = Setting + .simpleString( + "plugins.ml_commons.remote_job.status_regex.cancelled", + "(stopped|cancelled)", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX = Setting + .simpleString( + "plugins.ml_commons.remote_job.status_regex.cancelling", + "(stopping|cancelling)", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + public static final Setting ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX = Setting + .simpleString( + "plugins.ml_commons.remote_job.status_regex.expired", + "(expired|timeout)", + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); } diff --git a/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java b/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java index 3707c89eae..1c9a1c449a 100644 --- a/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/action/tasks/GetTaskTransportActionTests.java @@ -15,12 +15,18 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX; +import static org.opensearch.ml.settings.MLCommonsSettings.ML_COMMONS_REMOTE_JOB_STATUS_FIELD; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.junit.Before; import org.junit.Ignore; @@ -36,6 +42,7 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; @@ -49,13 +56,16 @@ import org.opensearch.ml.common.FunctionName; import org.opensearch.ml.common.MLModel; import org.opensearch.ml.common.MLTask; +import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.MLTaskType; import org.opensearch.ml.common.connector.Connector; import org.opensearch.ml.common.connector.ConnectorAction; import org.opensearch.ml.common.connector.HttpConnector; +import org.opensearch.ml.common.dataset.MLInputDataType; import org.opensearch.ml.common.output.model.ModelTensor; import org.opensearch.ml.common.output.model.ModelTensorOutput; import org.opensearch.ml.common.output.model.ModelTensors; +import org.opensearch.ml.common.transport.MLTaskResponse; import org.opensearch.ml.common.transport.task.MLTaskGetRequest; import org.opensearch.ml.common.transport.task.MLTaskGetResponse; import org.opensearch.ml.engine.encryptor.EncryptorImpl; @@ -118,7 +128,14 @@ public void setup() throws IOException { MockitoAnnotations.openMocks(this); mlTaskGetRequest = MLTaskGetRequest.builder().taskId("test_id").build(); - Settings settings = Settings.builder().build(); + Settings settings = Settings + .builder() + .putList(ML_COMMONS_REMOTE_JOB_STATUS_FIELD.getKey(), List.of("status", "TransformJobStatus")) + .put(ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX.getKey(), "(complete|completed)") + .put(ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX.getKey(), "(stopped|cancelled)") + .put(ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX.getKey(), "(stopping|cancelling)") + .put(ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX.getKey(), "(expired|timeout)") + .build(); threadContext = new ThreadContext(settings); when(client.threadPool()).thenReturn(threadPool); when(threadPool.getThreadContext()).thenReturn(threadContext); @@ -127,6 +144,21 @@ public void setup() throws IOException { doReturn(metaData).when(clusterState).metadata(); doReturn(true).when(metaData).hasIndex(anyString()); + when(clusterService.getSettings()).thenReturn(settings); + when(this.clusterService.getClusterSettings()) + .thenReturn( + new ClusterSettings( + settings, + Set + .of( + ML_COMMONS_REMOTE_JOB_STATUS_FIELD, + ML_COMMONS_REMOTE_JOB_STATUS_COMPLETED_REGEX, + ML_COMMONS_REMOTE_JOB_STATUS_CANCELLED_REGEX, + ML_COMMONS_REMOTE_JOB_STATUS_CANCELLING_REGEX, + ML_COMMONS_REMOTE_JOB_STATUS_EXPIRED_REGEX + ) + ) + ); getTaskTransportAction = spy( new GetTaskTransportAction( @@ -139,7 +171,8 @@ public void setup() throws IOException { connectorAccessControlHelper, encryptor, mlTaskManager, - mlModelManager + mlModelManager, + settings ) ); @@ -331,4 +364,61 @@ public GetResponse prepareMLTask(FunctionName functionName, MLTaskType mlTaskTyp GetResponse getResponse = new GetResponse(getResult); return getResponse; } + + public void test_processTaskResponse_complete() { + processTaskResponse("TransformJobStatus", "complete", MLTaskState.COMPLETED); + } + + public void test_processTaskResponse_cancelling() { + processTaskResponse("status", "cancelling", MLTaskState.CANCELLING); + } + + public void test_processTaskResponse_cancelled() { + processTaskResponse("status", "cancelled", MLTaskState.CANCELLED); + } + + public void test_processTaskResponse_expired() { + processTaskResponse("status", "expired", MLTaskState.EXPIRED); + } + + public void test_processTaskResponse_WrongStatusField() { + processTaskResponse("wrong_status_field", "expired", null); + } + + public void test_processTaskResponse_UnknownStatusField() { + processTaskResponse("status", "unkown_status", null); + } + + private void processTaskResponse(String statusField, String remoteJobResponseStatus, MLTaskState taskState) { + String taskId = "testTaskId"; + String remoteJobName = randomAlphaOfLength(5); + Map remoteJob = new HashMap(); + remoteJob.put(statusField, "running"); + remoteJob.put("name", remoteJobName); + MLTask mlTask = MLTask + .builder() + .taskId(taskId) + .taskType(MLTaskType.BATCH_PREDICTION) + .inputType(MLInputDataType.REMOTE) + .state(MLTaskState.RUNNING) + .remoteJob(remoteJob) + .build(); + ModelTensor modelTensor = ModelTensor.builder().name("response").dataAsMap(Map.of(statusField, remoteJobResponseStatus)).build(); + ModelTensorOutput modelTensorOutput = ModelTensorOutput + .builder() + .mlModelOutputs(List.of(ModelTensors.builder().mlModelTensors(List.of(modelTensor)).build())) + .build(); + MLTaskResponse taskResponse = MLTaskResponse.builder().output(modelTensorOutput).build(); + ActionListener actionListener = mock(ActionListener.class); + ArgumentCaptor> updatedTaskCaptor = ArgumentCaptor.forClass(Map.class); + + getTaskTransportAction.processTaskResponse(mlTask, taskId, taskResponse, mlTask.getRemoteJob(), actionListener); + + verify(mlTaskManager).updateMLTaskDirectly(any(), updatedTaskCaptor.capture(), any()); + Map updatedTask = updatedTaskCaptor.getValue(); + assertEquals(taskState, updatedTask.get("state")); + Map updatedRemoteJob = (Map) updatedTask.get("remote_job"); + assertEquals(remoteJobResponseStatus, updatedRemoteJob.get(statusField)); + assertEquals(remoteJobName, updatedRemoteJob.get("name")); + } } diff --git a/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java b/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java index 1498750e6a..bd1d321fef 100644 --- a/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java +++ b/plugin/src/test/java/org/opensearch/ml/rest/RestMLCancelBatchJobActionTests.java @@ -74,7 +74,7 @@ public void testConstructor() { public void testGetName() { String actionName = restMLCancelBatchJobAction.getName(); assertFalse(Strings.isNullOrEmpty(actionName)); - assertEquals("ml_cancel_batch_action", actionName); + assertEquals("ml_cancel_task_action", actionName); } public void testRoutes() { @@ -83,7 +83,7 @@ public void testRoutes() { assertFalse(routes.isEmpty()); RestHandler.Route route = routes.get(0); assertEquals(RestRequest.Method.POST, route.getMethod()); - assertEquals("/_plugins/_ml/tasks/{task_id}/_cancel_batch", route.getPath()); + assertEquals("/_plugins/_ml/tasks/{task_id}/_cancel", route.getPath()); } public void test_PrepareRequest() throws Exception {