Skip to content

Commit

Permalink
change neural sparse model to pretrained tokenizer
Browse files Browse the repository at this point in the history
Signed-off-by: zhichao-aws <[email protected]>
  • Loading branch information
zhichao-aws committed Dec 29, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 3ab9388 commit 4593afe
Showing 3 changed files with 8 additions and 12 deletions.
Original file line number Diff line number Diff line change
@@ -114,7 +114,7 @@ protected Response waitTaskComplete(String taskId) {
if (state.equals(MLTaskState.FAILED.toString())
|| state.equals(MLTaskState.CANCELLED.toString())
|| state.equals(MLTaskState.COMPLETED_WITH_ERROR.toString())) {
break;
fail("The task failed with state " + state);
}
Thread.sleep(DEFAULT_TASK_RESULT_QUERY_INTERVAL_IN_MILLISECOND);
}
Original file line number Diff line number Diff line change
@@ -96,8 +96,8 @@ public void testNeuralSparseSearchToolInFlowAgent() {
String result = executeAgent(agentId, "{\"parameters\": {\"question\": \"a\"}}");
assertEquals(
"The agent execute response not equal with expected.",
"{\"_index\":\"test_index\",\"_source\":{\"text\":\"text doc 3\"},\"_id\":\"2\",\"_score\":6.0}\n"
+ "{\"_index\":\"test_index\",\"_source\":{\"text\":\"text doc 2\"},\"_id\":\"1\",\"_score\":3.0}\n",
"{\"_index\":\"test_index\",\"_source\":{\"text\":\"text doc 3\"},\"_id\":\"2\",\"_score\":2.4136734}\n"
+ "{\"_index\":\"test_index\",\"_source\":{\"text\":\"text doc 2\"},\"_id\":\"1\",\"_score\":1.2068367}\n",
result
);

@@ -120,8 +120,8 @@ public void testNeuralSparseSearchToolInFlowAgent_withIllegalSourceField_thenGet
String result = executeAgent(agentId, "{\"parameters\": {\"question\": \"a\"}}");
assertEquals(
"The agent execute response not equal with expected.",
"{\"_index\":\"test_index\",\"_source\":{},\"_id\":\"2\",\"_score\":6.0}\n"
+ "{\"_index\":\"test_index\",\"_source\":{},\"_id\":\"1\",\"_score\":3.0}\n",
"{\"_index\":\"test_index\",\"_source\":{},\"_id\":\"2\",\"_score\":2.4136734}\n"
+ "{\"_index\":\"test_index\",\"_source\":{},\"_id\":\"1\",\"_score\":1.2068367}\n",
result
);
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
{
"name": "tokenize-idf-0915",
"version": "1.0.0",
"function_name": "SPARSE_TOKENIZE",
"description": "test model",
"model_format": "TORCH_SCRIPT",
"model_content_hash_value": "b345e9e943b62c405a8dd227ef2c46c84c5ff0a0b71b584be9132b37bce91a9a",
"url": "https://github.com/opensearch-project/ml-commons/raw/main/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/sparse_encoding/sparse_demo.zip"
"name":"amazon/neural-sparse/opensearch-neural-sparse-tokenizer-v1",
"version":"1.0.1",
"model_format": "TORCH_SCRIPT"
}

0 comments on commit 4593afe

Please sign in to comment.