Add API Consistency Tests with ML-Common and Set Up Daily GitHub Acti…

…on Trigger (#937)

* Added the rest api spec comparison test

Signed-off-by: Junwei Dai <[email protected]>

* Added github Action to run api consistency test daily

Signed-off-by: Junwei Dai <[email protected]>

* fix formating issue

Signed-off-by: Junwei Dai <[email protected]>

* add change log

Signed-off-by: Junwei Dai <[email protected]>

* fix *import

Signed-off-by: Junwei Dai <[email protected]>

* addrssed all comment

Signed-off-by: Junwei Dai <[email protected]>

---------

Signed-off-by: Junwei Dai <[email protected]>
Co-authored-by: Junwei Dai <[email protected]>

.github/workflows/test-api-consistency.yml

@@ -0,0 +1,26 @@
+name: Daily API Consistency Test
+
+on:
+  schedule:
+    - cron: '0 8 * * *'  # Runs daily at 8 AM UTC
+  workflow_dispatch:
+
+jobs:
+  API-consistency-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21]
+
+    steps:
+      - name: Checkout Flow Framework
+        uses: actions/checkout@v3
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run API Consistency Tests
+        run: ./gradlew test --tests "org.opensearch.flowframework.workflow.*"

CHANGELOG.md

@@ -21,6 +21,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 ### Features
 - Add ApiSpecFetcher for Fetching and Comparing API Specifications ([#651](https://github.com/opensearch-project/flow-framework/issues/651))
 - Add optional config field to tool step ([#899](https://github.com/opensearch-project/flow-framework/pull/899))
+- Add API Consistency Tests with ML-Common and Set Up Daily GitHub Action Trigger([#908](https://github.com/opensearch-project/flow-framework/issues/908))
 
 ### Enhancements
 - Incrementally remove resources from workflow state during deprovisioning ([#898](https://github.com/opensearch-project/flow-framework/pull/898))

src/main/java/org/opensearch/flowframework/util/ApiSpecFetcher.java

@@ -14,7 +14,6 @@
 import org.opensearch.flowframework.exception.ApiSpecParseException;
 import org.opensearch.rest.RestRequest;
 
-import java.util.HashSet;
 import java.util.List;
 
 import io.swagger.v3.oas.models.OpenAPI;
@@ -81,7 +80,7 @@ public static boolean compareRequiredFields(List<String> requiredEnumParams, Str
 
             List<String> requiredApiParams = schema.getRequired();
             if (requiredApiParams != null && !requiredApiParams.isEmpty()) {
-                return new HashSet<>(requiredEnumParams).equals(new HashSet<>(requiredApiParams));
+                return requiredApiParams.stream().allMatch(requiredEnumParams::contains);
             }
         }
         return false;

src/test/java/org/opensearch/flowframework/workflow/CreateConnectorStepTests.java

@@ -14,20 +14,24 @@
 import org.opensearch.flowframework.common.CommonValue;
 import org.opensearch.flowframework.exception.FlowFrameworkException;
 import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
+import org.opensearch.flowframework.util.ApiSpecFetcher;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.ml.common.connector.ConnectorAction;
 import org.opensearch.ml.common.transport.connector.MLCreateConnectorInput;
 import org.opensearch.ml.common.transport.connector.MLCreateConnectorResponse;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
+import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
 import static org.opensearch.flowframework.common.WorkflowResources.CONNECTOR_ID;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -134,4 +138,17 @@ public void testCreateConnectorFailure() throws IOException {
         assertEquals("Failed to create connector", ex.getCause().getMessage());
     }
 
+    public void testApiSpecCreateConnectorInputParamComparison() throws Exception {
+        List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.CREATE_CONNECTOR.inputs();
+
+        boolean isMatch = ApiSpecFetcher.compareRequiredFields(
+            requiredEnumParams,
+            ML_COMMONS_API_SPEC_YAML_URI,
+            "/_plugins/_ml/connectors/_create",
+            RestRequest.Method.POST
+        );
+
+        assertTrue(isMatch);
+    }
+
 }

src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java

@@ -20,11 +20,13 @@
 import org.opensearch.flowframework.exception.FlowFrameworkException;
 import org.opensearch.flowframework.exception.WorkflowStepException;
 import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
+import org.opensearch.flowframework.util.ApiSpecFetcher;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.ml.common.MLTask;
 import org.opensearch.ml.common.MLTaskState;
 import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
 import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.ScalingExecutorBuilder;
 import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +35,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -43,6 +46,7 @@
 
 import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
 import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
+import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
 import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
 import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
 import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
@@ -398,4 +402,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
         assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
         assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
     }
+
+    public void testApiSpecRegisterLocalCustomModelInputParamComparison() throws Exception {
+        List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_CUSTOM_MODEL.inputs();
+
+        boolean isMatch = ApiSpecFetcher.compareRequiredFields(
+            requiredEnumParams,
+            ML_COMMONS_API_SPEC_YAML_URI,
+            "/_plugins/_ml/models/_register",
+            RestRequest.Method.POST
+        );
+
+        assertTrue(isMatch);
+    }
 }

src/test/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java

@@ -20,11 +20,13 @@
 import org.opensearch.flowframework.exception.FlowFrameworkException;
 import org.opensearch.flowframework.exception.WorkflowStepException;
 import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
+import org.opensearch.flowframework.util.ApiSpecFetcher;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.ml.common.MLTask;
 import org.opensearch.ml.common.MLTaskState;
 import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
 import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.ScalingExecutorBuilder;
 import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +35,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -42,6 +45,7 @@
 
 import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
 import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
+import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
 import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
 import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
 import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
@@ -303,4 +307,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
         assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
         assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
     }
+
+    public void testApiSpecRegisterLocalPretrainedModelInputParamComparison() throws Exception {
+        List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_PRETRAINED_MODEL.inputs();
+
+        boolean isMatch = ApiSpecFetcher.compareRequiredFields(
+            requiredEnumParams,
+            ML_COMMONS_API_SPEC_YAML_URI,
+            "/_plugins/_ml/models/_register",
+            RestRequest.Method.POST
+        );
+
+        assertTrue(isMatch);
+    }
 }

src/test/java/org/opensearch/flowframework/workflow/RegisterLocalSparseEncodingModelStepTests.java

@@ -20,11 +20,13 @@
 import org.opensearch.flowframework.exception.FlowFrameworkException;
 import org.opensearch.flowframework.exception.WorkflowStepException;
 import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
+import org.opensearch.flowframework.util.ApiSpecFetcher;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.ml.common.MLTask;
 import org.opensearch.ml.common.MLTaskState;
 import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
 import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.ScalingExecutorBuilder;
 import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +35,7 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -42,6 +45,7 @@
 
 import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
 import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
+import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
 import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
 import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
 import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
@@ -310,4 +314,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
         assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
         assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
     }
+
+    public void testApiSpecRegisterLocalSparseEncodingModelInputParamComparison() throws Exception {
+        List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_SPARSE_ENCODING_MODEL.inputs();
+
+        boolean isMatch = ApiSpecFetcher.compareRequiredFields(
+            requiredEnumParams,
+            ML_COMMONS_API_SPEC_YAML_URI,
+            "/_plugins/_ml/models/_register",
+            RestRequest.Method.POST
+        );
+
+        assertTrue(isMatch);
+    }
 }

src/test/java/org/opensearch/flowframework/workflow/RegisterModelGroupStepTests.java

@@ -14,11 +14,13 @@
 import org.opensearch.flowframework.exception.FlowFrameworkException;
 import org.opensearch.flowframework.exception.WorkflowStepException;
 import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
+import org.opensearch.flowframework.util.ApiSpecFetcher;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.ml.common.AccessMode;
 import org.opensearch.ml.common.MLTaskState;
 import org.opensearch.ml.common.transport.model_group.MLRegisterModelGroupInput;
 import org.opensearch.ml.common.transport.model_group.MLRegisterModelGroupResponse;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
@@ -31,6 +33,7 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
+import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
 import static org.opensearch.flowframework.common.CommonValue.MODEL_GROUP_STATUS;
 import static org.opensearch.flowframework.common.WorkflowResources.MODEL_GROUP_ID;
 import static org.mockito.ArgumentMatchers.any;
@@ -204,4 +207,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
         assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
         assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
     }
+
+    public void testApiSpecRegisterModelGroupInputParamComparison() throws Exception {
+        List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_MODEL_GROUP.inputs();
+
+        boolean isMatch = ApiSpecFetcher.compareRequiredFields(
+            requiredEnumParams,
+            ML_COMMONS_API_SPEC_YAML_URI,
+            "/_plugins/_ml/model_groups/_register",
+            RestRequest.Method.POST
+        );
+
+        assertTrue(isMatch);
+    }
 }

src/test/java/org/opensearch/flowframework/workflow/RegisterRemoteModelStepTests.java

@@ -17,15 +17,18 @@
 import org.opensearch.flowframework.exception.FlowFrameworkException;
 import org.opensearch.flowframework.exception.WorkflowStepException;
 import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
+import org.opensearch.flowframework.util.ApiSpecFetcher;
 import org.opensearch.ml.client.MachineLearningNodeClient;
 import org.opensearch.ml.common.MLTaskState;
 import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
 import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
+import org.opensearch.rest.RestRequest;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.transport.RemoteTransportException;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -35,6 +38,7 @@
 
 import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
 import static org.opensearch.flowframework.common.CommonValue.INTERFACE_FIELD;
+import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
 import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
 import static org.opensearch.flowframework.common.WorkflowResources.CONNECTOR_ID;
 import static org.opensearch.flowframework.common.WorkflowResources.MODEL_ID;
@@ -416,4 +420,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
         assertEquals("Failed to parse value [yes] as only [true] or [false] are allowed.", w.getMessage());
         assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
     }
+
+    public void testApiSpecRegisterRemoteModelInputParamComparison() throws Exception {
+        List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_REMOTE_MODEL.inputs();
+
+        boolean isMatch = ApiSpecFetcher.compareRequiredFields(
+            requiredEnumParams,
+            ML_COMMONS_API_SPEC_YAML_URI,
+            "/_plugins/_ml/model_groups/_register",
+            RestRequest.Method.POST
+        );
+
+        assertTrue(isMatch);
+    }
 }