Merge branch 'feature/trt_llm_lora' of https://github.com/pytorch/serve…

… into feature/trt_llm_lora

.github/workflows/ci_graviton_cpu.yml

@@ -0,0 +1,48 @@
+name: CI CPU Graviton
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  merge_group:
+
+
+concurrency:
+  group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ci-cpu:
+    runs-on: [self-hosted, graviton-test]
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          architecture: arm64
+      - name: Setup Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: Checkout TorchServe
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          python ts_scripts/install_dependencies.py --environment=dev
+      - name: Torchserve Sanity
+        uses: nick-fields/retry@v3
+        env:
+          TS_MAC_ARM64_CPU_ONLY: 'True'
+        with:
+          timeout_minutes: 60
+          max_attempts: 3
+          retry_on: error
+          command: |
+            python torchserve_sanity.py

.github/workflows/regression_tests_graviton_cpu.yml

@@ -0,0 +1,41 @@
+name: Run Regression Tests on CPU for Graviton
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+  merge_group:
+
+concurrency:
+  group: ci-cpu-${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  regression-cpu:
+    runs-on: [self-hosted, graviton-test]
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          architecture: arm64
+      - name: Setup Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: Checkout TorchServe
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          python ts_scripts/install_dependencies.py --environment=dev
+      - name: Torchserve Regression Tests
+        env:
+          TS_MAC_ARM64_CPU_ONLY: 'True'
+        run: |
+          python test/regression_tests.py

examples/custom_endpoint_plugin/ModelReady.java

@@ -0,0 +1,58 @@
+package org.pytorch.serve.plugins.endpoint;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import org.pytorch.serve.servingsdk.Context;
+import org.pytorch.serve.servingsdk.Model;
+import org.pytorch.serve.servingsdk.ModelServerEndpoint;
+import org.pytorch.serve.servingsdk.Worker;
+import org.pytorch.serve.servingsdk.annotations.Endpoint;
+import org.pytorch.serve.servingsdk.annotations.helpers.EndpointTypes;
+import org.pytorch.serve.servingsdk.http.Request;
+import org.pytorch.serve.servingsdk.http.Response;
+
+@Endpoint(
+        urlPattern = "model-ready",
+        endpointType = EndpointTypes.INFERENCE,
+        description = "Endpoint indicating registered model/s ready to serve inference requests")
+public class ModelReady extends ModelServerEndpoint {
+    private boolean modelsLoaded(Context ctx) {
+        Map<String, Model> modelMap = ctx.getModels();
+
+        if (modelMap.isEmpty()) {
+            return false;
+        }
+
+        for (Map.Entry<String, Model> entry : modelMap.entrySet()) {
+            boolean workerReady = false;
+            for (Worker w : entry.getValue().getModelWorkers()) {
+                if (w.isRunning()) {
+                    workerReady = true;
+                    break;
+                }
+            }
+            if (!workerReady) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public void doGet(Request req, Response rsp, Context ctx) throws IOException {
+        if (modelsLoaded(ctx)) {
+            rsp.setStatus(200, "Model/s ready");
+            rsp.getOutputStream()
+                    .write(
+                            "{\n\t\"Status\": \"Model/s ready\"\n}\n"
+                                    .getBytes(StandardCharsets.UTF_8));
+        } else {
+            rsp.setStatus(503, "Model/s not ready");
+            rsp.getOutputStream()
+                    .write(
+                            "{\n\t\"Status\": \"Model/s not ready\"\n}\n"
+                                    .getBytes(StandardCharsets.UTF_8));
+        }
+    }
+}

examples/custom_endpoint_plugin/README.md

@@ -0,0 +1,145 @@
+# Torchserve custom endpoint plugin
+
+In this example, we demonstrate how to create a custom HTTP API endpoint plugin for TorchServe. Endpoint plugins enable us to dynamically add custom functionality to TorchServe at start time, without having to rebuild TorchServe. For more details on endpoint plugins and TorchServe SDK, refer to the following links:
+- [Plugins Readme](https://github.com/pytorch/serve/tree/master/plugins)
+- [TorchServe SDK source](https://github.com/pytorch/serve/tree/master/serving-sdk)
+
+In this example, we will build an endpoint plugin that implements the functionality of a HTTP API endpoint that reports the readiness of models registered on TorchServe to serve inference requests.
+
+Run the commands given in the following steps from the root directory of the repository. For example, if you cloned the repository into `/home/my_path/serve`, run the steps from `/home/my_path/serve`
+
+## Steps
+
+- Step 1: Install the necessary dependencies for TorchServe development environment
+
+  ```bash
+  $ python ts_scripts/install_dependencies.py --environment=dev
+  ```
+
+- Step 2: Copy [ModelReady.java](ModelReady.java) to the endpoint plugins directory
+
+  ```bash
+  $ cp examples/custom_endpoint_plugin/ModelReady.java plugins/endpoints/src/main/java/org/pytorch/serve/plugins/endpoint
+  ```
+  For reference on implemeting your own custom plugin, review the utilization of the [TorchServe SDK API](https://github.com/pytorch/serve/tree/master/serving-sdk/src/main/java/org/pytorch/serve/servingsdk) and its corresponding [implementation](https://github.com/pytorch/serve/tree/master/frontend/server/src/main/java/org/pytorch/serve/servingsdk/impl) in [ModelReady.java](ModelReady.java).
+
+- Step 3: Copy [org.pytorch.serve.servingsdk.ModelServerEndpoint](org.pytorch.serve.servingsdk.ModelServerEndpoint) to the plugins service provider configuration directory
+
+  ```bash
+  $ cp examples/custom_endpoint_plugin/org.pytorch.serve.servingsdk.ModelServerEndpoint plugins/endpoints/src/main/resources/META-INF/services
+  ```
+
+- Step 4: Update the [endpoint plugins build script](../../plugins/endpoints/build.gradle) to only include the required plugins in the JAR
+
+  ```bash
+  .....
+  .....
+  /**
+   * By default, include all endpoint plugins in the JAR.
+   * In order to build a custom JAR with specific endpoint plugins, specify the required paths.
+   * For example:
+   * include "org/pytorch/serve/plugins/endpoint/Ping*"
+   * include "org/pytorch/serve/plugins/endpoint/ExecutionParameters*"
+   */
+  include "org/pytorch/serve/plugins/endpoint/ModelReady*"
+  .....
+  .....
+  ```
+
+- Step 5: Build the custom endpoint plugin
+
+  ```bash
+  $ cd plugins
+  $ ./gradlew clean build
+  $ cd ..
+  ``` 
+
+- Step 6: Create two example model archives to test the plugin with
+
+  ```bash
+  $ mkdir -p model_store
+  $ torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler  examples/image_classifier/mnist/mnist_handler.py
+  $ mv mnist.mar ./model_store 
+  ```
+
+  ```bash 
+  $ wget https://download.pytorch.org/models/resnet18-f37072fd.pth
+  $ torch-model-archiver --model-name resnet-18 --version 1.0 --model-file ./examples/image_classifier/resnet_18/model.py --serialized-file resnet18-f37072fd.pth --handler image_classifier --extra-files ./examples/image_classifier/index_to_name.json
+  $ mv resnet-18.mar ./model_store
+  ```
+
+- Step 7: Start Torchserve with the appropriate plugins path containing the JAR we just built.
+  The plugin JAR will be contained in the `plugins/endpoints/build/libs` directory. For Ex: `plugins/endpoints/build/libs/endpoints-1.0.jar`
+  ```bash
+  $ torchserve --ncs --start --model-store ./model_store --disable-token-auth --enable-model-api --plugins-path ./plugins/endpoints/build/libs
+  ```
+
+- Step 8: Register the models and test the custom endpoint
+
+  ```bash
+  $ curl -X POST  "http://localhost:8081/models?url=mnist.mar"
+  {
+    "status": "Model \"mnist\" Version: 1.0 registered with 0 initial workers. Use scale workers API to add workers for the model."
+  }
+
+  $ curl -X POST  "http://localhost:8081/models?url=resnet-18.mar"
+  {
+    "status": "Model \"resnet-18\" Version: 1.0 registered with 0 initial workers. Use scale workers API to add workers for the model."
+  }
+  ```
+
+  ```bash
+  $ curl -X GET http://localhost:8080/model-ready
+  {
+    "Status": "Model/s not ready"
+  }
+  ```
+
+  The `model-ready` endpoint reports that the models are not ready since there are no workers that have loaded the models and ready to serve inference requests.
+
+- Step 9: Scale up workers for one of the models and test the custom endpoint
+
+  ```bash
+  $ curl -X PUT "http://localhost:8081/models/mnist?min_worker=1&synchronous=true"
+  {
+    "status": "Workers scaled to 1 for model: mnist"
+  }
+  ```
+
+  ```bash
+  $ curl -X GET http://localhost:8080/model-ready
+  {
+    "Status": "Model/s not ready"
+  }
+  ```
+
+  The `model-ready` endpoint reports that the models are not ready since not all registered models have atleast one worker ready to serve inference requests.
+
+- Step 10: Scale up workers for both models and test the custom endpoint
+
+  ```bash
+  $ curl -X PUT "http://localhost:8081/models/mnist?min_worker=1&synchronous=true"
+  {
+    "status": "Workers scaled to 1 for model: mnist"
+  }
+
+  $ curl -X PUT "http://localhost:8081/models/resnet-18?min_worker=1&synchronous=true"
+  {
+    "status": "Workers scaled to 1 for model: resnet-18"
+  }
+  ```
+
+  ```bash
+  $ curl -X GET http://localhost:8080/model-ready
+  {
+    "Status": "Model/s ready"
+  }
+  ```
+
+  The `model-ready` endpoint reports that the models are now ready since there is atleast one worker per registered model that is ready to serve inference requests.
+
+- Step 11: Stop TorchServe
+  ```bash
+  $ torchserve --stop
+  ```
+

examples/custom_endpoint_plugin/org.pytorch.serve.servingsdk.ModelServerEndpoint

@@ -0,0 +1 @@
+org.pytorch.serve.plugins.endpoint.ModelReady

model-archiver/model_archiver/model_packaging_utils.py

@@ -184,11 +184,21 @@ def copy_artifacts(model_name, runtime, **kwargs):
 
                 if file_type == "extra_files":
                     for path_or_wildcard in path.split(","):
-                        if not Path(path_or_wildcard).exists():
+                        maybe_wildcard = "*" in path_or_wildcard
+                        maybe_wildcard |= "?" in path_or_wildcard
+                        maybe_wildcard |= (
+                            "[" in path_or_wildcard and "]" in path_or_wildcard
+                        )
+                        if not (maybe_wildcard or Path(path_or_wildcard).exists()):
                             raise FileNotFoundError(
                                 f"File does not exist: {path_or_wildcard}"
                             )
-                        for file in glob.glob(path_or_wildcard.strip()):
+                        files = glob.glob(path_or_wildcard.strip())
+                        if maybe_wildcard and len(files) == 0:
+                            logging.warning(
+                                f"Given wildcard pattern did not match any file: {path_or_wildcard}"
+                            )
+                        for file in files:
                             if os.path.isfile(file):
                                 shutil.copy2(file, model_path)
                             elif os.path.isdir(file) and file != model_path:

plugins/endpoints/build.gradle

@@ -22,8 +22,15 @@ jar {
     exclude "META-INF/MANIFEST*"
     exclude "META-INF//LICENSE*"
     exclude "META-INF//NOTICE*"
-    exclude "org/pytorch/serve/plugins/endpoint/ExecutionParameters*" // Comment out if ExecutionParameter endpoint is needed
-    exclude "org/pytorch/serve/plugins/endpoint/Ping*" // Comment out if Ping endpoint is needed
+    include "META-INF/services/*"
+    /**
+     * By default, include all endpoint plugins in the JAR.
+     * In order to build a custom JAR with specific endpoint plugins, specify the required paths.
+     * For example:
+     * include "org/pytorch/serve/plugins/endpoint/Ping*"
+     * include "org/pytorch/serve/plugins/endpoint/ExecutionParameters*"
+     */
+    include "org/pytorch/serve/plugins/endpoint/*"
 }
 
 java {

plugins/endpoints/src/main/resources/META-INF/services/org.pytorch.serve.servingsdk.ModelServerEndpoint

@@ -0,0 +1,2 @@
+org.pytorch.serve.plugins.endpoint.ExecutionParameters
+org.pytorch.serve.plugins.endpoint.Ping

test/pytest/test_gRPC_inference_api.py

@@ -1,10 +1,12 @@
 import json
 import os
+import platform
 import threading
 from ast import literal_eval
 
 import inference_pb2
 import management_pb2
+import pytest
 import test_gRPC_utils
 import test_utils
 
@@ -50,6 +52,9 @@ def __infer(stub, model_name, model_input):
     return prediction
 
 
+@pytest.mark.skipif(
+    platform.machine() == "aarch64", reason="Test skipped on aarch64 architecture"
+)
 def test_inference_apis():
     with open(os.path.join(os.path.dirname(__file__), inference_data_json), "rb") as f:
         test_data = json.loads(f.read())

test/pytest/test_model_custom_dependencies.py

@@ -1,7 +1,9 @@
 import os
 import pathlib
+import platform
 import subprocess
 
+import pytest
 import requests
 import test_utils
 from model_archiver import ModelArchiver, ModelArchiverConfig
@@ -140,6 +142,9 @@ def register_model_and_make_inference_request(expect_model_load_failure=False):
         resp.raise_for_status()
 
 
+@pytest.mark.skipif(
+    platform.machine() == "aarch64", reason="Test skipped on aarch64 architecture"
+)
 def test_install_dependencies_to_target_directory_with_requirements():
     test_utils.torchserve_cleanup()