diff --git a/docs/getting_started.md b/docs/getting_started.md index 11daf6db7b..7354f5515b 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -170,6 +170,10 @@ All the logs you've seen as output to stdout related to model registration, mana High level performance data like Throughput or Percentile Precision can be generated with [Benchmark](https://github.com/pytorch/serve/tree/master/benchmarks/README.md) and visualized in a report. +## Debugging Handler Code + +If you want to debug your handler code, you can run TorchServe with just the backend and hence use any python debugger. You can refer to an example defined [here](../examples/image_classifier/resnet_18/README.md#debug-torchserve-backend) + ### Contributing If you plan to develop with TorchServe and change some source code, follow the [contributing guide](https://github.com/pytorch/serve/blob/master/CONTRIBUTING.md). diff --git a/examples/image_classifier/resnet_18/README.md b/examples/image_classifier/resnet_18/README.md index 5611b14811..1c5bbe3639 100644 --- a/examples/image_classifier/resnet_18/README.md +++ b/examples/image_classifier/resnet_18/README.md @@ -35,8 +35,8 @@ curl http://127.0.0.1:8080/predictions/resnet-18 -T ./examples/image_classifier/ example_input = torch.rand(1, 3, 224, 224) traced_script_module = torch.jit.trace(model, example_input) traced_script_module.save("resnet-18.pt") - ``` - + ``` + * Use following commands to register Resnet18 torchscript model on TorchServe and run image prediction ```bash @@ -46,3 +46,63 @@ curl http://127.0.0.1:8080/predictions/resnet-18 -T ./examples/image_classifier/ torchserve --start --model-store model_store --models resnet-18=resnet-18.mar curl http://127.0.0.1:8080/predictions/resnet-18 -T ./serve/examples/image_classifier/kitten.jpg ``` + +### Debug TorchServe Backend + +If you want to test your handler code, you can use the example in `debugging_backend/test_handler.py` + +``` +python debugging_backend/test_handler.py --batch_size 2 +``` + +results in + +``` +Torch TensorRT not enabled +DEBUG:ts.torch_handler.base_handler:Model file /home/ubuntu/serve/examples/image_classifier/resnet_18/resnet-18.pt loaded successfully +INFO:__main__:Result is [{'tabby': 0.4096629023551941, 'tiger_cat': 0.34670525789260864, 'Egyptian_cat': 0.13002872467041016, 'lynx': 0.02391958236694336, 'bucket': 0.011532173492014408}, {'tabby': 0.4096629023551941, 'tiger_cat': 0.34670525789260864, 'Egyptian_cat': 0.13002872467041016, 'lynx': 0.02391958236694336, 'bucket': 0.011532173492014408}] +``` + +If this doesn't work, you can use a debugger to find the problem in your backend handler code. +Once you are confident this works, you can use your handler to deploy the model using TorchServe + +Below is a screenshot of debugger running with this handler + +![image info](./debugging_backend/debugger_screenshot.png) + +You can also use this with pytest + +``` +pytest debugging_backend/test_handler.py +``` + +results in + +``` +================================================================================== test session starts =================================================================================== +platform linux -- Python 3.8.18, pytest-7.3.1, pluggy-1.0.0 +rootdir: /home/ubuntu/serve +plugins: mock-3.10.0, anyio-3.6.1, cov-4.1.0, hypothesis-6.54.3 +collected 1 item + +debugging_backend/test_handler.py . [100%] + +==================================================================================== warnings summary ==================================================================================== +../../../../anaconda3/envs/torchserve/lib/python3.8/site-packages/ts/torch_handler/base_handler.py:13 + /home/ubuntu/anaconda3/envs/torchserve/lib/python3.8/site-packages/ts/torch_handler/base_handler.py:13: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html + from pkg_resources import packaging + +../../../../anaconda3/envs/torchserve/lib/python3.8/site-packages/pkg_resources/__init__.py:2871 + /home/ubuntu/anaconda3/envs/torchserve/lib/python3.8/site-packages/pkg_resources/__init__.py:2871: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('mpl_toolkits')`. + Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages + declare_namespace(pkg) + +../../../../anaconda3/envs/torchserve/lib/python3.8/site-packages/pkg_resources/__init__.py:2871 +../../../../anaconda3/envs/torchserve/lib/python3.8/site-packages/pkg_resources/__init__.py:2871 + /home/ubuntu/anaconda3/envs/torchserve/lib/python3.8/site-packages/pkg_resources/__init__.py:2871: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('ruamel')`. + Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages + declare_namespace(pkg) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +============================================================================= 1 passed, 4 warnings in 2.29s ============================================================================== +``` diff --git a/examples/image_classifier/resnet_18/debugging_backend/debugger_screenshot.png b/examples/image_classifier/resnet_18/debugging_backend/debugger_screenshot.png new file mode 100644 index 0000000000..e7f3eb7f11 Binary files /dev/null and b/examples/image_classifier/resnet_18/debugging_backend/debugger_screenshot.png differ diff --git a/examples/image_classifier/resnet_18/debugging_backend/test_handler.py b/examples/image_classifier/resnet_18/debugging_backend/test_handler.py new file mode 100644 index 0000000000..f777e748fa --- /dev/null +++ b/examples/image_classifier/resnet_18/debugging_backend/test_handler.py @@ -0,0 +1,70 @@ +import argparse +import io +import logging +from pathlib import Path + +import torch + +from ts.torch_handler.image_classifier import ImageClassifier +from ts.torch_handler.unit_tests.test_utils.mock_context import MockContext + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + +CURR_FILE_PATH = Path(__file__).parent.absolute() +REPO_ROOT_DIR = CURR_FILE_PATH.parents[3] +EXAMPLE_ROOT_DIR = REPO_ROOT_DIR.joinpath("examples", "image_classifier", "resnet_18") +TEST_DATA = REPO_ROOT_DIR.joinpath("examples", "image_classifier", "kitten.jpg") +MODEL_PT_FILE = "resnet-18.pt" + + +def prepare_data(batch_size): + """ + Function to prepare data based on the desired batch size + """ + f = io.open(TEST_DATA, "rb", buffering=0) + read_data = f.read() + data = [] + for i in range(batch_size): + tmp = {} + tmp["data"] = read_data + data.append(tmp) + return data + + +def test_resnet18(batch_size=1): + # Define your handler + handler = ImageClassifier() + + # Context definition + ctx = MockContext( + model_pt_file=MODEL_PT_FILE, + model_dir=EXAMPLE_ROOT_DIR.as_posix(), + model_file=None, + ) + + torch.manual_seed(42 * 42) + handler.initialize(ctx) + handler.context = ctx + + data = prepare_data(batch_size) + + # Here we are using the BaseHandler's handle method. You can define your own + result = handler.handle(data, ctx) + logger.info(f"Result is {result}") + + # Can be used with pytest + value = max(result[0], key=result[0].get) + assert value == "tabby" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--batch_size", + default=1, + type=int, + help="Batch size for testing inference", + ) + args = parser.parse_args() + test_resnet18(args.batch_size) diff --git a/ts/torch_handler/unit_tests/test_utils/mock_context.py b/ts/torch_handler/unit_tests/test_utils/mock_context.py index 287074c6eb..b9bd7ededa 100644 --- a/ts/torch_handler/unit_tests/test_utils/mock_context.py +++ b/ts/torch_handler/unit_tests/test_utils/mock_context.py @@ -2,11 +2,13 @@ Mocks for adding model context without loading all of Torchserve """ +import os import uuid import torch from ts.metrics.metrics_store import MetricsStore +from ts.utils.util import get_yaml_config class MockContext: