-
Notifications
You must be signed in to change notification settings - Fork 842
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Profile TorchServe Handler (preprocess vs inference vs post-process) (#…
…2470) * Profile TS Handler using ab tool * Added an example * Added an example * handler class not needed * Add model_yaml_config to MockCOntext * remove unnecessary config * based on review comments * Added details on how to enable this * Added details on how to enable this * lint fix * lint fix * lint fix --------- Co-authored-by: Geeta Chauhan <[email protected]>
- Loading branch information
Showing
12 changed files
with
239 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
|
||
# Benchmark ResNet50 and profile the detailed split of PredictionTime | ||
|
||
This example shows how to run the benchmark ab tool on ResNet50 and identify the time spent on preprocess, inference and postprocess | ||
|
||
Change directory to the root of `serve` | ||
Ex: if `serve` is under `/home/ubuntu`, change directory to `/home/ubuntu/serve` | ||
|
||
|
||
## Download the weights | ||
|
||
``` | ||
wget https://download.pytorch.org/models/resnet50-11ad3fa6.pth | ||
``` | ||
|
||
### Create model archive | ||
|
||
To enable profiling of TorchServe Handler, add the following config in model-config.yaml | ||
``` | ||
handler: | ||
profile: true | ||
``` | ||
|
||
``` | ||
torch-model-archiver --model-name resnet-50 --version 1.0 --model-file ./examples/benchmarking/resnet50/model.py --serialized-file resnet50-11ad3fa6.pth --handler image_classifier --extra-files ./examples/image_classifier/index_to_name.json --config-file ./examples/benchmarking/resnet50/model-config.yaml | ||
mkdir model_store | ||
mv resnet-50.mar model_store/. | ||
``` | ||
|
||
### Install dependencies for benchmark tool | ||
|
||
``` | ||
sudo apt-get update -y | ||
sudo apt-get install -y apache2-utils | ||
pip install -r benchmarks/requirements-ab.txt | ||
``` | ||
|
||
### Run ab tool for benchmarking | ||
|
||
``` | ||
python benchmarks/auto_benchmark.py --input examples/benchmarking/resnet50/benchmark_profile.yaml --skip true | ||
``` | ||
|
||
This generates the report under `/tmp/ts_benchmarking/report.md` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
# Torchserve version is to be installed. It can be one of the options | ||
# - branch : "master" | ||
# - nightly: "2022.3.16" | ||
# - release: "0.5.3" | ||
# Nightly build will be installed if "ts_version" is not specifiged | ||
#ts_version: | ||
# branch: &ts_version "master" | ||
|
||
# a list of model configure yaml files defined in benchmarks/models_config | ||
# or a list of model configure yaml files with full path | ||
models: | ||
- "/home/ubuntu/serve/examples/benchmarking/resnet50/resnet50.yaml" | ||
|
||
# benchmark on "cpu" or "gpu". | ||
# "cpu" is set if "hardware" is not specified | ||
hardware: &hardware "gpu" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
handler: | ||
profile: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
from torchvision.models.resnet import Bottleneck, ResNet | ||
|
||
|
||
class ImageClassifier(ResNet): | ||
def __init__(self): | ||
super(ImageClassifier, self).__init__(Bottleneck, [3, 4, 6, 3]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
--- | ||
resnet50: | ||
eager_mode: | ||
benchmark_engine: "ab" | ||
url: "file:///home/ubuntu/serve/model_store/resnet-50.mar" | ||
workers: | ||
- 4 | ||
batch_delay: 100 | ||
batch_size: | ||
- 1 | ||
- 2 | ||
- 4 | ||
- 8 | ||
- 16 | ||
- 32 | ||
- 64 | ||
requests: 10000 | ||
concurrency: 100 | ||
input: "./examples/image_classifier/kitten.jpg" | ||
handler_profiling: true | ||
exec_env: "local" | ||
processors: | ||
- "cpu" | ||
- "gpus": "all" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
""" | ||
Decorator for timing handler methods | ||
Use this decorator to compute the execution time for your preprocesss, inference and | ||
postprocess methods. | ||
By default this feature is not enabled. | ||
To enable this, add the following section in your model-config.yaml file | ||
handler: | ||
profile: true | ||
An example of running benchmarks with the profiling enabled is in | ||
https://github.com/pytorch/serve/tree/master/examples/benchmarking/resnet50 | ||
""" | ||
|
||
import time | ||
|
||
import torch | ||
|
||
|
||
def timed(func): | ||
def wrap_func(self, *args, **kwargs): | ||
# Measure time if config specified in model_yaml_config | ||
if ( | ||
"handler" in self.context.model_yaml_config | ||
and "profile" in self.context.model_yaml_config["handler"] | ||
): | ||
if self.context.model_yaml_config["handler"]["profile"]: | ||
# Measure start time | ||
if torch.cuda.is_available(): | ||
start = torch.cuda.Event(enable_timing=True) | ||
end = torch.cuda.Event(enable_timing=True) | ||
start.record() | ||
else: | ||
start = time.time() | ||
|
||
result = func(self, *args, **kwargs) | ||
|
||
# Measure end time | ||
if torch.cuda.is_available(): | ||
end.record() | ||
torch.cuda.synchronize() | ||
else: | ||
end = time.time() | ||
|
||
# Measure time taken to execute the function in miliseconds | ||
if torch.cuda.is_available(): | ||
duration = start.elapsed_time(end) | ||
else: | ||
duration = (end - start) * 1000 | ||
|
||
# Add metrics for profiling | ||
metrics = self.context.metrics | ||
metrics.add_time("ts_handler_" + func.__name__, duration) | ||
else: | ||
# If profile config specified in model_yaml_config is False | ||
result = func(self, *args, **kwargs) | ||
else: | ||
# If no profile config specified in model_yaml_config | ||
result = func(self, *args, **kwargs) | ||
|
||
return result | ||
|
||
return wrap_func |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.