diff --git a/.gitignore b/.gitignore index a1c1ed948..6f2fb2ee0 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,6 @@ variables.data* # Local files /*.txt + +# Top Level Script Files +/*.sh \ No newline at end of file diff --git a/tftrt/examples/benchmark_args.py b/tftrt/examples/benchmark_args.py index 93d379f94..9bdbce246 100644 --- a/tftrt/examples/benchmark_args.py +++ b/tftrt/examples/benchmark_args.py @@ -140,13 +140,6 @@ def __init__(self): help="If set to True, the benchmark will use XLA JIT Compilation." ) - self._add_bool_argument( - name="skip_accuracy_testing", - default=False, - required=False, - help="If set to True, accuracy calculation will be skipped." - ) - self._add_bool_argument( name="use_synthetic_data", default=False, @@ -187,6 +180,15 @@ def __init__(self): help="Minimum number of TensorFlow ops in a TRT engine." ) + self._parser.add_argument( + "--num_build_batches", + type=int, + default=1, + help="How many iterations(batches) to use to build the TF-TRT " + "engines. If not supplied, only one batch will be used. This " + "parameter has only an effect if `--optimize_offline=True`" + ) + self._parser.add_argument( "--num_calib_batches", type=int, diff --git a/tftrt/examples/benchmark_runner.py b/tftrt/examples/benchmark_runner.py index ee39840c5..bf1a66624 100644 --- a/tftrt/examples/benchmark_runner.py +++ b/tftrt/examples/benchmark_runner.py @@ -169,7 +169,11 @@ def get_trt_precision(precision): print("\n[*] TF-TRT Converter Parameters:") print_dict(trt_converter_params) - converter = trt.TrtGraphConverterV2(**trt_converter_params) + try: + converter = trt.TrtGraphConverterV2(**trt_converter_params) + except TypeError: + del trt_converter_params["enable_sparse_compute"] + converter = trt.TrtGraphConverterV2(**trt_converter_params) def engine_build_input_fn(num_batches, model_phase): dataset, _ = self.get_dataset_batches() @@ -218,14 +222,14 @@ def engine_build_input_fn(num_batches, model_phase): except AttributeError: pass - if strtobool(os.environ.get("TF_TRT_BENCHMARK_QUIT_AFTER_SUMMARY", - "0")): + if strtobool(os.environ.get("TF_TRT_BENCHMARK_EARLY_QUIT", "0")): sys.exit(0) - if self._args.optimize_offline or self._args.use_dynamic_shape: + if self._args.optimize_offline: offline_opt_input_fn = lambda: engine_build_input_fn( - num_batches=1, model_phase="Building" + num_batches=self._args.num_build_batches, + model_phase="Building" ) with timed_section("Building TensorRT engines"): @@ -248,7 +252,9 @@ def engine_build_input_fn(num_batches, model_phase): del converter del graph_func graph_func = load_model_from_disk( - self._args.output_saved_model_dir + self._args.output_saved_model_dir, + tags=self._args.model_tag.split(","), + signature_key=self._args.input_signature_key ) if isinstance(graph_func.structured_outputs, (tuple, list)): diff --git a/tftrt/examples/image_classification/scripts/base_script.sh b/tftrt/examples/image_classification/base_run_inference.sh similarity index 96% rename from tftrt/examples/image_classification/scripts/base_script.sh rename to tftrt/examples/image_classification/base_run_inference.sh index d6c63053d..2b66ddfda 100755 --- a/tftrt/examples/image_classification/scripts/base_script.sh +++ b/tftrt/examples/image_classification/base_run_inference.sh @@ -40,7 +40,7 @@ do shift # Remove --output_tensors_name= from processing ;; --use_xla_auto_jit) - TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=--tf_xla_auto_jit=2" + TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=\"--tf_xla_auto_jit=2 --tf_xla_cpu_global_jit\"" shift # Remove --use_xla_auto_jit from processing ;; *) @@ -54,7 +54,7 @@ done INPUT_SIZE=224 PREPROCESS_METHOD="vgg" NUM_CLASSES=1001 -MAX_SAMPLES=49920 +MAX_SAMPLES=50000 OUTPUT_TENSORS_NAME="logits" case ${MODEL_NAME} in @@ -153,7 +153,7 @@ fi # %%%%%%%%%%%%%%%%%%%%%%% ARGUMENT VALIDATION %%%%%%%%%%%%%%%%%%%%%%% # -BENCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" +BENCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd ${BENCH_DIR} # Execute the example diff --git a/tftrt/examples/image_classification/models/inception_v3/analysis.txt b/tftrt/examples/image_classification/models/inception_v3/analysis.txt new file mode 100644 index 000000000..9c078994c --- /dev/null +++ b/tftrt/examples/image_classification/models/inception_v3/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 299, 299, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/inception_v3/run_inference.sh b/tftrt/examples/image_classification/models/inception_v3/run_inference.sh new file mode 100755 index 000000000..6f4996d78 --- /dev/null +++ b/tftrt/examples/image_classification/models/inception_v3/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="inception_v3" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/inception_v4/analysis.txt b/tftrt/examples/image_classification/models/inception_v4/analysis.txt new file mode 100644 index 000000000..9c078994c --- /dev/null +++ b/tftrt/examples/image_classification/models/inception_v4/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 299, 299, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/inception_v4/run_inference.sh b/tftrt/examples/image_classification/models/inception_v4/run_inference.sh new file mode 100755 index 000000000..444fe998c --- /dev/null +++ b/tftrt/examples/image_classification/models/inception_v4/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="inception_v4" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/mobilenet_v1/analysis.txt b/tftrt/examples/image_classification/models/mobilenet_v1/analysis.txt new file mode 100644 index 000000000..f9b55358a --- /dev/null +++ b/tftrt/examples/image_classification/models/mobilenet_v1/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/mobilenet_v1/run_inference.sh b/tftrt/examples/image_classification/models/mobilenet_v1/run_inference.sh new file mode 100755 index 000000000..ddad56b16 --- /dev/null +++ b/tftrt/examples/image_classification/models/mobilenet_v1/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="mobilenet_v1" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/mobilenet_v2/analysis.txt b/tftrt/examples/image_classification/models/mobilenet_v2/analysis.txt new file mode 100644 index 000000000..f9b55358a --- /dev/null +++ b/tftrt/examples/image_classification/models/mobilenet_v2/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/mobilenet_v2/run_inference.sh b/tftrt/examples/image_classification/models/mobilenet_v2/run_inference.sh new file mode 100755 index 000000000..a1cbbffb1 --- /dev/null +++ b/tftrt/examples/image_classification/models/mobilenet_v2/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="mobilenet_v2" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/nasnet_large/analysis.txt b/tftrt/examples/image_classification/models/nasnet_large/analysis.txt new file mode 100644 index 000000000..19354b025 --- /dev/null +++ b/tftrt/examples/image_classification/models/nasnet_large/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 331, 331, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/nasnet_large/run_inference.sh b/tftrt/examples/image_classification/models/nasnet_large/run_inference.sh new file mode 100755 index 000000000..763d33ada --- /dev/null +++ b/tftrt/examples/image_classification/models/nasnet_large/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="nasnet_large" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/nasnet_mobile/analysis.txt b/tftrt/examples/image_classification/models/nasnet_mobile/analysis.txt new file mode 100644 index 000000000..f9b55358a --- /dev/null +++ b/tftrt/examples/image_classification/models/nasnet_mobile/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/nasnet_mobile/run_inference.sh b/tftrt/examples/image_classification/models/nasnet_mobile/run_inference.sh new file mode 100755 index 000000000..7a3a198b0 --- /dev/null +++ b/tftrt/examples/image_classification/models/nasnet_mobile/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="nasnet_mobile" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/analysis.txt b/tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/analysis.txt new file mode 100644 index 000000000..6b49b55e2 --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/analysis.txt @@ -0,0 +1,36 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['predict']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['classes'] tensor_info: + dtype: DT_INT32 + shape: (-1) + name: ArgMax:0 + outputs['probabilities'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: resnet50/output/softmax:0 + Method name is: tensorflow/serving/predict + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['classes'] tensor_info: + dtype: DT_INT32 + shape: (-1) + name: ArgMax:0 + outputs['probabilities'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: resnet50/output/softmax:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh b/tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh new file mode 100755 index 000000000..f01361b3f --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="resnet50-v1.5_tf1_ngc" ${@} diff --git a/tftrt/examples/image_classification/models/resnet50v2_backbone/analysis.txt b/tftrt/examples/image_classification/models/resnet50v2_backbone/analysis.txt new file mode 100644 index 000000000..a6aaaa941 --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet50v2_backbone/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['resnet50v2']: + The given SavedModel SignatureDef contains the following input(s): + inputs['inputs'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 256, 256, 3) + name: conv2d_input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['outputs'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 8, 8, 2048) + name: outputs:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/resnet50v2_backbone/run_inference.sh b/tftrt/examples/image_classification/models/resnet50v2_backbone/run_inference.sh new file mode 100755 index 000000000..7c388697b --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet50v2_backbone/run_inference.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh \ + --model_name="resnet50v2_backbone" \ + --input_signature_key="resnet50v2" \ + --use_synthetic_data \ + ${@} diff --git a/tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/analysis.txt b/tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/analysis.txt new file mode 100644 index 000000000..e78f096cf --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['resnet50v2']: + The given SavedModel SignatureDef contains the following input(s): + inputs['inputs'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 256, 256, 3) + name: inputs:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['outputs'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 8, 8, 2048) + name: outputs:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/run_inference.sh b/tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/run_inference.sh new file mode 100755 index 000000000..62753f3fe --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet50v2_sparse_backbone/run_inference.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh \ + --model_name="resnet50v2_sparse_backbone" \ + --input_signature_key="resnet50v2" \ + --use_synthetic_data \ + ${@} diff --git a/tftrt/examples/image_classification/models/resnet_v1.5_50_tfv2/analysis.txt b/tftrt/examples/image_classification/models/resnet_v1.5_50_tfv2/analysis.txt new file mode 100644 index 000000000..dcae4fb1c --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet_v1.5_50_tfv2/analysis.txt @@ -0,0 +1,115 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['__saved_model_init_op']: + The given SavedModel SignatureDef contains the following input(s): + The given SavedModel SignatureDef contains the following output(s): + outputs['__saved_model_init_op'] tensor_info: + dtype: DT_INVALID + shape: unknown_rank + name: NoOp + Method name is: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input_1'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: serving_default_input_1:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['activation_49'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: StatefulPartitionedCall:0 + Method name is: tensorflow/serving/predict + +Concrete Functions: + Function Name: '__call__' + Option #1 + Callable with: + Argument #1 + input_1: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='input_1') + Argument #2 + DType: bool + Value: True + Argument #3 + DType: NoneType + Value: None + Option #2 + Callable with: + Argument #1 + inputs: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='inputs') + Argument #2 + DType: bool + Value: True + Argument #3 + DType: NoneType + Value: None + Option #3 + Callable with: + Argument #1 + inputs: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='inputs') + Argument #2 + DType: bool + Value: False + Argument #3 + DType: NoneType + Value: None + Option #4 + Callable with: + Argument #1 + input_1: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='input_1') + Argument #2 + DType: bool + Value: False + Argument #3 + DType: NoneType + Value: None + + Function Name: '_default_save_signature' + Option #1 + Callable with: + Argument #1 + input_1: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='input_1') + + Function Name: 'call_and_return_all_conditional_losses' + Option #1 + Callable with: + Argument #1 + inputs: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='inputs') + Argument #2 + DType: bool + Value: False + Argument #3 + DType: NoneType + Value: None + Option #2 + Callable with: + Argument #1 + inputs: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='inputs') + Argument #2 + DType: bool + Value: True + Argument #3 + DType: NoneType + Value: None + Option #3 + Callable with: + Argument #1 + input_1: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='input_1') + Argument #2 + DType: bool + Value: True + Argument #3 + DType: NoneType + Value: None + Option #4 + Callable with: + Argument #1 + input_1: TensorSpec(shape=(None, 224, 224, 3), dtype=tf.float32, name='input_1') + Argument #2 + DType: bool + Value: False + Argument #3 + DType: NoneType + Value: None diff --git a/tftrt/examples/image_classification/models/resnet_v1.5_50_tfv2/run_inference.sh b/tftrt/examples/image_classification/models/resnet_v1.5_50_tfv2/run_inference.sh new file mode 100755 index 000000000..4aaeeb7f3 --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet_v1.5_50_tfv2/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="resnet_v1.5_50_tfv2" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/resnet_v1_50/analysis.txt b/tftrt/examples/image_classification/models/resnet_v1_50/analysis.txt new file mode 100644 index 000000000..f9b55358a --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet_v1_50/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/resnet_v1_50/run_inference.sh b/tftrt/examples/image_classification/models/resnet_v1_50/run_inference.sh new file mode 100755 index 000000000..40b8d0f40 --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet_v1_50/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="resnet_v1_50" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/resnet_v2_50/analysis.txt b/tftrt/examples/image_classification/models/resnet_v2_50/analysis.txt new file mode 100644 index 000000000..f9b55358a --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet_v2_50/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1001) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/resnet_v2_50/run_inference.sh b/tftrt/examples/image_classification/models/resnet_v2_50/run_inference.sh new file mode 100755 index 000000000..dced4cea1 --- /dev/null +++ b/tftrt/examples/image_classification/models/resnet_v2_50/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="resnet_v2_50" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/vgg_16/analysis.txt b/tftrt/examples/image_classification/models/vgg_16/analysis.txt new file mode 100644 index 000000000..34426f823 --- /dev/null +++ b/tftrt/examples/image_classification/models/vgg_16/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1000) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/vgg_16/run_inference.sh b/tftrt/examples/image_classification/models/vgg_16/run_inference.sh new file mode 100755 index 000000000..29e0f6727 --- /dev/null +++ b/tftrt/examples/image_classification/models/vgg_16/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="vgg_16" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/models/vgg_19/analysis.txt b/tftrt/examples/image_classification/models/vgg_19/analysis.txt new file mode 100644 index 000000000..34426f823 --- /dev/null +++ b/tftrt/examples/image_classification/models/vgg_19/analysis.txt @@ -0,0 +1,15 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 224, 224, 3) + name: input:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['logits'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1000) + name: logits:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/image_classification/models/vgg_19/run_inference.sh b/tftrt/examples/image_classification/models/vgg_19/run_inference.sh new file mode 100755 index 000000000..fee01f337 --- /dev/null +++ b/tftrt/examples/image_classification/models/vgg_19/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="vgg_19" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/inception_v3.sh b/tftrt/examples/image_classification/scripts/inception_v3.sh deleted file mode 100755 index a626e27e1..000000000 --- a/tftrt/examples/image_classification/scripts/inception_v3.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="inception_v3" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/inception_v4.sh b/tftrt/examples/image_classification/scripts/inception_v4.sh deleted file mode 100755 index 1076d05a1..000000000 --- a/tftrt/examples/image_classification/scripts/inception_v4.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="inception_v4" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/mobilenet_v1.sh b/tftrt/examples/image_classification/scripts/mobilenet_v1.sh deleted file mode 100755 index 20839bc47..000000000 --- a/tftrt/examples/image_classification/scripts/mobilenet_v1.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="mobilenet_v1" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/mobilenet_v2.sh b/tftrt/examples/image_classification/scripts/mobilenet_v2.sh deleted file mode 100755 index 0e8cc2a95..000000000 --- a/tftrt/examples/image_classification/scripts/mobilenet_v2.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="mobilenet_v2" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/nasnet_large.sh b/tftrt/examples/image_classification/scripts/nasnet_large.sh deleted file mode 100755 index 433d3ec8a..000000000 --- a/tftrt/examples/image_classification/scripts/nasnet_large.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="nasnet_large" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/nasnet_mobile.sh b/tftrt/examples/image_classification/scripts/nasnet_mobile.sh deleted file mode 100755 index beedb610f..000000000 --- a/tftrt/examples/image_classification/scripts/nasnet_mobile.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="nasnet_mobile" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/resnet50-v1.5_tf1_ngc.sh b/tftrt/examples/image_classification/scripts/resnet50-v1.5_tf1_ngc.sh deleted file mode 100755 index 892551788..000000000 --- a/tftrt/examples/image_classification/scripts/resnet50-v1.5_tf1_ngc.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="resnet50-v1.5_tf1_ngc" ${@} diff --git a/tftrt/examples/image_classification/scripts/resnet50_v2_backbone.sh b/tftrt/examples/image_classification/scripts/resnet50_v2_backbone.sh deleted file mode 100755 index dbf636f02..000000000 --- a/tftrt/examples/image_classification/scripts/resnet50_v2_backbone.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh \ - --model_name="resnet50v2_backbone" \ - --input_signature_key="resnet50v2" \ - --skip_accuracy_testing \ - ${@} diff --git a/tftrt/examples/image_classification/scripts/resnet50_v2_sparse_backbone.sh b/tftrt/examples/image_classification/scripts/resnet50_v2_sparse_backbone.sh deleted file mode 100755 index 79fd27407..000000000 --- a/tftrt/examples/image_classification/scripts/resnet50_v2_sparse_backbone.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh \ - --model_name="resnet50v2_sparse_backbone" \ - --input_signature_key="resnet50v2" \ - --skip_accuracy_testing \ - ${@} diff --git a/tftrt/examples/image_classification/scripts/resnet_v1.5_50_tfv2.sh b/tftrt/examples/image_classification/scripts/resnet_v1.5_50_tfv2.sh deleted file mode 100755 index c6bec7bdf..000000000 --- a/tftrt/examples/image_classification/scripts/resnet_v1.5_50_tfv2.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="resnet_v1.5_50_tfv2" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/resnet_v1_50.sh b/tftrt/examples/image_classification/scripts/resnet_v1_50.sh deleted file mode 100755 index 808342477..000000000 --- a/tftrt/examples/image_classification/scripts/resnet_v1_50.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="resnet_v1_50" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/resnet_v2_50.sh b/tftrt/examples/image_classification/scripts/resnet_v2_50.sh deleted file mode 100755 index 94eb3a8d1..000000000 --- a/tftrt/examples/image_classification/scripts/resnet_v2_50.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="resnet_v2_50" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/vgg_16.sh b/tftrt/examples/image_classification/scripts/vgg_16.sh deleted file mode 100755 index 4ee2fd9e9..000000000 --- a/tftrt/examples/image_classification/scripts/vgg_16.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="vgg_16" ${@} \ No newline at end of file diff --git a/tftrt/examples/image_classification/scripts/vgg_19.sh b/tftrt/examples/image_classification/scripts/vgg_19.sh deleted file mode 100755 index eee34087e..000000000 --- a/tftrt/examples/image_classification/scripts/vgg_19.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="vgg_19" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/base_script.sh b/tftrt/examples/object_detection/base_run_inference.sh similarity index 96% rename from tftrt/examples/object_detection/scripts/base_script.sh rename to tftrt/examples/object_detection/base_run_inference.sh index aa849398b..de2e79589 100755 --- a/tftrt/examples/object_detection/scripts/base_script.sh +++ b/tftrt/examples/object_detection/base_run_inference.sh @@ -45,7 +45,7 @@ do shift # Remove --input_saved_model_dir= from processing ;; --use_xla_auto_jit) - TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=--tf_xla_auto_jit=2" + TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=\"--tf_xla_auto_jit=2 --tf_xla_cpu_global_jit\"" shift # Remove --use_xla_auto_jit from processing ;; *) @@ -136,7 +136,7 @@ fi # %%%%%%%%%%%%%%%%%%%%%%% ARGUMENT VALIDATION %%%%%%%%%%%%%%%%%%%%%%% # -BENCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" +BENCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd ${BENCH_DIR} # Step 1: Installing dependencies if needed: diff --git a/tftrt/examples/object_detection/models/faster_rcnn_nas/analysis.txt b/tftrt/examples/object_detection/models/faster_rcnn_nas/analysis.txt new file mode 100644 index 000000000..46be39734 --- /dev/null +++ b/tftrt/examples/object_detection/models/faster_rcnn_nas/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1, -1) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/faster_rcnn_nas/run_inference.sh b/tftrt/examples/object_detection/models/faster_rcnn_nas/run_inference.sh new file mode 100755 index 000000000..0eb873ad3 --- /dev/null +++ b/tftrt/examples/object_detection/models/faster_rcnn_nas/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="faster_rcnn_nas" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/faster_rcnn_resnet50_coco/analysis.txt b/tftrt/examples/object_detection/models/faster_rcnn_resnet50_coco/analysis.txt new file mode 100644 index 000000000..46be39734 --- /dev/null +++ b/tftrt/examples/object_detection/models/faster_rcnn_resnet50_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1, -1) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/faster_rcnn_resnet50_coco/run_inference.sh b/tftrt/examples/object_detection/models/faster_rcnn_resnet50_coco/run_inference.sh new file mode 100755 index 000000000..594bc6574 --- /dev/null +++ b/tftrt/examples/object_detection/models/faster_rcnn_resnet50_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="faster_rcnn_resnet50_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/mask_rcnn_resnet50_atrous_coco/analysis.txt b/tftrt/examples/object_detection/models/mask_rcnn_resnet50_atrous_coco/analysis.txt new file mode 100644 index 000000000..46be39734 --- /dev/null +++ b/tftrt/examples/object_detection/models/mask_rcnn_resnet50_atrous_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1, -1) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/mask_rcnn_resnet50_atrous_coco/run_inference.sh b/tftrt/examples/object_detection/models/mask_rcnn_resnet50_atrous_coco/run_inference.sh new file mode 100755 index 000000000..bf80acf82 --- /dev/null +++ b/tftrt/examples/object_detection/models/mask_rcnn_resnet50_atrous_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="mask_rcnn_resnet50_atrous_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/ssd_inception_v2_coco/analysis.txt b/tftrt/examples/object_detection/models/ssd_inception_v2_coco/analysis.txt new file mode 100644 index 000000000..9a61d5c77 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_inception_v2_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100, 4) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/ssd_inception_v2_coco/run_inference.sh b/tftrt/examples/object_detection/models/ssd_inception_v2_coco/run_inference.sh new file mode 100755 index 000000000..7e849bb29 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_inception_v2_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="ssd_inception_v2_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/ssd_mobilenet_v1_coco/analysis.txt b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_coco/analysis.txt new file mode 100644 index 000000000..9a61d5c77 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100, 4) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/ssd_mobilenet_v1_coco/run_inference.sh b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_coco/run_inference.sh new file mode 100755 index 000000000..c28721dd5 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="ssd_mobilenet_v1_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/ssd_mobilenet_v1_fpn_coco/analysis.txt b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_fpn_coco/analysis.txt new file mode 100644 index 000000000..46be39734 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_fpn_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1, -1) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, -1) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/ssd_mobilenet_v1_fpn_coco/run_inference.sh b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_fpn_coco/run_inference.sh new file mode 100755 index 000000000..005d3f0c7 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_mobilenet_v1_fpn_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="ssd_mobilenet_v1_fpn_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/ssd_mobilenet_v2_coco/analysis.txt b/tftrt/examples/object_detection/models/ssd_mobilenet_v2_coco/analysis.txt new file mode 100644 index 000000000..9a61d5c77 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_mobilenet_v2_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100, 4) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/ssd_mobilenet_v2_coco/run_inference.sh b/tftrt/examples/object_detection/models/ssd_mobilenet_v2_coco/run_inference.sh new file mode 100755 index 000000000..3f17eafb7 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_mobilenet_v2_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="ssd_mobilenet_v2_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/ssd_resnet_50_fpn_coco/analysis.txt b/tftrt/examples/object_detection/models/ssd_resnet_50_fpn_coco/analysis.txt new file mode 100644 index 000000000..9a61d5c77 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_resnet_50_fpn_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100, 4) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/ssd_resnet_50_fpn_coco/run_inference.sh b/tftrt/examples/object_detection/models/ssd_resnet_50_fpn_coco/run_inference.sh new file mode 100755 index 000000000..ad5278098 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssd_resnet_50_fpn_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="ssd_resnet_50_fpn_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/models/ssdlite_mobilenet_v2_coco/analysis.txt b/tftrt/examples/object_detection/models/ssdlite_mobilenet_v2_coco/analysis.txt new file mode 100644 index 000000000..9a61d5c77 --- /dev/null +++ b/tftrt/examples/object_detection/models/ssdlite_mobilenet_v2_coco/analysis.txt @@ -0,0 +1,27 @@ + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input'] tensor_info: + dtype: DT_UINT8 + shape: (8, -1, -1, 3) + name: image_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['boxes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100, 4) + name: detection_boxes:0 + outputs['classes'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_classes:0 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (8) + name: num_detections:0 + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (8, 100) + name: detection_scores:0 + Method name is: tensorflow/serving/predict diff --git a/tftrt/examples/object_detection/models/ssdlite_mobilenet_v2_coco/run_inference.sh b/tftrt/examples/object_detection/models/ssdlite_mobilenet_v2_coco/run_inference.sh new file mode 100755 index 000000000..1bda1f18b --- /dev/null +++ b/tftrt/examples/object_detection/models/ssdlite_mobilenet_v2_coco/run_inference.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." + +bash ${BASE_DIR}/base_run_inference.sh --model_name="ssdlite_mobilenet_v2_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/object_detection.py b/tftrt/examples/object_detection/object_detection.py index 94aa9ff49..1a385c2de 100644 --- a/tftrt/examples/object_detection/object_detection.py +++ b/tftrt/examples/object_detection/object_detection.py @@ -186,6 +186,12 @@ def evaluate_model(self, predictions, expected, bypass_data_to_eval): # write coco detections to file tmp_dir = "/tmp/tmp_detection_results" + + try: + shutil.rmtree(tmp_dir) + except FileNotFoundError: + pass + os.makedirs(tmp_dir) coco_detections_path = os.path.join(tmp_dir, 'coco_detections.json') diff --git a/tftrt/examples/object_detection/scripts/faster_rcnn_resnet50_coco.sh b/tftrt/examples/object_detection/scripts/faster_rcnn_resnet50_coco.sh deleted file mode 100755 index e2f35ae81..000000000 --- a/tftrt/examples/object_detection/scripts/faster_rcnn_resnet50_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="faster_rcnn_resnet50_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/ssd_inception_v2_coco.sh b/tftrt/examples/object_detection/scripts/ssd_inception_v2_coco.sh deleted file mode 100755 index aef3fc1b0..000000000 --- a/tftrt/examples/object_detection/scripts/ssd_inception_v2_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="ssd_inception_v2_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/ssd_mobilenet_v1_coco.sh b/tftrt/examples/object_detection/scripts/ssd_mobilenet_v1_coco.sh deleted file mode 100755 index 8bedd3f59..000000000 --- a/tftrt/examples/object_detection/scripts/ssd_mobilenet_v1_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="ssd_mobilenet_v1_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/ssd_mobilenet_v1_fpn_coco.sh b/tftrt/examples/object_detection/scripts/ssd_mobilenet_v1_fpn_coco.sh deleted file mode 100755 index c9bc53738..000000000 --- a/tftrt/examples/object_detection/scripts/ssd_mobilenet_v1_fpn_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="ssd_mobilenet_v1_fpn_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/ssd_mobilenet_v2_coco.sh b/tftrt/examples/object_detection/scripts/ssd_mobilenet_v2_coco.sh deleted file mode 100755 index 860baf73a..000000000 --- a/tftrt/examples/object_detection/scripts/ssd_mobilenet_v2_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="ssd_mobilenet_v2_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/ssd_resnet_50_fpn_coco.sh b/tftrt/examples/object_detection/scripts/ssd_resnet_50_fpn_coco.sh deleted file mode 100755 index 9c8bfd902..000000000 --- a/tftrt/examples/object_detection/scripts/ssd_resnet_50_fpn_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="ssd_resnet_50_fpn_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/object_detection/scripts/ssdlite_mobilenet_v2_coco.sh b/tftrt/examples/object_detection/scripts/ssdlite_mobilenet_v2_coco.sh deleted file mode 100755 index 2835c7586..000000000 --- a/tftrt/examples/object_detection/scripts/ssdlite_mobilenet_v2_coco.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -bash ${SCRIPT_DIR}/base_script.sh --model_name="ssdlite_mobilenet_v2_coco" ${@} \ No newline at end of file diff --git a/tftrt/examples/transformers/base_run_inference.sh b/tftrt/examples/transformers/base_run_inference.sh index 8e69bb40c..b13cc6fea 100755 --- a/tftrt/examples/transformers/base_run_inference.sh +++ b/tftrt/examples/transformers/base_run_inference.sh @@ -47,7 +47,7 @@ do shift # Remove --input_saved_model_dir= from processing ;; --use_xla_auto_jit) - TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=--tf_xla_auto_jit=2" + TF_AUTO_JIT_XLA_FLAG="TF_XLA_FLAGS=\"--tf_xla_auto_jit=2 --tf_xla_cpu_global_jit\"" shift # Remove --use_xla_auto_jit from processing ;; --vocab_size=*) diff --git a/tftrt/examples/transformers/models/bert_base_uncased/run_inference.sh b/tftrt/examples/transformers/models/bert_base_uncased/run_inference.sh index 616cc2912..d0e7afb8d 100755 --- a/tftrt/examples/transformers/models/bert_base_uncased/run_inference.sh +++ b/tftrt/examples/transformers/models/bert_base_uncased/run_inference.sh @@ -2,4 +2,4 @@ BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." -bash ${BASE_DIR}/../../base_run_inference.sh --model_name="bert_base_uncased" ${@} +bash ${BASE_DIR}/base_run_inference.sh --model_name="bert_base_uncased" ${@} \ No newline at end of file