Results from R50 GH action on ubuntu-latest

mlcommons · Oct 5, 2024 · 7cbeabd · 7cbeabd
1 parent 1544ec6
commit 7cbeabd
Show file tree

Hide file tree

Showing 9 changed files with 246 additions and 245 deletions.
diff --git a/...gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md b/...gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/README.md
@@ -1,3 +1,3 @@
 | Model    | Scenario   |   Accuracy |   Throughput | Latency (in ms)   |
 |----------|------------|------------|--------------|-------------------|
-| resnet50 | offline    |         76 |       23.047 | -                 |
+| resnet50 | offline    |         76 |       23.153 | -                 |
diff --git a/...x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/README.md b/...x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/README.md
@@ -18,7 +18,7 @@ pip install -U cmind
 
 cm rm cache -f
 
-cm pull repo anandhu-eng@cm4mlops --checkout=ad0b930c1cb125ae9b74ba9b38efea5fa968ee8f
+cm pull repo anandhu-eng@cm4mlops --checkout=d9b2082cb9a9622d07f485348bf054ea576a31f3
 
 cm run script \
 	--tags=run-mlperf,inference,_submission,_short \
@@ -54,4 +54,4 @@ Model Precision: fp32
 `acc`: `76.0`, Required accuracy for closed division `>= 75.6954`
 
 ### Performance Results 
-`Samples per second`: `23.0467`
+`Samples per second`: `23.1526`
diff --git a/...86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/cm-version-info.json b/...86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/cm-version-info.json
@@ -87,15 +87,15 @@
         "script_tags": "detect-os,detect,os,info",
         "script_variations": "",
         "version": "",
-        "parent": "extract-file,3f0b76219d004817 ( path./home/runner/CM/repos/local/cache/844ee4e40baa4661/caffe_ilsvrc12.tar.gz )"
+        "parent": "extract-file,3f0b76219d004817 ( path./home/runner/CM/repos/local/cache/0c3169269b85489d/caffe_ilsvrc12.tar.gz )"
       }
     },
     {
-      "extract,file,_path./home/runner/CM/repos/local/cache/844ee4e40baa4661/caffe_ilsvrc12.tar.gz": {
+      "extract,file,_path./home/runner/CM/repos/local/cache/0c3169269b85489d/caffe_ilsvrc12.tar.gz": {
         "script_uid": "3f0b76219d004817",
         "script_alias": "extract-file",
         "script_tags": "extract,file",
-        "script_variations": "path./home/runner/CM/repos/local/cache/844ee4e40baa4661/caffe_ilsvrc12.tar.gz",
+        "script_variations": "path./home/runner/CM/repos/local/cache/0c3169269b85489d/caffe_ilsvrc12.tar.gz",
         "version": "",
         "parent": "download-and-extract,c67e81a4ce2649f5 ( extract,_wget,_url.https://www.dropbox.com/s/92n2fyej3lzy3s3/caffe_ilsvrc12.tar.gz )"
       }
@@ -127,7 +127,7 @@
         "script_tags": "detect-os,detect,os,info",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -147,7 +147,7 @@
         "script_tags": "detect,cpu,detect-cpu,info",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -167,7 +167,7 @@
         "script_tags": "get,python,python3,get-python,get-python3",
         "script_variations": "",
         "version": "3.12.6",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -227,7 +227,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "onnxruntime",
         "version": "1.19.2",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -267,7 +267,7 @@
         "script_tags": "get,raw,ml-model,resnet50,ml-model-resnet50,image-classification",
         "script_variations": "onnx,fp32",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -317,15 +317,15 @@
         "script_tags": "detect-os,detect,os,info",
         "script_variations": "",
         "version": "",
-        "parent": "extract-file,3f0b76219d004817 ( path./home/runner/CM/repos/local/cache/756f0629594140de/ILSVRC2012_img_val_500.tar )"
+        "parent": "extract-file,3f0b76219d004817 ( path./home/runner/CM/repos/local/cache/945d64a4d6694fe7/ILSVRC2012_img_val_500.tar )"
       }
     },
     {
-      "extract,file,_path./home/runner/CM/repos/local/cache/756f0629594140de/ILSVRC2012_img_val_500.tar": {
+      "extract,file,_path./home/runner/CM/repos/local/cache/945d64a4d6694fe7/ILSVRC2012_img_val_500.tar": {
         "script_uid": "3f0b76219d004817",
         "script_alias": "extract-file",
         "script_tags": "extract,file",
-        "script_variations": "path./home/runner/CM/repos/local/cache/756f0629594140de/ILSVRC2012_img_val_500.tar",
+        "script_variations": "path./home/runner/CM/repos/local/cache/945d64a4d6694fe7/ILSVRC2012_img_val_500.tar",
         "version": "",
         "parent": "download-and-extract,c67e81a4ce2649f5 ( extract,_url.http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar )"
       }
@@ -497,7 +497,7 @@
         "script_tags": "get,dataset,imagenet,ILSVRC,image-classification,preprocessed",
         "script_variations": "NCHW",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -507,7 +507,7 @@
         "script_tags": "get,aux,dataset-aux,image-classification,imagenet-aux",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -587,7 +587,7 @@
         "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -887,7 +887,7 @@
         "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons",
         "script_variations": "",
         "version": "master",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -897,7 +897,7 @@
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
         "script_variations": "",
         "version": "master-git-41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -907,7 +907,7 @@
         "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons",
         "script_variations": "",
         "version": "master-git-41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -967,7 +967,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "package.psutil",
         "version": "6.0.0",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -1027,7 +1027,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "opencv-python",
         "version": "4.10.0.84",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -1087,7 +1087,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "numpy",
         "version": "1.26.4",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -1147,7 +1147,7 @@
         "script_tags": "get,install,generic,generic-python-lib",
         "script_variations": "pycocotools",
         "version": "2.0.8",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     },
     {
@@ -1187,7 +1187,7 @@
         "script_tags": "mlperf,benchmark-mlperf",
         "script_variations": "",
         "version": "",
-        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( cpu,_resnet50,_onnxruntime,_offline,_fp32 )"
+        "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( onnxruntime,_resnet50,_cpu,_offline,_fp32 )"
       }
     }
   ]

diff --git a/...latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/os_info.json b/...latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/os_info.json
@@ -26,5 +26,5 @@
   ],
   "CM_HOST_PLATFORM_FLAVOR": "x86_64",
   "CM_HOST_PYTHON_BITS": "64",
-  "CM_HOST_SYSTEM_NAME": "fv-az654-117"
+  "CM_HOST_SYSTEM_NAME": "fv-az698-410"
 }
diff --git a/...6-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/accuracy/accuracy.txt b/...6-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/accuracy/accuracy.txt
@@ -1,2 +1,2 @@
 accuracy=76.000%, good=380, total=500
-hash=cfb837190c96eb1f31fb3c8740b1cbc31f1f45b6ad37970e8a67748bd626a237
+hash=27b3ede91b586db15aca7af7fc101f37907c64682b6834d73b1511d8389388d2
diff --git a/...cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/accuracy/mlperf_log_accuracy.json b/...cpu-onnxruntime_v1.19.2-default_config/resnet50/offline/accuracy/mlperf_log_accuracy.json
@@ -1,4 +1,36 @@
 [
+{ "seq_id" : 0, "qsl_idx" : 230, "data" : "0000FA42" },
+{ "seq_id" : 1, "qsl_idx" : 119, "data" : "00402144" },
+{ "seq_id" : 2, "qsl_idx" : 189, "data" : "00001244" },
+{ "seq_id" : 3, "qsl_idx" : 383, "data" : "00401B44" },
+{ "seq_id" : 4, "qsl_idx" : 406, "data" : "00009B43" },
+{ "seq_id" : 5, "qsl_idx" : 142, "data" : "00C04044" },
+{ "seq_id" : 6, "qsl_idx" : 291, "data" : "00005644" },
+{ "seq_id" : 7, "qsl_idx" : 87, "data" : "0000E041" },
+{ "seq_id" : 8, "qsl_idx" : 220, "data" : "00003B44" },
+{ "seq_id" : 9, "qsl_idx" : 70, "data" : "0000C343" },
+{ "seq_id" : 10, "qsl_idx" : 135, "data" : "00009543" },
+{ "seq_id" : 11, "qsl_idx" : 496, "data" : "00806144" },
+{ "seq_id" : 12, "qsl_idx" : 371, "data" : "00005744" },
+{ "seq_id" : 13, "qsl_idx" : 77, "data" : "00808943" },
+{ "seq_id" : 14, "qsl_idx" : 206, "data" : "00001243" },
+{ "seq_id" : 15, "qsl_idx" : 427, "data" : "00803444" },
+{ "seq_id" : 16, "qsl_idx" : 85, "data" : "0000F943" },
+{ "seq_id" : 17, "qsl_idx" : 243, "data" : "00001544" },
+{ "seq_id" : 18, "qsl_idx" : 193, "data" : "00C04044" },
+{ "seq_id" : 19, "qsl_idx" : 29, "data" : "00805644" },
+{ "seq_id" : 20, "qsl_idx" : 372, "data" : "00802D44" },
+{ "seq_id" : 21, "qsl_idx" : 499, "data" : "00002F43" },
+{ "seq_id" : 22, "qsl_idx" : 249, "data" : "00805F44" },
+{ "seq_id" : 23, "qsl_idx" : 443, "data" : "0000AF43" },
+{ "seq_id" : 24, "qsl_idx" : 200, "data" : "00003044" },
+{ "seq_id" : 25, "qsl_idx" : 473, "data" : "00003E43" },
+{ "seq_id" : 26, "qsl_idx" : 157, "data" : "00C03E44" },
+{ "seq_id" : 27, "qsl_idx" : 167, "data" : "00802644" },
+{ "seq_id" : 28, "qsl_idx" : 279, "data" : "00406044" },
+{ "seq_id" : 29, "qsl_idx" : 165, "data" : "00C02B44" },
+{ "seq_id" : 30, "qsl_idx" : 326, "data" : "00006A43" },
+{ "seq_id" : 31, "qsl_idx" : 27, "data" : "00002643" },
 { "seq_id" : 64, "qsl_idx" : 153, "data" : "0000E642" },
 { "seq_id" : 65, "qsl_idx" : 282, "data" : "00808D43" },
 { "seq_id" : 66, "qsl_idx" : 216, "data" : "00008041" },
@@ -39,60 +71,29 @@
 { "seq_id" : 101, "qsl_idx" : 203, "data" : "00401444" },
 { "seq_id" : 102, "qsl_idx" : 221, "data" : "00802544" },
 { "seq_id" : 103, "qsl_idx" : 445, "data" : "0080B443" },
-{ "seq_id" : 104, "qsl_idx" : 471, "data" : "00407844" },
-{ "seq_id" : 105, "qsl_idx" : 141, "data" : "0000A443" },
-{ "seq_id" : 106, "qsl_idx" : 349, "data" : "00004343" },
-{ "seq_id" : 107, "qsl_idx" : 397, "data" : "00405544" },
-{ "seq_id" : 108, "qsl_idx" : 412, "data" : "00005144" },
-{ "seq_id" : 109, "qsl_idx" : 318, "data" : "0000CC42" },
-{ "seq_id" : 110, "qsl_idx" : 440, "data" : "00002F43" },
-{ "seq_id" : 111, "qsl_idx" : 137, "data" : "0000A443" },
-{ "seq_id" : 112, "qsl_idx" : 296, "data" : "00002C42" },
-{ "seq_id" : 113, "qsl_idx" : 242, "data" : "00802944" },
-{ "seq_id" : 114, "qsl_idx" : 264, "data" : "00804344" },
-{ "seq_id" : 115, "qsl_idx" : 335, "data" : "0080E743" },
-{ "seq_id" : 116, "qsl_idx" : 105, "data" : "00806244" },
-{ "seq_id" : 117, "qsl_idx" : 273, "data" : "00805744" },
-{ "seq_id" : 118, "qsl_idx" : 117, "data" : "00802144" },
-{ "seq_id" : 119, "qsl_idx" : 68, "data" : "00805944" },
-{ "seq_id" : 120, "qsl_idx" : 306, "data" : "0000803F" },
-{ "seq_id" : 121, "qsl_idx" : 428, "data" : "00003743" },
-{ "seq_id" : 122, "qsl_idx" : 129, "data" : "00808D43" },
-{ "seq_id" : 123, "qsl_idx" : 154, "data" : "00406A44" },
-{ "seq_id" : 124, "qsl_idx" : 235, "data" : "0000E642" },
-{ "seq_id" : 125, "qsl_idx" : 116, "data" : "00801F44" },
-{ "seq_id" : 126, "qsl_idx" : 240, "data" : "00000544" },
-{ "seq_id" : 127, "qsl_idx" : 488, "data" : "00403244" },
-{ "seq_id" : 0, "qsl_idx" : 230, "data" : "0000FA42" },
-{ "seq_id" : 1, "qsl_idx" : 119, "data" : "00402144" },
-{ "seq_id" : 2, "qsl_idx" : 189, "data" : "00001244" },
-{ "seq_id" : 3, "qsl_idx" : 383, "data" : "00401B44" },
-{ "seq_id" : 4, "qsl_idx" : 406, "data" : "00009B43" },
-{ "seq_id" : 5, "qsl_idx" : 142, "data" : "00C04044" },
-{ "seq_id" : 6, "qsl_idx" : 291, "data" : "00005644" },
-{ "seq_id" : 7, "qsl_idx" : 87, "
+{ "seq_i
 
 ...
 
- "seq_id" : 429, "qsl_idx" : 90, "data" : "00008743" },
-{ "seq_id" : 430, "qsl_idx" : 23, "data" : "00C03544" },
-{ "seq_id" : 431, "qsl_idx" : 329, "data" : "00400344" },
-{ "seq_id" : 432, "qsl_idx" : 209, "data" : "0080D743" },
-{ "seq_id" : 433, "qsl_idx" : 4, "data" : "00000144" },
-{ "seq_id" : 434, "qsl_idx" : 493, "data" : "0000D043" },
-{ "seq_id" : 435, "qsl_idx" : 18, "data" : "0000EF43" },
-{ "seq_id" : 436, "qsl_idx" : 343, "data" : "0080FB43" },
-{ "seq_id" : 437, "qsl_idx" : 9, "data" : "0000A643" },
-{ "seq_id" : 438, "qsl_idx" : 44, "data" : "00008043" },
-{ "seq_id" : 439, "qsl_idx" : 352, "data" : "0080F043" },
-{ "seq_id" : 440, "qsl_idx" : 110, "data" : "0000F442" },
-{ "seq_id" : 441, "qsl_idx" : 76, "data" : "00001F43" },
-{ "seq_id" : 442, "qsl_idx" : 364, "data" : "00C05844" },
-{ "seq_id" : 443, "qsl_idx" : 258, "data" : "00002144" },
-{ "seq_id" : 444, "qsl_idx" : 229, "data" : "00403944" },
-{ "seq_id" : 445, "qsl_idx" : 345, "data" : "0080F743" },
-{ "seq_id" : 446, "qsl_idx" : 11, "data" : "00008F43" },
-{ "seq_id" : 447, "qsl_idx" : 470, "data" : "00001C42" },
+id" : 397, "qsl_idx" : 486, "data" : "00008F43" },
+{ "seq_id" : 398, "qsl_idx" : 69, "data" : "00405244" },
+{ "seq_id" : 399, "qsl_idx" : 28, "data" : "00005C42" },
+{ "seq_id" : 400, "qsl_idx" : 322, "data" : "00008C43" },
+{ "seq_id" : 401, "qsl_idx" : 340, "data" : "00807544" },
+{ "seq_id" : 402, "qsl_idx" : 451, "data" : "00803744" },
+{ "seq_id" : 403, "qsl_idx" : 82, "data" : "00004544" },
+{ "seq_id" : 404, "qsl_idx" : 95, "data" : "00403044" },
+{ "seq_id" : 405, "qsl_idx" : 411, "data" : "00807944" },
+{ "seq_id" : 406, "qsl_idx" : 100, "data" : "0080EC43" },
+{ "seq_id" : 407, "qsl_idx" : 254, "data" : "00C07544" },
+{ "seq_id" : 408, "qsl_idx" : 250, "data" : "00407744" },
+{ "seq_id" : 409, "qsl_idx" : 374, "data" : "00005043" },
+{ "seq_id" : 410, "qsl_idx" : 124, "data" : "00009C43" },
+{ "seq_id" : 411, "qsl_idx" : 360, "data" : "00009643" },
+{ "seq_id" : 412, "qsl_idx" : 391, "data" : "0080EB43" },
+{ "seq_id" : 413, "qsl_idx" : 460, "data" : "00006B44" },
+{ "seq_id" : 414, "qsl_idx" : 146, "data" : "00808443" },
+{ "seq_id" : 415, "qsl_idx" : 378, "data" : "00004743" },
 { "seq_id" : 480, "qsl_idx" : 19, "data" : "00400144" },
 { "seq_id" : 481, "qsl_idx" : 297, "data" : "00008042" },
 { "seq_id" : 482, "qsl_idx" : 64, "data" : "00805244" },