Issue in JSON generation scripts (some test suites are badly generated)

fluster/codec.py

@@ -47,7 +47,7 @@ class OutputFormat(Enum):
     YUV444P12LE = "yuv444p12le"
     YUV444P16LE = "yuv444p16le"
     GRAY = "gray"
+    GRAY10LE = "gray10le"
     GRAY12LE = "gray12le"
     GRAY16LE = "gray16le"
     GBRP14LE = "gbrp14le"
-    UNKNOWN = "Unknown"

fluster/utils.py

@@ -162,12 +162,16 @@ def find_by_ext(
                 if not excluded:
                     candidates.append(filepath)
 
-    # Prioritize files with 'L0' in the name only for JCT-VC-SHVC
     for candidate in candidates:
+        # Prioritize files with 'L0' in the name (for JCT-VC-SHVC)
         if "L0" in candidate.upper():
             return candidate
+        # Prioritize files with 'norpt' in the name (for JVT-AVC_V1)
+        # Special case only for CVSEFDFT3_Sony_E.zip and CVSE3_Sony_H.zip
+        if "norpt" in candidate.lower():
+            return candidate
 
-    # If no file with 'L0' is found, return the first candidate
+    # If none of the above 2 cases is fulfilled, return the first candidate
     return candidates[0] if candidates else None
 
 

scripts/gen_jct_vc.py

@@ -159,7 +159,7 @@ def generate(self, download, jobs):
                 pix_fmt = result[0]
                 try:
                     test_vector.output_format = OutputFormat[pix_fmt.upper()]
-                except KeyError as e:
+                except KeyError as key_err:
                     exceptions = {
                         # Feature: Test unequal luma and chroma bitdepth
                         # setting. The luma bitdepth is higher than the chroma
@@ -182,7 +182,7 @@ def generate(self, download, jobs):
                     if test_vector.name in exceptions.keys():
                         test_vector.output_format = exceptions[test_vector.name]
                     else:
-                        raise e
+                        raise key_err
 
             self._fill_checksum_h265(test_vector, dest_dir)
 
@@ -212,7 +212,7 @@ def _fill_checksum_h265(self, test_vector, dest_dir):
             # Example 6:
             # 9cab6bcd74491062a8523b5a7ff6a540  CCP_8bit_RExt_QCOM.bin
             # f3e914fccdb820eac85f46642ea0e168  CCP_8bit_RExt_QCOM.gbr
-            regex = re.compile(r"([a-fA-F0-9]{{32,}}).*\.(yuv|rgb|gbr)")
+            regex = re.compile(rf"([a-fA-F0-9]{{32,}}).*\.(yuv|rgb|gbr)")
             lines = checksum_file.readlines()
             # Filter out empty lines and lines that start with "#"
             filtered_lines = [line.strip() for line in lines if line.strip() and not line.strip().startswith("#")]

scripts/gen_jvet.py

@@ -23,6 +23,7 @@
 import sys
 import urllib.request
 import multiprocessing
+from subprocess import CalledProcessError
 
 # pylint: disable=wrong-import-position
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -134,18 +135,39 @@ def generate(self, download, jobs):
                 raise Exception(f"Bitstream file not found in {dest_dir}")
             test_vector.source_checksum = utils.file_checksum(dest_path)
             if self.use_ffprobe:
-                ffprobe = utils.normalize_binary_cmd('ffprobe')
-                command = [ffprobe, '-v', 'error', '-select_streams', 'v:0',
-                           '-show_entries', 'stream=pix_fmt', '-of',
-                           'default=nokey=1:noprint_wrappers=1',
-                           absolute_input_path]
-
-                result = utils.run_command_with_output(command).splitlines()
-                pix_fmt = result[0]
                 try:
+                    ffprobe = utils.normalize_binary_cmd('ffprobe')
+                    command = [ffprobe, '-v', 'error', '-strict', '-2',
+                               '-select_streams', 'v:0',
+                               '-show_entries', 'stream=pix_fmt', '-of',
+                               'default=nokey=1:noprint_wrappers=1',
+                               absolute_input_path]
+
+                    result = utils.run_command_with_output(command).splitlines()
+                    pix_fmt = result[0]
                     test_vector.output_format = OutputFormat[pix_fmt.upper()]
-                except KeyError as e:
-                    raise e
+                except KeyError as key_err:
+                    exceptions = {
+                        # All below test vectors need to be analysed with respect
+                        # to output format, for now remains undetermined
+                        "VPS_C_ERICSSON_1": OutputFormat.NONE
+                    }
+                    if test_vector.name in exceptions.keys():
+                        test_vector.output_format = exceptions[test_vector.name]
+                    else:
+                        raise key_err
+                except CalledProcessError as proc_err:
+                    exceptions = {
+                        # All below test vectors need cause ffprobe to crash
+                        "MNUT_A_Nokia_3": OutputFormat.NONE,
+                        "MNUT_B_Nokia_2": OutputFormat.NONE,
+                        "SUBPIC_C_ERICSSON_1": OutputFormat.NONE,
+                        "SUBPIC_D_ERICSSON_1": OutputFormat.NONE
+                    }
+                    if test_vector.name in exceptions.keys():
+                        test_vector.output_format = exceptions[test_vector.name]
+                    else:
+                        raise proc_err
 
         test_suite.to_json_file(output_filepath)
         print("Generate new test suite: " + test_suite.name + ".json")

scripts/gen_jvt.py

@@ -156,8 +156,29 @@ def generate(self, download, jobs):
                 pix_fmt = result[0]
                 try:
                     test_vector.output_format = OutputFormat[pix_fmt.upper()]
-                except KeyError as e:
-                    raise e
+                except KeyError as key_err:
+                    exceptions = {
+                        # All below test vectors from JVT-Professional_profiles_V1
+                        # need to be analysed with respect to output format,
+                        # for now it remains undetermined
+                        "PPCV444I4_Mitsubishi_A": OutputFormat.NONE,
+                        "PPCV444I5_Mitsubishi_A": OutputFormat.NONE,
+                        "PPCV444I6_Mitsubishi_A": OutputFormat.NONE,
+                        "PPCV444I7_SejongUniv_A": OutputFormat.NONE,
+                        "PPH444I4_Mitsubishi_A": OutputFormat.NONE,
+                        "PPH444I5_Mitsubishi_A": OutputFormat.NONE,
+                        "PPH444I6_Mitsubishi_A": OutputFormat.NONE,
+                        "PPH444I7_SejongUniv_A": OutputFormat.NONE,
+                        "PPH444P10_SejongUniv_A": OutputFormat.NONE,
+                        "PPH444P6_Mitsubishi_A": OutputFormat.NONE,
+                        "PPH444P7_Mitsubishi_A": OutputFormat.NONE,
+                        "PPH444P8_Mitsubishi_A": OutputFormat.NONE,
+                        "PPH444P9_Mitsubishi_A": OutputFormat.NONE
+                    }
+                    if test_vector.name in exceptions.keys():
+                        test_vector.output_format = exceptions[test_vector.name]
+                    else:
+                        raise key_err
 
             if self.name != "Professional_profiles":  # result md5 generated from h264_reference_decoder
                 if self.name == "SVC":  # result md5 generated for different Lines (L0, L1...)

test_suites/h264/JVT-Professional_profiles_V1.json

@@ -32,31 +32,31 @@
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPCV444I4_Mitsubishi_A.zip",
             "source_checksum": "078604ffd6509d9bfb9c6cae121188d8",
             "input_file": "PPCV444I4_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "6ddc08441ca450ae94c661e4978b777d"
         },
         {
             "name": "PPCV444I5_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPCV444I5_Mitsubishi_A.zip",
             "source_checksum": "c7a722cc6173be92c4ec170faa4d7ab7",
             "input_file": "PPCV444I5_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "679ca1edd2902c500fc137d7fdca3954"
         },
         {
             "name": "PPCV444I6_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPCV444I6_Mitsubishi_A.zip",
             "source_checksum": "08c7439b4fccb77f5e5e9cd8b4ad59a3",
             "input_file": "PPCV444I6_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "6ddc08441ca450ae94c661e4978b777d"
         },
         {
             "name": "PPCV444I7_SejongUniv_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPCV444I7_SejongUniv_A.zip",
             "source_checksum": "410d50227f11871c334721e95f450fc0",
             "input_file": "PPCV444I-7.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "70d81a76c8722f28917e2ea2307f1988"
         },
         {
@@ -200,39 +200,39 @@
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444I4_Mitsubishi_A.zip",
             "source_checksum": "e2b72b15c00cce5fe58684ab6b75f25c",
             "input_file": "PPH444I4_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "65f523ebe1fd966b1f86d2b3e415d7d5"
         },
         {
             "name": "PPH444I5_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444I5_Mitsubishi_A.zip",
             "source_checksum": "f7901b3579c43f7681b44b37ccbe4a20",
             "input_file": "PPH444I5_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "7362fc0ab046c7f0434f1cc4f26dbeed"
         },
         {
             "name": "PPH444I6_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444I6_Mitsubishi_A.zip",
             "source_checksum": "86950ac7efffcbc8dd9b56eaed37dedb",
             "input_file": "PPH444I6_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "65f523ebe1fd966b1f86d2b3e415d7d5"
         },
         {
             "name": "PPH444I7_SejongUniv_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444I7_SejongUniv_A.zip",
             "source_checksum": "86d97c2fde74f7172f91a3e148db25c4",
             "input_file": "PPH444I-7.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "70d81a76c8722f28917e2ea2307f1988"
         },
         {
             "name": "PPH444P10_SejongUniv_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444P10_SejongUniv_A.zip",
             "source_checksum": "72273f7382842423059666d67fe57fb7",
             "input_file": "PPH444P-10.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "70d81a76c8722f28917e2ea2307f1988"
         },
         {
@@ -280,31 +280,31 @@
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444P6_Mitsubishi_A.zip",
             "source_checksum": "28e7b3772d79d90b648f4382f0a782c4",
             "input_file": "PPH444P6_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "80a3e16e08a662e5b784aa1380c048e1"
         },
         {
             "name": "PPH444P7_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444P7_Mitsubishi_A.zip",
             "source_checksum": "27e193897b21a9b732c105edd9c15bad",
             "input_file": "PPH444P7_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "2de6c4b551178c74aca5375f95bb15f8"
         },
         {
             "name": "PPH444P8_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444P8_Mitsubishi_A.zip",
             "source_checksum": "c0054b570abb81f036657bcf1353344a",
             "input_file": "PPH444P8_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "42c6e811b6386158811a8a10fbf72e46"
         },
         {
             "name": "PPH444P9_Mitsubishi_A",
             "source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/Professional_profiles/PPH444P9_Mitsubishi_A.zip",
             "source_checksum": "1d92115ba2820084e44f7c62cb50333f",
             "input_file": "PPH444P9_Mitsubishi_A.264",
-            "output_format": "Unknown",
+            "output_format": "None",
             "result": "3a77d0018ccb8784e298269a0c2e9aa7"
         }
     ]

test_suites/h265/JCT-VC-SHVC.json

@@ -489,7 +489,7 @@
             "source_checksum": "ca8c6b57a3966a9831d3d4cdba00242c",
             "input_file": "SREXT_A_FUJITSU_1.bit",
             "output_format": "yuv420p",
-            "result": "fa16e88f8dfc2b537482564dbc357738"
+            "result": "00bdc05491a4fdd93890af779cb455a9"
         },
         {
             "name": "SREXT_B_FUJITSU_1",
@@ -513,7 +513,7 @@
             "source_checksum": "278730c33286d5817b7688d2dd7acf71",
             "input_file": "SREXT_D_FUJITSU_1.bit",
             "output_format": "yuv420p",
-            "result": "fa16e88f8dfc2b537482564dbc357738"
+            "result": "00bdc05491a4fdd93890af779cb455a9"
         },
         {
             "name": "SREXT_E_FUJITSU_1",