diff --git a/scripts/ivas_conformance/runConformance.py b/scripts/ivas_conformance/runConformance.py index b69659880487583514c7439325172c2756826a18..4723b7a918e6119fe8e267b812a66764aea7064d 100644 --- a/scripts/ivas_conformance/runConformance.py +++ b/scripts/ivas_conformance/runConformance.py @@ -29,6 +29,7 @@ submitted to and settled by the final, binding jurisdiction of the courts of Mun accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and the United Nations Convention on Contracts on the International Sales of Goods. """ + import argparse import os import platform @@ -474,9 +475,9 @@ class MLDConformance: def genEncoderReferences(self, tag: str, encPytestTag: str): # RUN ENCODER'S OUTPUT DECODED WITH REF DECODER testDesc = self.TestDesc[tag][encPytestTag] - assert isinstance( - testDesc, BitstreamTestDescriptor - ), f"Expected bitstream test descriptor for {tag}" + assert isinstance(testDesc, BitstreamTestDescriptor), ( + f"Expected bitstream test descriptor for {tag}" + ) # Decode the encoded output with Reference IVAS decoder if tag == "ENC": @@ -510,9 +511,9 @@ class MLDConformance: def analyseWavOutputTest(self, tag: str, dutPytestTag: str): testDesc = self.TestDesc[tag][dutPytestTag] - assert isinstance( - testDesc, TestDescriptor - ), f"Expected pcm test descriptor for {tag}" + assert isinstance(testDesc, TestDescriptor), ( + f"Expected pcm test descriptor for {tag}" + ) if self.args.be_test: DUTmdFiles = self.getMDfileList(outFile=testDesc.dutOutput) @@ -537,9 +538,9 @@ class MLDConformance: def analyseOneEncoderTest(self, tag: str, encPytestTag: str): testDesc = self.TestDesc[tag][encPytestTag] - assert isinstance( - testDesc, BitstreamTestDescriptor - ), f"Expected bitstream test descriptor for {tag}" + assert isinstance(testDesc, BitstreamTestDescriptor), ( + f"Expected bitstream test descriptor for {tag}" + ) if self.args.be_test: self.beTest( @@ -566,9 +567,9 @@ class MLDConformance: def analyseOneIsarEncoderTest(self, tag: str, pytestTag: str): testDesc = self.TestDesc[tag][pytestTag] - assert isinstance( - testDesc, BitstreamTestDescriptor - ), f"Expected bitstream test descriptor for {tag}" + assert isinstance(testDesc, BitstreamTestDescriptor), ( + f"Expected bitstream test descriptor for {tag}" + ) if self.args.be_test: self.beTest( @@ -699,7 +700,7 @@ class MLDConformance: self.totalTests = len(selectedTests) print( - f"Executing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Executing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: with Pool() as pool: @@ -728,7 +729,7 @@ class MLDConformance: self.totalTests = len(selectedTests) print( - f"Analysing tests for {tag} {'Filter='+self.filter if self.filter else ''} ({self.totalTests} tests)" + f"Analysing tests for {tag} {'Filter=' + self.filter if self.filter else ''} ({self.totalTests} tests)" ) if not self.args.no_multi_processing: with Pool() as pool: @@ -787,9 +788,9 @@ class MLDConformance: with tempfile.TemporaryDirectory() as tmpdir: refSamples, fsR = readfile(refFile, outdtype="float") dutSamples, fsD = readfile(dutFile, outdtype="float") - assert ( - refSamples.shape[1] == dutSamples.shape[1] - ), "No of channels mismatch if ref vs cut" + assert refSamples.shape[1] == dutSamples.shape[1], ( + "No of channels mismatch if ref vs cut" + ) maxDiff, rmsdB, beSamplesPercent = self.getSampleStats( refSamples, dutSamples ) @@ -865,13 +866,33 @@ class MLDConformance: with open(self.BEcsv[tag], "a") as f: f.write(f"{pytestTag}, {BE_flag}\n") - for i in range(0, len(DUTmdFileList)): - if os.path.exists(DUTmdFileList[i]): - BE_flag = 0 - if not filecmp.cmp(REFmdFileList[i], DUTmdFileList[i]): - BE_flag = 1 - with open(self.BEcsv[tag], "a") as f: - f.write(f"{DUTmdFileList[i]}, {BE_flag}\n") + assert len(DUTmdFileList) == len(REFmdFileList) + for refMDfile, dutMDfile in zip(REFmdFileList, DUTmdFileList): + ref_exist = os.path.exists(refMDfile) + dut_exist = os.path.exists(dutMDfile) + + assert ref_exist == dut_exist + if not ref_exist: + continue + + _, ref_suffix = os.path.splitext(refMDfile) + _, dut_suffix = os.path.splitext(dutMDfile) + assert ref_suffix == dut_suffix + + BE_flag = 0 + if ref_suffix == ".csv": + with open(refMDfile, "r") as f_ref: + with open(dutMDfile, "r") as f_dut: + ref_content = f_ref.read() + dut_content = f_dut.read() + BE_flag = int(not (ref_content == dut_content)) + elif ref_suffix == ".met": + BE_flag = int(not filecmp.cmp(refMDfile, dutMDfile)) + else: + assert False, f"MD file has unknown suffix {ref_suffix}" + + with open(self.BEcsv[tag], "a") as f: + f.write(f"{dutMDfile}, {BE_flag}\n") def doBEanalysis(self, selectTag="all"): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] @@ -902,9 +923,13 @@ class MLDConformance: dutMLD = mldCutWithTags["MLD"][indDut] diff = dutMLD - refMLD if diff.max() > threshold: - print(f"\033[91mMLD Corridor failed for {tag} with max MLD diff of {diff.max()} \033[00m") + print( + f"\033[91mMLD Corridor failed for {tag} with max MLD diff of {diff.max()} \033[00m" + ) else: - print(f"\033[92mMLD Corridor passed for {tag} with max MLD diff of {diff.max()} \033[00m") + print( + f"\033[92mMLD Corridor passed for {tag} with max MLD diff of {diff.max()} \033[00m" + ) def doAnalysis(self, selectTag="all", corridor=False): keys = IVAS_Bins.keys() if selectTag == "all" else [selectTag] @@ -968,7 +993,9 @@ class MLDConformance: ) self.computeCorridor(mldRefWithTags, mdlCutWithTags, tag) else: - print(f"\033[91mMissing reference MLD file for {tag} : {refMldFile} \033[00m") + print( + f"\033[91mMissing reference MLD file for {tag} : {refMldFile} \033[00m" + ) if __name__ == "__main__": diff --git a/tests/conformance-test/test_26252.py b/tests/conformance-test/test_26252.py index 395e258e126c0013fcdaa05c14f32af1c7a4189d..183f01e2ea27c96bbe45cad04126e4931886267c 100644 --- a/tests/conformance-test/test_26252.py +++ b/tests/conformance-test/test_26252.py @@ -125,12 +125,45 @@ def test_26252(test_tag, encoder_path, decoder_path, renderer_path, isar_post_re subprocess.run([renderer_path] + rend_opts.split()[1:], check = True) if isar_post_rend_opts: isar_post_rend_opts = replace_paths(isar_post_rend_opts, testv_path, ref_path, cut_path) - subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True) + subprocess.run([isar_post_renderer_path] + isar_post_rend_opts.split()[1:], check = True) diff_opts = replace_paths(diff_opts, testv_path, ref_path, cut_path) - result = True + + diff_in = {".wav": False, ".met": False, ".csv": False, ".192": False, ".bit": False} + for cmd in diff_opts.split(';'): - result = result and filecmp.cmp(cmd.split()[1], cmd.split()[2]) - if not result: - assert False, "Output differs" + file_a = Path(cmd.split()[1]) + file_b = Path(cmd.split()[2]) + + suffix = file_a.suffix + assert suffix == file_b.suffix + assert suffix in diff_in + + # for .csv ISM metadata files, do text-based comparison to not take line endings into account + # everything else (.wav output files and MASA metadata files) is compared as binary files + if suffix == ".csv": + with open(file_a, "r") as f_a: + with open(file_b, "r") as f_b: + a_content = f_a.read() + b_content = f_b.read() + + files_equal = a_content == b_content + else: + files_equal = filecmp.cmp(file_a, file_b) + + if not files_equal: + diff_in[suffix] = True + + if any(diff_in.values()): + result_str = "Output differs in: " + if diff_in[".csv"]: + result_str += "object metadata " + if diff_in[".met"]: + result_str += "MASA metadata " + if diff_in[".wav"]: + result_str += "waveform" + if diff_in[".192"] or diff_in[".bit"]: + result_str += "bitstream" + + pytest.fail(result_str)