Merge branch 'ci/compare-to-input' into 'main' (299c9e92) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+40 −1

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@ variables:
		- 'ivas-conformance'
		- 'ivas-conformance-linux'
		- 'check-float-reference'
		- 'test-branch-vs-input-passthrough'
		GIT_CLEAN_FLAGS: -ffdxq
		TESTCASE_TIMEOUT_STV_SANITIZERS: 180
		TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200
		@@ -69,7 +70,9 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-float-reference'
		variables:
		IVAS_PIPELINE_NAME: 'check-float-reference: $CI_COMMIT_BRANCH'

		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
		variables:
		IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH'

		stages:
		- .pre
		@@ -265,6 +268,8 @@ stages:
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'check-float-reference'
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
		when: never
		- when: on_success

		.rules-merge-request:
		@@ -1639,6 +1644,40 @@ test-long-self-test:
		junit:
		- report-junit-ltv.xml

		test-branch-vs-input-passthrough:
		tags:
		- ivas-linux
		stage: compare
		timeout: "30 minutes" # TBD
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'
		allow_failure:
		exit_codes:
		- 123
		script:
		- *print-common-info
		- make -j

		- exit_code=0
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true
		- python3 scripts/parse_xml_report.py report-junit.xml report.csv
		- if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
		- if [ $exit_code -eq 1 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi
		- exit 0

		artifacts:
		name: "test-branch-vs-input-passthrough--sha-$CI_COMMIT_SHORT_SHA--results"
		when: always
		expire_in: 4 mos
		paths:
		- report-junit.xml
		- report.html
		- report.csv
		expose_as: "test-branch-vs-input-passthrough results"
		reports:
		junit:
		- report-junit.xml

		# ---------------------------------------------------------------
		# Scheduled jobs on main

scripts/parse_xml_report.py

+14 −3

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@ EVS_FORMATS = {
		"EVS_JBM_dec": r"Readme_JBM_dec",
		}

		NO_FORMATS = {"Default": r".*"}

		IVAS_CATEGORIES = {
		"Normal operation": r".*",
		@@ -48,6 +49,8 @@ EVS_CATEGORIES = {
		"JBM": r"JBM",
		}

		NO_CATEGORIES = {"N/A": r".*"}

		# Main routine
		if __name__ == "__main__":
		parser = argparse.ArgumentParser(
		@@ -64,15 +67,23 @@ if __name__ == "__main__":
		action="store_true",
		help="Parse using EVS 26.444 formats",
		)
		parser.add_argument(
		"--skip_formats",
		action="store_true",
		help="Parse without formats and categories. Suitable for general tests which do not match the IVAS categories.",
		)
		args = parser.parse_args()
		xml_report = args.xml_report
		csv_file = args.csv_file
		FORMATS = IVAS_FORMATS
		CATEGORIES = IVAS_CATEGORIES
		if args.evs:
		FORMATS = EVS_FORMATS
		CATEGORIES = EVS_CATEGORIES
		else:
		FORMATS = IVAS_FORMATS
		CATEGORIES = IVAS_CATEGORIES
		if args.skip_formats:
		FORMATS = NO_FORMATS
		CATEGORIES = NO_CATEGORIES

		tree = ElementTree.parse(xml_report)

		testsuite = tree.find(".//testsuite")

tests/cmp_pcm.py

+14 −1

Original line number	Diff line number	Diff line
		@@ -59,7 +59,18 @@ def cmp_pcm(
		# In case of wav input, override the nchannels with the one from the wav header
		nchannels = s1.shape[1]

		if s1.shape != s2.shape and not allow_differing_lengths:
		# In case number of channels do not match, fail already now. Could happen in case of
		# comparison to input with for a non-passthrough mode.
		if s1.shape[1] != s2.shape[1]:
		reason = "FAIL: Number of channels differ."
		return 1, reason

		if allow_differing_lengths:
		# to allow for MLD comparison, pad shorter file
		max_len = max(s1.shape[0], s2.shape[0])
		s1 = np.pad(s1,((0,max_len - s1.shape[0]),(0,0)),mode='constant',constant_values=0)
		s2 = np.pad(s2,((0,max_len - s2.shape[0]),(0,0)),mode='constant',constant_values=0)
		elif s1.shape != s2.shape:
		print(
		f"file size in samples: file 1 = {s1.shape[0]},",
		f"file 2 = {s2.shape[0]}",
		@@ -175,6 +186,8 @@ if __name__ == "__main__":
		parser.add_argument("--get_mld", action="store_true")
		parser.add_argument("--mld_lim", type=float, default=0, dest="mld_lim")
		parser.add_argument("--get_odg", action="store_true")
		parser.add_argument("--get_ssnr", action="store_true")
		parser.add_argument("--allow_differing_lengths", action="store_true")
		args = parser.parse_args()

		result, msg = cmp_pcm(**vars(args))

tests/codec_be_on_mr_nonselection/test_param_file.py

+53 −6

Original line number	Diff line number	Diff line
		@@ -39,6 +39,7 @@ import platform
		from pathlib import Path
		from subprocess import run
		import pytest
		import re
		import numpy as np

		from tests.cmp_pcm import cmp_pcm
		@@ -51,6 +52,17 @@ from tests.constants import (
		SCRIPTS_DIR,
		)

		PASSTHROUGH_CONF = [
		(r"[\s\S]*", "EXT"), # Any input with EXT output is pass-through
		(r"-stereo", "STEREO"),
		(r"-sba\s?[-+]?1", "FOA"),
		(r"-sba\s?[-+]?2", "HOA2"),
		(r"-sba\s?[-+]?3", "HOA3"),
		(r"-mc\s5_1\s", "5_1"),
		(r"-mc\s7_1\s", "7_1"),
		(r"-mc\s5_1_4", "5_1_4"),
		(r"-mc\s7_1_4", "7_1_4"),
		]

		VALID_DEC_OUTPUT_CONF = [
		"MONO",
		@@ -161,9 +173,22 @@ def test_param_file_tests(
		get_ssnr,
		get_enc_stats,
		get_odg,
		compare_to_input,
		):
		enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]

		# If compare_to_input is set, only run pass-through test cases
		if compare_to_input:
		passthrough = [
		(a, b)
		for (a, b) in PASSTHROUGH_CONF
		if re.search(a, enc_opts) and b in dec_opts.split()
		]
		if len(passthrough) != 1:
		pytest.skip(
		"All non-passthrough modes are skipped when --compare-to-input is set"
		)

		tag_str = convert_test_string_to_tag(test_tag)

		# evaluate encoder options
		@@ -378,7 +403,7 @@ def test_param_file_tests(
		# set to false per default even if this is no JBM case - makes later check for failure easier
		tracefile_last_rtp_numbers_differ = False

		if len(tracefile_dec) > 0:
		if len(tracefile_dec) > 0 and not compare_to_input:
		dut_tracefile_dec = f"{dut_base_path}/param_file/dec/{tracefile_dec}"
		ref_tracefile_dec = f"{reference_path}/param_file/dec/{tracefile_dec}"

		@@ -399,9 +424,17 @@ def test_param_file_tests(
		# shift differences between the two signals - cut longer signal to shorter size
		allow_differing_lengths = not tracefile_last_rtp_numbers_differ

		if compare_to_input:
		# If comparing to input, set input as reference and allow different length
		ref_file = testv_file
		allow_differing_lengths = True
		input_md = [x for x in enc_split if "csv" in x]
		else:
		ref_file = ref_output_file

		fs = int(sampling_rate) * 1000
		output_differs, reason = cmp_pcm(
		ref_output_file,
		ref_file,
		dut_output_file,
		output_config,
		fs,
		@@ -414,7 +447,6 @@ def test_param_file_tests(
		ref_jbm_tf=ref_tracefile_dec,
		cut_jbm_tf=dut_tracefile_dec,
		)
		md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)

		cmp_result_msg += reason

		@@ -423,9 +455,24 @@ def test_param_file_tests(
		record_property(k, v)

		metadata_differs = False
		for md_file in md_out_files:
		dut_metadata_file = Path(f"{dut_base_path}/param_file/dec/{md_file}")
		ref_metadata_file = Path(f"{reference_path}/param_file/dec/{md_file}")

		md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)

		if compare_to_input:
		md_file_pairs = [
		(Path(f"{dut_base_path}/param_file/dec/{md_file}"), md_in_file)
		for (md_file, md_in_file) in zip(md_out_files, input_md)
		]
		else:
		md_file_pairs = [
		(
		Path(f"{dut_base_path}/param_file/dec/{md_file}"),
		Path(f"{reference_path}/param_file/dec/{md_file}"),
		)
		for md_file in md_out_files
		]
		for dut_metadata_file, ref_metadata_file in md_file_pairs:
		md_file = os.path.basename(dut_metadata_file)
		try:
		if not filecmp.cmp(dut_metadata_file, ref_metadata_file):
		print("Output metadata differs for file: " + md_file)

tests/codec_be_on_mr_nonselection/test_sba.py

+27 −5

Original line number	Diff line number	Diff line
		@@ -114,6 +114,7 @@ def test_pca_enc(
		get_ssnr,
		get_odg,
		get_enc_stats,
		compare_to_input,
		):
		pca = True
		bitrate = "256000"
		@@ -165,6 +166,7 @@ def test_pca_enc(
		sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -187,6 +189,7 @@ def test_pca_enc(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		compare_to_input=compare_to_input,
		)


		@@ -223,6 +226,7 @@ def test_sba_enc_system(
		get_ssnr,
		get_odg,
		get_enc_stats,
		compare_to_input,
		):

		plc_pattern = None
		@@ -333,6 +337,7 @@ def test_sba_enc_system(
		sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -355,6 +360,7 @@ def test_sba_enc_system(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		compare_to_input=compare_to_input,
		)


		@@ -383,6 +389,7 @@ def test_spar_hoa2_enc_system(
		get_ssnr,
		get_odg,
		get_enc_stats,
		compare_to_input,
		):
		sampling_rate = "48"
		pca = False
		@@ -467,6 +474,7 @@ def test_spar_hoa2_enc_system(
		sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -489,6 +497,7 @@ def test_spar_hoa2_enc_system(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		compare_to_input=compare_to_input,
		)


		@@ -517,6 +526,7 @@ def test_spar_hoa3_enc_system(
		get_ssnr,
		get_odg,
		get_enc_stats,
		compare_to_input,
		):
		sampling_rate = "48"
		pca = False
		@@ -595,6 +605,7 @@ def test_spar_hoa3_enc_system(
		sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -617,6 +628,7 @@ def test_spar_hoa3_enc_system(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		compare_to_input=compare_to_input,
		)


		@@ -649,6 +661,7 @@ def test_sba_enc_BWforce_system(
		get_ssnr,
		get_odg,
		get_enc_stats,
		compare_to_input,
		):
		sid = 0
		plc_pattern = None
		@@ -743,6 +756,7 @@ def test_sba_enc_BWforce_system(
		sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -765,6 +779,7 @@ def test_sba_enc_BWforce_system(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		compare_to_input=compare_to_input,
		)


		@@ -801,6 +816,7 @@ def test_sba_plc_system(
		get_ssnr,
		get_odg,
		get_enc_stats,
		compare_to_input,
		):
		sid = 0
		pca = False
		@@ -872,6 +888,7 @@ def test_sba_plc_system(
		sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -894,6 +911,7 @@ def test_sba_plc_system(
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		compare_to_input=compare_to_input,
		)


		@@ -1026,6 +1044,7 @@ def sba_enc(
		def sba_dec(
		record_property,
		props_to_record,
		test_vector_path,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -1048,6 +1067,7 @@ def sba_dec(
		abs_tol=0,
		get_ssnr=False,
		get_odg=False,
		compare_to_input=False,
		):
		dut_pkt_dir = f"{dut_base_path}/sba_bs/pkt"
		ref_pkt_dir = f"{reference_path}/sba_bs/pkt"
		@@ -1115,6 +1135,12 @@ def sba_dec(
		plc_file=plc_file,
		)

		allow_differing_lengths = False
		if compare_to_input:
		# If comparing to input, set input as reference and allow different length
		ref_out_file = f"{test_vector_path}/{tag}.wav"
		allow_differing_lengths = True

		sampling_rate_Hz = int(sampling_rate) * 1000
		cmp_result, reason = cmp_pcm(
		dut_out_file,
		@@ -1124,6 +1150,7 @@ def sba_dec(
		get_mld=get_mld,
		mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		allow_differing_lengths=allow_differing_lengths,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)
		@@ -1136,8 +1163,3 @@ def sba_dec(
		# report compare result
		if cmp_result != 0:
		pytest.fail(text_to_parse)

		# remove DUT output files when test result is OK (to save disk space)
		# if not keep_files:
		# os.remove(dut_out_file)
		# os.remove(dut_pkt_file)