add encoder verfication mechanism to the sba tests (5ec9da74) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/codec_be_on_mr_nonselection/test_sba.py

+203 −217

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ __doc__ = """

		import errno
		import os
		from typing import Tuple

		import pytest
		from cut_bs import cut_from_start
		@@ -42,7 +43,8 @@ from cut_bs import cut_from_start
		from tests.cmp_pcm import cmp_pcm
		from tests.conftest import DecoderFrontend, EncoderFrontend
		from ..conftest import parse_properties
		from ..constants import TESTV_DIR
		from ..cmp_stats_files import cmp_stats_files
		from ..constants import TESTV_DIR, MIN_ENC_FILE_LENGTH_DIFF, MIN_ENC_STATS_DIFF

		# params

		@@ -116,6 +118,7 @@ def test_pca_enc(
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		):
		pca = True
		tag = tag + sampling_rate + "c"
		@@ -123,7 +126,6 @@ def test_pca_enc(
		ivas_max_bw = "FB"
		sba_order = "+1"
		output_config = "FOA"
		br_switch_file_path = None
		cut_testv = True

		run_sba(
		@@ -136,7 +138,6 @@ def test_pca_enc(
		test_vector_path,
		reference_path,
		dut_base_path,
		br_switch_file_path,
		tag,
		sampling_rate,
		bitrate,
		@@ -150,6 +151,7 @@ def test_pca_enc(
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only,
		decoder_only,
		cut_testv=cut_testv,
		@@ -158,12 +160,12 @@ def test_pca_enc(


		@pytest.mark.create_ref
		@pytest.mark.parametrize("ivas_br", ivas_br_FOA)
		@pytest.mark.parametrize("bitrate", ivas_br_FOA)
		@pytest.mark.parametrize("dtx", dtx_set)
		@pytest.mark.parametrize("tag", tag_list)
		@pytest.mark.parametrize("fs", sample_rate_list)
		@pytest.mark.parametrize("sampling_rate", sample_rate_list)
		@pytest.mark.parametrize("gain_flag", gain_list)
		@pytest.mark.parametrize("SID", SID_list)
		@pytest.mark.parametrize("sid", SID_list)
		def test_sba_enc_system(
		record_property,
		props_to_record,
		@@ -177,39 +179,41 @@ def test_sba_enc_system(
		br_switch_file_path,
		update_ref,
		keep_files,
		ivas_br,
		bitrate,
		dtx,
		tag,
		fs,
		sampling_rate,
		gain_flag,
		SID,
		sid,
		get_mld,
		get_mld_lim,
		encoder_only,
		decoder_only,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		):
		if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
		if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]:
		# skip high bitrates for DTX until DTX issue is resolved
		pytest.skip()
		if SID == 1:
		if sid == 1:
		if (
		ivas_br not in ["13200", "16400", "64000"]
		or fs == "16"
		bitrate not in ["13200", "16400", "64000"]
		or sampling_rate == "16"
		or gain_flag == 1
		or dtx == "0"
		):
		pytest.skip()
		else:
		if ivas_br in ["13200", "16400"]:
		if bitrate in ["13200", "16400"]:
		pytest.skip()
		if ivas_br == "sw_24k4_256k.bin" and gain_flag != 1:
		if bitrate == "sw_24k4_256k.bin" and gain_flag != 1:
		pytest.skip()
		if gain_flag == 1 and ivas_br not in ["13200", "16400", "24400", "32000"]:
		if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]:
		pytest.skip()
		tag = tag + fs + "c"
		max_bw = "FB"
		tag = tag + sampling_rate + "c"
		ivas_max_bw = "FB"
		sba_order = "+1"
		output_config = "FOA"
		if gain_flag == 1:
		@@ -219,59 +223,43 @@ def test_sba_enc_system(
		else:
		cut_gain = "1.0"

		if not decoder_only:
		# enc
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		br_switch_file_path,
		tag,
		fs,
		ivas_br,
		dtx,
		SID,
		max_bw,
		sba_order,
		update_ref,
		gain_flag,
		keep_files,
		cut_gain=cut_gain,
		create_dutenc=True,
		cut_testv=True,
		)

		# dec
		sba_dec(
		run_sba(
		record_property,
		props_to_record,
		dut_encoder_frontend,
		dut_decoder_frontend,
		ref_encoder_frontend,
		ref_decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		tag,
		fs,
		ivas_br,
		dtx,
		SID,
		max_bw,
		sampling_rate,
		bitrate,
		ivas_max_bw,
		sba_order,
		output_config,
		update_ref,
		gain_flag,
		keep_files,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only,
		decoder_only,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		cut_gain=cut_gain,
		br_switch_file_path=br_switch_file_path,
		sid=sid,
		create_dutenc=True,
		gain_flag=gain_flag,
		dtx=dtx,
		)


		@pytest.mark.create_ref
		@pytest.mark.parametrize("ivas_br", ivas_br_HOA2)
		@pytest.mark.parametrize("bitrate", ivas_br_HOA2)
		@pytest.mark.parametrize("tag", tag_list_HOA2)
		def test_spar_hoa2_enc_system(
		record_property,
		@@ -285,74 +273,55 @@ def test_spar_hoa2_enc_system(
		ref_decoder_frontend,
		update_ref,
		keep_files,
		ivas_br,
		bitrate,
		tag,
		get_mld,
		get_mld_lim,
		encoder_only,
		decoder_only,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		):
		fs = "48"
		dtx = "0"
		gain_flag = -1
		sampling_rate = "48"

		tag = tag + fs + "c"
		max_bw = "FB"
		tag = tag + sampling_rate + "c"
		ivas_max_bw = "FB"
		sba_order = "+2"
		output_config = "HOA2"

		if not decoder_only:
		# enc
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		tag,
		fs,
		ivas_br,
		dtx,
		None,
		max_bw,
		sba_order,
		update_ref,
		gain_flag,
		keep_files,
		)

		# dec
		sba_dec(
		run_sba(
		record_property,
		props_to_record,
		dut_encoder_frontend,
		dut_decoder_frontend,
		ref_encoder_frontend,
		ref_decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		tag,
		fs,
		ivas_br,
		dtx,
		None,
		max_bw,
		sampling_rate,
		bitrate,
		ivas_max_bw,
		sba_order,
		output_config,
		update_ref,
		gain_flag,
		keep_files,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only,
		decoder_only,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)


		@pytest.mark.create_ref
		@pytest.mark.parametrize("ivas_br", ivas_br_HOA3)
		@pytest.mark.parametrize("bitrate", ivas_br_HOA3)
		@pytest.mark.parametrize("tag", tag_list_HOA3)
		def test_spar_hoa3_enc_system(
		record_property,
		@@ -366,74 +335,55 @@ def test_spar_hoa3_enc_system(
		ref_decoder_frontend,
		update_ref,
		keep_files,
		ivas_br,
		bitrate,
		tag,
		get_mld,
		get_mld_lim,
		encoder_only,
		decoder_only,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		):
		fs = "48"
		dtx = "0"
		gain_flag = -1
		sampling_rate = "48"

		tag = tag + fs + "c"
		max_bw = "FB"
		tag = tag + sampling_rate + "c"
		ivas_max_bw = "FB"
		sba_order = "+3"
		output_config = "HOA3"

		if not decoder_only:
		# enc
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		tag,
		fs,
		ivas_br,
		dtx,
		None,
		max_bw,
		sba_order,
		update_ref,
		gain_flag,
		keep_files,
		)

		# dec
		sba_dec(
		run_sba(
		record_property,
		props_to_record,
		dut_encoder_frontend,
		dut_decoder_frontend,
		ref_encoder_frontend,
		ref_decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		tag,
		fs,
		ivas_br,
		dtx,
		None,
		max_bw,
		sampling_rate,
		bitrate,
		ivas_max_bw,
		sba_order,
		output_config,
		update_ref,
		gain_flag,
		keep_files,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only,
		decoder_only,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)


		@pytest.mark.create_ref
		@pytest.mark.parametrize("ivas_br", ivas_br_FOA)
		@pytest.mark.parametrize("bitrate", ivas_br_FOA)
		@pytest.mark.parametrize("dtx", dtx_set)
		@pytest.mark.parametrize("tag", tag_list_bw_force)
		@pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list)
		@@ -449,77 +399,59 @@ def test_sba_enc_BWforce_system(
		ref_decoder_frontend,
		update_ref,
		keep_files,
		ivas_br,
		bitrate,
		dtx,
		tag,
		sample_rate_bw_idx,
		get_mld,
		get_mld_lim,
		encoder_only,
		decoder_only,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		):
		if dtx == "1" and ivas_br not in ["32000", "64000"]:
		if dtx == "1" and bitrate not in ["32000", "64000"]:
		# skip high bitrates for DTX until DTX issue is resolved
		pytest.skip()
		if ivas_br == "13200" or ivas_br == "16400":
		if bitrate == "13200" or bitrate == "16400":
		pytest.skip()
		if ivas_br == "sw_24k4_256k.bin":
		if bitrate == "sw_24k4_256k.bin":
		pytest.skip()
		fs = sample_rate_bw_idx[0]
		bw = sample_rate_bw_idx[1]
		tag = tag + fs + "c"
		gain_flag = -1
		sampling_rate = sample_rate_bw_idx[0]
		ivas_max_bw = sample_rate_bw_idx[1]
		tag = tag + sampling_rate + "c"
		sba_order = "+1"
		output_config = "FOA"

		if not decoder_only:
		# enc
		sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_frontend,
		reference_path,
		dut_base_path,
		None,
		tag,
		fs,
		ivas_br,
		dtx,
		None,
		bw,
		sba_order,
		update_ref,
		gain_flag,
		keep_files,
		cut_testv=True,
		)

		# dec
		sba_dec(
		run_sba(
		record_property,
		props_to_record,
		dut_encoder_frontend,
		dut_decoder_frontend,
		ref_encoder_frontend,
		ref_decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		tag,
		fs,
		ivas_br,
		dtx,
		None,
		bw,
		sampling_rate,
		bitrate,
		ivas_max_bw,
		sba_order,
		output_config,
		update_ref,
		gain_flag,
		keep_files,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only,
		decoder_only,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		dtx=dtx,
		)


		@@ -530,11 +462,11 @@ def test_sba_enc_BWforce_system(
		# -> the reference generation for this test (reference decoder output) needs to be done after completion of test_sba_enc_system
		# -> therefore the marker create_ref_part2
		@pytest.mark.create_ref_part2
		@pytest.mark.parametrize("ivas_br", ivas_br_plc)
		@pytest.mark.parametrize("bitrate", ivas_br_plc)
		@pytest.mark.parametrize("dtx", dtx_set)
		@pytest.mark.parametrize("tag", tag_list)
		@pytest.mark.parametrize("plc_pattern", plc_patterns)
		@pytest.mark.parametrize("fs", sample_rate_list)
		@pytest.mark.parametrize("sampling_rate", sample_rate_list)
		@pytest.mark.parametrize("gain_flag", gain_list)
		def test_sba_plc_system(
		record_property,
		@@ -546,61 +478,73 @@ def test_sba_plc_system(
		ref_decoder_path,
		update_ref,
		keep_files,
		ivas_br,
		bitrate,
		dtx,
		tag,
		plc_pattern,
		fs,
		sampling_rate,
		gain_flag,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		test_vector_path,
		):
		SID = 0
		if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
		sid = 0
		if dtx == "1" and bitrate not in ["13200", "16400", "24400", "32000", "64000"]:
		# skip high bitrates for DTX until DTX issue is resolved
		pytest.skip()
		if ivas_br == "13200" or ivas_br == "16400":
		if bitrate == "13200" or bitrate == "16400":
		if (
		dtx == "1"
		and gain_flag == 0
		and fs != "16"
		and sampling_rate != "16"
		and plc_pattern == "PLperc12mblen5"
		):
		SID = 1
		sid = 1
		else:
		pytest.skip()
		if gain_flag == 1 and ivas_br not in ["13200", "16400", "24400", "32000"]:
		if gain_flag == 1 and bitrate not in ["13200", "16400", "24400", "32000"]:
		pytest.skip()
		tag = tag + fs + "c"
		tag = tag + sampling_rate + "c"

		# dec
		sba_dec(
		ivas_max_bw = "FB"
		output_config = "FOA"
		sba_order = "+1"
		dut_encoder_frontend = None
		ref_encoder_frontend = None

		run_sba(
		record_property,
		props_to_record,
		dut_encoder_frontend,
		dut_decoder_frontend,
		ref_encoder_frontend,
		ref_decoder_frontend,
		test_vector_path,
		reference_path,
		dut_base_path,
		tag,
		fs,
		ivas_br,
		dtx,
		SID,
		"FB",
		"FOA",
		sampling_rate,
		bitrate,
		ivas_max_bw,
		sba_order,
		output_config,
		update_ref,
		gain_flag,
		keep_files,
		False,
		plc_pattern=plc_pattern,
		get_mld=get_mld,
		get_mld_lim=get_mld_lim,
		abs_tol=abs_tol,
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		get_mld,
		get_mld_lim,
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only=False,
		decoder_only=True,
		sid=sid,
		dtx=dtx,
		gain_flag=gain_flag,
		)


		@@ -627,7 +571,8 @@ def sba_enc(
		create_dutenc=False,
		cut_testv=False,
		pca=False,
		):
		get_enc_stats=False,
		) -> Tuple[str, str]:
		# ------------ run cmd ------------
		dut_out_dir = f"{dut_base_path}/sba_bs/pkt"
		ref_out_dir = f"{reference_path}/sba_bs/pkt"
		@@ -675,6 +620,9 @@ def sba_enc(

		input_path = cut_file

		stats_file_ref = ref_pkt_file.replace(".192", ".stats")
		stats_file_dut = dut_pkt_file.replace(".192", ".stats")

		if ref_encoder_frontend:
		# call REF encoder
		ref_encoder_frontend.run(
		@@ -686,6 +634,7 @@ def sba_enc(
		max_band=ivas_max_bw,
		pca=pca,
		dtx_mode=dtx_mode,
		stats_file=stats_file_ref if get_enc_stats else None,
		)
		if create_dutenc:
		# for PLC decoder tests, create bitstream using DUT encoder
		@@ -711,6 +660,7 @@ def sba_enc(
		max_band=ivas_max_bw,
		pca=pca,
		dtx_mode=dtx_mode,
		stats_file=stats_file_dut if get_enc_stats else None,
		)

		if SID == 1:
		@@ -731,10 +681,13 @@ def sba_enc(
		if not keep_files:
		os.remove(dut_pkt_file)

		return stats_file_ref, stats_file_dut


		def sba_dec(
		record_property,
		props_to_record,
		cmp_result_msg,
		decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -838,13 +791,14 @@ def sba_dec(
		get_odg=get_odg,
		)

		props = parse_properties(reason, cmp_result != 0, props_to_record)
		text_to_parse = cmp_result_msg + reason
		props = parse_properties(text_to_parse, cmp_result != 0, props_to_record)
		for k, v in props.items():
		record_property(k, v)

		# report compare result
		if cmp_result != 0:
		pytest.fail(reason)
		pytest.fail(text_to_parse)

		# remove DUT output files when test result is OK (to save disk space)
		if not keep_files:
		@@ -863,7 +817,6 @@ def run_sba(
		test_vector_path,
		reference_path,
		dut_base_path,
		br_switch_file_path,
		tag,
		sampling_rate,
		bitrate,
		@@ -877,18 +830,21 @@ def run_sba(
		abs_tol,
		get_ssnr,
		get_odg,
		get_enc_stats,
		encoder_only,
		decoder_only,
		dtx="0",
		sid=None,
		sid=0,
		br_switch_file_path=None,
		gain_flag=-1,
		cut_gain="1.0",
		create_dutenc=False,
		cut_testv=False,
		pca=False,
		):
		cmp_result_msg = ""
		if not decoder_only:
		sba_enc(
		stats_file_ref, stats_file_dut = sba_enc(
		dut_encoder_frontend,
		test_vector_path,
		ref_encoder_frontend,
		@@ -909,12 +865,44 @@ def run_sba(
		create_dutenc,
		cut_testv,
		pca,
		get_enc_stats,
		)

		if get_enc_stats:
		print("Comparing encoder files")
		print("=======================\n")

		# compare ref and dut .stats files
		enc_test_result, enc_test_result_msg = cmp_stats_files(
		stats_file_ref,
		stats_file_dut,
		min_enc_file_length_diff=MIN_ENC_FILE_LENGTH_DIFF,
		min_enc_stats_diff=MIN_ENC_STATS_DIFF,
		)

		print("")
		cmp_result_msg += enc_test_result_msg

		# TODO: there is cleanup potential here:
		# - move filename handling outside of the sba_enc/dec functions into start of this function
		# - move file cleanup part out as well
		# - move this comparison to the end and the one in sba_dec there as well
		if encoder_only:
		props = parse_properties(
		cmp_result_msg, enc_test_result != 0, props_to_record
		)
		for k, v in props.items():
		record_property(k, v)

		# report compare result
		if enc_test_result != 0:
		pytest.fail(cmp_result_msg)

		if not encoder_only:
		sba_dec(
		record_property,
		props_to_record,
		cmp_result_msg,
		dut_decoder_frontend,
		ref_decoder_frontend,
		reference_path,
		@@ -923,7 +911,7 @@ def run_sba(
		sampling_rate,
		bitrate,
		dtx,
		None,
		sid,
		ivas_max_bw,
		output_config,
		update_ref,
		@@ -937,5 +925,3 @@ def run_sba(
		get_ssnr=get_ssnr,
		get_odg=get_odg,
		)

		# record properties tbd