From 61176f36a3dd7940f43c634fc6a2d79786d60654 Mon Sep 17 00:00:00 2001 From: vaclav Date: Fri, 13 Sep 2024 09:22:31 +0200 Subject: [PATCH 01/15] output number of clipped samples in downmixed channels; under DEBUGGING --- apps/encoder.c | 5 +++++ lib_com/prot.h | 9 +++++++++ lib_com/tools.c | 33 +++++++++++++++++++++++++++++++++ lib_enc/ivas_cpe_enc.c | 4 ++++ lib_enc/ivas_init_enc.c | 3 +++ lib_enc/ivas_ism_enc.c | 4 ++++ lib_enc/ivas_sce_enc.c | 4 ++++ lib_enc/ivas_stat_enc.h | 3 +++ lib_enc/lib_enc.c | 16 ++++++++++++++++ lib_enc/lib_enc.h | 5 +++++ 10 files changed, 86 insertions(+) diff --git a/apps/encoder.c b/apps/encoder.c index f64986ec84..fca2997b64 100644 --- a/apps/encoder.c +++ b/apps/encoder.c @@ -195,6 +195,7 @@ int main( #ifdef DEBUG_SBA int16_t numTransportChannels = 1; #endif + int32_t cnt_frames_limited, noClipping; #endif #ifdef DEBUGGING @@ -792,6 +793,10 @@ int main( } #ifdef DEBUGGING + if ( ( noClipping = IVAS_ENC_GetNoCLipping( hIvasEnc ) ) > 0 ) + { + fprintf( stdout, "Clipping (saturation) detected: %d samples clipped!!!\n\n", noClipping ); + } print_snr(); #endif /*------------------------------------------------------------------------------------------* diff --git a/lib_com/prot.h b/lib_com/prot.h index c8f70bb4fc..f26e7b6ac8 100644 --- a/lib_com/prot.h +++ b/lib_com/prot.h @@ -222,6 +222,15 @@ void mvs2s( const int16_t n /* i : vector size */ ); +#ifdef DEBUGGING +/*! r: number of clipped samples */ +uint32_t check_clipping( + const float x[], /* i : input vector */ + const int16_t n /* i : vector size */ +); + +#endif +/*! r: number of clipped samples */ uint32_t mvr2s( const float x[], /* i : input vector */ int16_t y[], /* o : output vector */ diff --git a/lib_com/tools.c b/lib_com/tools.c index 9dfa8b9f81..0f90282039 100644 --- a/lib_com/tools.c +++ b/lib_com/tools.c @@ -373,6 +373,39 @@ void mvs2s( return; } +#ifdef DEBUGGING +/*! r: number of clipped samples */ +uint32_t check_clipping( + const float x[], /* i : input vector */ + const int16_t n /* i : vector size */ +) +{ + int16_t i; + float temp; + uint32_t noClipping = 0; + + for ( i = 0; i < n; i++ ) + { + temp = x[i]; + temp = (float) floor( temp + 0.5f ); + + if ( temp > MAX16B_FLT ) + { + temp = MAX16B_FLT; + noClipping++; + } + else if ( temp < MIN16B_FLT ) + { + temp = MIN16B_FLT; + noClipping++; + } + } + + return noClipping; +} + +#endif +/*! r: number of clipped samples */ uint32_t mvr2s( const float x[], /* i : input vector */ int16_t y[], /* o : output vector */ diff --git a/lib_enc/ivas_cpe_enc.c b/lib_enc/ivas_cpe_enc.c index a3a2bec257..5ab323c39d 100644 --- a/lib_enc/ivas_cpe_enc.c +++ b/lib_enc/ivas_cpe_enc.c @@ -471,6 +471,10 @@ ivas_error ivas_cpe_enc( for ( n = 0; n < n_CoreChannels; n++ ) { +#ifdef DEBUGGING + st_ivas->noClipping += check_clipping( hCPE->hCoreCoder[n]->input, input_frame ); + +#endif error = pre_proc_front_ivas( NULL, hCPE, hCPE->element_brate, nb_bits_metadata, input_frame, n, old_inp_12k8[n], old_inp_16k[n], &ener[n], &relE[n], A[n], Aw[n], epsP[n], lsp_new[n], lsp_mid[n], &vad_hover_flag[n], &attack_flag[n], realBuffer[n], imagBuffer[n], old_wsp[n], pitch_fr[n], voicing_fr[n], &loc_harm[n], &cor_map_sum[n], &vad_flag_dtx[n], enerBuffer[n], diff --git a/lib_enc/ivas_init_enc.c b/lib_enc/ivas_init_enc.c index 05621dfaaa..5a014fbfae 100644 --- a/lib_enc/ivas_init_enc.c +++ b/lib_enc/ivas_init_enc.c @@ -410,6 +410,9 @@ ivas_error ivas_init_encoder( st_ivas->nchan_transport = -1; +#ifdef DEBUGGING + st_ivas->noClipping = 0; +#endif /*-----------------------------------------------------------------* * Allocate floating-point input audio buffers *-----------------------------------------------------------------*/ diff --git a/lib_enc/ivas_ism_enc.c b/lib_enc/ivas_ism_enc.c index 6e2732f44a..e3569f44ac 100644 --- a/lib_enc/ivas_ism_enc.c +++ b/lib_enc/ivas_ism_enc.c @@ -172,6 +172,10 @@ ivas_error ivas_ism_enc( * Front Pre-processing *----------------------------------------------------------------*/ +#ifdef DEBUGGING + st_ivas->noClipping += check_clipping( hSCE->hCoreCoder[0]->input, input_frame ); + +#endif error = pre_proc_front_ivas( hSCE, NULL, hSCE->element_brate, nb_bits_metadata[sce_id], input_frame, 0, old_inp_12k8[sce_id][0], old_inp_16k[sce_id][0], &ener[sce_id][0], &relE[sce_id][0], A[sce_id][0], Aw[sce_id][0], epsP[sce_id][0], lsp_new[sce_id][0], lsp_mid[sce_id][0], &vad_hover_flag[sce_id][0], &attack_flag[sce_id][0], realBuffer[sce_id][0], imagBuffer[sce_id][0], old_wsp[sce_id][0], pitch_fr[sce_id][0], voicing_fr[sce_id][0], &loc_harm[sce_id][0], &cor_map_sum[sce_id][0], &vad_flag_dtx[sce_id][0], enerBuffer[sce_id][0], diff --git a/lib_enc/ivas_sce_enc.c b/lib_enc/ivas_sce_enc.c index 8f5d472ea9..cb3d8dea6e 100644 --- a/lib_enc/ivas_sce_enc.c +++ b/lib_enc/ivas_sce_enc.c @@ -180,6 +180,10 @@ ivas_error ivas_sce_enc( * Front Pre-processing *----------------------------------------------------------------*/ +#ifdef DEBUGGING + st_ivas->noClipping += check_clipping( hSCE->hCoreCoder[0]->input, input_frame ); + +#endif error = pre_proc_front_ivas( hSCE, NULL, hSCE->element_brate, nb_bits_metadata, input_frame, 0, old_inp_12k8[0], old_inp_16k[0], &ener[0], &relE[0], A[0], Aw[0], epsP[0], lsp_new[0], lsp_mid[0], &vad_hover_flag[0], &attack_flag[0], realBuffer[0], imagBuffer[0], old_wsp[0], pitch_fr[0], voicing_fr[0], &loc_harm[0], &cor_map_sum[0], &vad_flag_dtx[0], enerBuffer[0], diff --git a/lib_enc/ivas_stat_enc.h b/lib_enc/ivas_stat_enc.h index 4bb9a5f417..5a0dd72e6f 100644 --- a/lib_enc/ivas_stat_enc.h +++ b/lib_enc/ivas_stat_enc.h @@ -1246,6 +1246,9 @@ typedef struct int16_t nCPE; /* number of total CPEs */ SCE_ENC_HANDLE hSCE[MAX_SCE]; /* SCE handles */ CPE_ENC_HANDLE hCPE[MCT_MAX_BLOCKS]; /* CPE handles */ +#ifdef DEBUGGING + int32_t noClipping; /* number of clipped samples */ +#endif /* multichannel modules */ ISM_METADATA_HANDLE hIsmMetaData[MAX_NUM_OBJECTS]; /* ISM metadata handles (storage for one frame of read ISM metadata) */ diff --git a/lib_enc/lib_enc.c b/lib_enc/lib_enc.c index 23e666a7f0..dc62941199 100644 --- a/lib_enc/lib_enc.c +++ b/lib_enc/lib_enc.c @@ -2417,3 +2417,19 @@ static void init_encoder_config( return; } + +#ifdef DEBUGGING + +/*---------------------------------------------------------------------* + * IVAS_ENC_GetNoCLipping() + * + * return number of clipped samples + *---------------------------------------------------------------------*/ + +int32_t IVAS_ENC_GetNoCLipping( + IVAS_ENC_HANDLE hIvasEnc /* i : IVAS encoder handle */ +) +{ + return hIvasEnc->st_ivas->noClipping; +} +#endif diff --git a/lib_enc/lib_enc.h b/lib_enc/lib_enc.h index 2f40c1ab1b..1b1929e755 100644 --- a/lib_enc/lib_enc.h +++ b/lib_enc/lib_enc.h @@ -383,6 +383,11 @@ void IVAS_ENC_PrintDisclaimer( void ); +#ifdef DEBUGGING +int32_t IVAS_ENC_GetNoCLipping( + IVAS_ENC_HANDLE hIvasEnc /* i : IVAS encoder handle */ +); +#endif /* clang-format on */ #endif /* LIB_ENC_H */ -- GitLab From 1c340c37a59c947b1962ae395cae409a5c35c0ee Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 4 Oct 2024 16:34:14 +0200 Subject: [PATCH 02/15] Record clipping in xml report --- lib_com/options.h | 2 +- scripts/testv/stv3OA48c.wav | 4 ++-- .../test_param_file.py | 4 ++++ tests/codec_be_on_mr_nonselection/test_sba.py | 3 +++ tests/conftest.py | 15 ++++++++++++--- tests/constants.py | 3 ++- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index e58a02336a..5ba6984948 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -48,7 +48,7 @@ /* ################### Start DEBUGGING switches ########################### */ #ifndef RELEASE -/*#define DEBUGGING*/ /* Activate debugging part of the code */ +#define DEBUGGING /* Activate debugging part of the code */ #endif /*#define WMOPS*/ /* Activate complexity and memory counters */ /*#define WMOPS_PER_FRAME*/ /* Output per-frame complexity (writes one float value per frame to the file "wmops_analysis") */ diff --git a/scripts/testv/stv3OA48c.wav b/scripts/testv/stv3OA48c.wav index 960d9e2019..377f5e4f13 100644 --- a/scripts/testv/stv3OA48c.wav +++ b/scripts/testv/stv3OA48c.wav @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7ce54b1708a76f34eeedc40b01e806cb92d32b0813e8ac23bfceaaa8d25262e6 -size 30720844 +oid sha256:d2afc4cef4225191d0191ffc16e0de9684e57e0e6eaed34b4ec8411cffa8ae9b +size 30725886 diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index 34bffea7b6..ce49b52998 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -200,6 +200,7 @@ def test_param_file_tests( if not decoder_only: encode( + record_property, dut_encoder_frontend, ref_encoder_frontend, reference_path, @@ -476,6 +477,7 @@ def test_param_file_tests( def encode( + record_property, dut_encoder_frontend, ref_encoder_frontend, reference_path, @@ -511,6 +513,7 @@ def encode( # call REF encoder ref_encoder_frontend.run( + record_property, bitrate, sampling_rate, testv_file, @@ -524,6 +527,7 @@ def encode( # call DUT encoder dut_encoder_frontend.run( + record_property, bitrate, sampling_rate, testv_file, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 98f55b7fe5..0a1f227e35 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -901,6 +901,7 @@ def test_sba_plc_system( def sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -977,6 +978,7 @@ def sba_enc( if update_ref == 1: # call REF encoder ref_encoder_frontend.run( + record_property, bitrate, sampling_rate, input_path, @@ -991,6 +993,7 @@ def sba_enc( if update_ref == 0: # call DUT encoder dut_encoder_frontend.run( + record_property, bitrate, sampling_rate, input_path, diff --git a/tests/conftest.py b/tests/conftest.py index 636881ecef..c9422bb285 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,6 +57,8 @@ from .constants import ( ODG, MAX_ENC_DIFF, MAX_ENC_DIFF_PATTERN, + ENC_CORE_CLIPPING, + CLIPPING_PATTERN, SCRIPTS_DIR, ) @@ -316,11 +318,10 @@ def get_enc_stats(request) -> bool: @pytest.fixture(scope="session", autouse=True) def get_odg(request): """ - Return indication to compute ssnr during ref/dut comparison. + Return indication to compute PEAQ ODG during ref/dut comparison. """ return request.config.option.odg - @pytest.fixture(scope="session") def abs_tol(request) -> int: """ @@ -456,6 +457,7 @@ class EncoderFrontend: def run( self, + record_property, bitrate: Union[int, Path], input_sampling_rate: int, input_path: Path, @@ -527,6 +529,14 @@ class EncoderFrontend: self.returncode = result.returncode self.stderr = result.stderr.decode("ascii") self.stdout = result.stdout.decode("ascii") + + # Record core encoder clipping + clipping = 0 + search_result = re.search(CLIPPING_PATTERN, self.stdout) + if search_result: + clipping = search_result.group(1) + record_property( ENC_CORE_CLIPPING, clipping ) + if self.stdout: stdout_str = textwrap.indent(self.stdout, prefix="\t") log_dbg_msg(f"{self._type} encoder stdout:\n{stdout_str}") @@ -1047,7 +1057,6 @@ def parse_properties(text_to_parse: str, output_differs: bool, props_to_record: if max_enc_diff: max_enc_diff = float(max_enc_diff) props[MAX_ENC_DIFF] = max_enc_diff - return props diff --git a/tests/constants.py b/tests/constants.py index c2e84e50b7..8b6bd6beed 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -12,7 +12,7 @@ MAX_ABS_DIFF = "MAXIMUM ABS DIFF" SSNR = "SSNR" ODG = "ODG" MAX_ENC_DIFF = "MAXIMUM ENC DIFF" - +ENC_CORE_CLIPPING = "ENC_CORE_CLIPPING" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" @@ -21,6 +21,7 @@ ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" ODG_PATTERN = r"ODG: (-*\d*\.\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)" MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)" +CLIPPING_PATTERN = r"Clipping \(saturation\) detected: (\d+)" # minimum difference between ref and dut encoder file lengths -- GitLab From 0c29371023da429fc060f466e3dbfd21a2d0774a Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 4 Oct 2024 16:35:26 +0200 Subject: [PATCH 03/15] Set DEBUGGING disabled by default --- lib_com/options.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_com/options.h b/lib_com/options.h index 5ba6984948..e58a02336a 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -48,7 +48,7 @@ /* ################### Start DEBUGGING switches ########################### */ #ifndef RELEASE -#define DEBUGGING /* Activate debugging part of the code */ +/*#define DEBUGGING*/ /* Activate debugging part of the code */ #endif /*#define WMOPS*/ /* Activate complexity and memory counters */ /*#define WMOPS_PER_FRAME*/ /* Output per-frame complexity (writes one float value per frame to the file "wmops_analysis") */ -- GitLab From 9f0171c95998dd53987b968d8be9c58facf6d938 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 4 Oct 2024 16:38:04 +0200 Subject: [PATCH 04/15] Restore testv file --- scripts/testv/stv3OA48c.wav | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/testv/stv3OA48c.wav b/scripts/testv/stv3OA48c.wav index 377f5e4f13..960d9e2019 100644 --- a/scripts/testv/stv3OA48c.wav +++ b/scripts/testv/stv3OA48c.wav @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d2afc4cef4225191d0191ffc16e0de9684e57e0e6eaed34b4ec8411cffa8ae9b -size 30725886 +oid sha256:7ce54b1708a76f34eeedc40b01e806cb92d32b0813e8ac23bfceaaa8d25262e6 +size 30720844 -- GitLab From d6d18b08b62ebe65c5d543d857444bfb2b4eef5b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 6 Oct 2024 15:08:04 +0200 Subject: [PATCH 05/15] Add fixes for test_sba.py encoder clipping check --- tests/codec_be_on_mr_nonselection/test_sba.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index 0a1f227e35..d0adb52d31 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -138,6 +138,7 @@ def test_pca_enc( if not decoder_only: sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -271,6 +272,7 @@ def test_sba_enc_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -405,6 +407,7 @@ def test_spar_hoa2_enc_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -539,6 +542,7 @@ def test_spar_hoa3_enc_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -681,6 +685,7 @@ def test_sba_enc_BWforce_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -845,6 +850,7 @@ def test_sba_plc_system( if not decoder_only: sba_enc( + record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, -- GitLab From 363b4c3f06584f902782d66f85f48ea440aaccdb Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 6 Oct 2024 17:05:01 +0200 Subject: [PATCH 06/15] Add manual job to test clipping --- .gitlab-ci.yml | 41 +++++++++++++++++++++++++++++++++++++ scripts/parse_xml_report.py | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e5c88431c..543a655c35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ variables: - 'ivas-conformance' - 'ivas-conformance-linux' - 'check-float-reference' + - 'check-clipping' GIT_CLEAN_FLAGS: -ffdxq TESTCASE_TIMEOUT_STV_SANITIZERS: 180 TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 @@ -69,6 +70,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-float-reference' variables: IVAS_PIPELINE_NAME: 'check-float-reference: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' + variables: + IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH' stages: @@ -122,6 +126,11 @@ stages: # automatically disable #DEBUGGING macro in options.h using /**/-comment - sed -i.bak -e "s/^[[:space:]]*\(#define[[:space:]]*DEBUGGING\)/\/\*\1\*\//g" lib_com/options.h +.enable-debugging-macro: &enable-debugging-macro +# automatically enable #DEBUGGING macro in options.h using either /**/-comment or //-comment + - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h + - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h + .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - *disable-debugging-macro @@ -265,6 +274,8 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'check-float-reference' when: never + - if: $MANUAL_PIPELINE_TYPE == 'check-clipping' + when: never - when: on_success .rules-merge-request: @@ -1639,6 +1650,36 @@ test-long-self-test: junit: - report-junit-ltv.xml +check-clipping: + tags: + - ivas-linux + stage: test + timeout: "30 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping' + allow_failure: + exit_codes: + - 123 + script: + - *print-common-info + - *enable-debugging-macro + - make -j + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec + - python3 scripts/parse_xml_report.py report-junit.xml report.csv + artifacts: + name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 2 weeks + paths: + - report-junit.xml + - report.html + - report.csv + expose_as: "check-clipping results" + reports: + junit: + - report-junit.xml + + # --------------------------------------------------------------- # Scheduled jobs on main diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 3be7a9ca0b..d7d581d950 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -10,7 +10,7 @@ from xml.etree import ElementTree Parse a junit report and create a summary report. """ -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG","ENC_CORE_CLIPPING"] IVAS_FORMATS = { "Stereo": r"stereo", -- GitLab From 3737191d3af05dcdd4bb5c0e61b611224af8ebac Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 6 Oct 2024 17:10:08 +0200 Subject: [PATCH 07/15] Add +10 dB level scaling for check-clipping job --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 543a655c35..2fbf8ae994 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1664,6 +1664,7 @@ check-clipping: - *print-common-info - *enable-debugging-macro - make -j + - tests/scale_pcm.py ./scripts/testv/ 3.162 # +10 dB level - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec - python3 scripts/parse_xml_report.py report-junit.xml report.csv artifacts: -- GitLab From 5575ec8f576e6aec031a7ada43e615b437ca0f9c Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Sun, 6 Oct 2024 17:18:21 +0200 Subject: [PATCH 08/15] Add clipping argument to parse_xml_report.py to avoid problems with existing comparison scripts --- .gitlab-ci.yml | 2 +- scripts/parse_xml_report.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fbf8ae994..00181130a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1666,7 +1666,7 @@ check-clipping: - make -j - tests/scale_pcm.py ./scripts/testv/ 3.162 # +10 dB level - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec - - python3 scripts/parse_xml_report.py report-junit.xml report.csv + - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping artifacts: name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" when: always diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index d7d581d950..41004d2363 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -10,7 +10,7 @@ from xml.etree import ElementTree Parse a junit report and create a summary report. """ -PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG","ENC_CORE_CLIPPING"] +PROPERTIES = ["MLD", "MAXIMUM ABS DIFF", "MIN_SSNR", "MIN_ODG"] IVAS_FORMATS = { "Stereo": r"stereo", @@ -64,6 +64,11 @@ if __name__ == "__main__": action="store_true", help="Parse using EVS 26.444 formats", ) + parser.add_argument( + "--clipping", + action="store_true", + help="Extract clipping information. Available if encoder has been run with DEBUGGING active.", + ) args = parser.parse_args() xml_report = args.xml_report csv_file = args.csv_file @@ -73,6 +78,8 @@ if __name__ == "__main__": else: FORMATS = IVAS_FORMATS CATEGORIES = IVAS_CATEGORIES + if args.clipping: + PROPERTIES += ["ENC_CORE_CLIPPING"] tree = ElementTree.parse(xml_report) testsuite = tree.find(".//testsuite") -- GitLab From cb0dfc314c80373873fcf59d2ce76656cd18628c Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 7 Oct 2024 12:55:26 +0200 Subject: [PATCH 09/15] Add max/min overload value --- apps/encoder.c | 9 ++++++--- lib_com/prot.h | 8 +++++--- lib_com/tools.c | 23 +++++++++++++++-------- lib_enc/ivas_cpe_enc.c | 2 +- lib_enc/ivas_init_enc.c | 2 ++ lib_enc/ivas_ism_enc.c | 2 +- lib_enc/ivas_sce_enc.c | 2 +- lib_enc/ivas_stat_enc.h | 2 ++ lib_enc/lib_enc.c | 6 +++++- lib_enc/lib_enc.h | 4 +++- scripts/parse_xml_report.py | 2 +- tests/conftest.py | 19 ++++++++++++------- tests/constants.py | 8 ++++++-- 13 files changed, 60 insertions(+), 29 deletions(-) diff --git a/apps/encoder.c b/apps/encoder.c index fca2997b64..4caa60803d 100644 --- a/apps/encoder.c +++ b/apps/encoder.c @@ -195,7 +195,8 @@ int main( #ifdef DEBUG_SBA int16_t numTransportChannels = 1; #endif - int32_t cnt_frames_limited, noClipping; + int32_t noClipping; + float maxOverload, minOverload; #endif #ifdef DEBUGGING @@ -793,9 +794,11 @@ int main( } #ifdef DEBUGGING - if ( ( noClipping = IVAS_ENC_GetNoCLipping( hIvasEnc ) ) > 0 ) + if ( ( noClipping = IVAS_ENC_GetNoCLipping( hIvasEnc, &maxOverload, &minOverload ) ) > 0 ) { - fprintf( stdout, "Clipping (saturation) detected: %d samples clipped!!!\n\n", noClipping ); + fprintf( stdout, "Core input overload detected: %d samples!!!\n", noClipping ); + fprintf( stdout, "Max overload value: %f \n", maxOverload ); + fprintf( stdout, "Min overload value: %f \n\n", minOverload ); } print_snr(); #endif diff --git a/lib_com/prot.h b/lib_com/prot.h index f26e7b6ac8..1df0eb7838 100644 --- a/lib_com/prot.h +++ b/lib_com/prot.h @@ -223,10 +223,12 @@ void mvs2s( ); #ifdef DEBUGGING -/*! r: number of clipped samples */ +/*! r: number of overload samples */ uint32_t check_clipping( - const float x[], /* i : input vector */ - const int16_t n /* i : vector size */ + const float x[], /* i : input vector */ + const int16_t n, /* i : vector size */ + float *maxOverload, /* i/o: max overload value */ + float *minOverload /* i/o: max overload value */ ); #endif diff --git a/lib_com/tools.c b/lib_com/tools.c index 0f90282039..b235d5d499 100644 --- a/lib_com/tools.c +++ b/lib_com/tools.c @@ -374,10 +374,12 @@ void mvs2s( } #ifdef DEBUGGING -/*! r: number of clipped samples */ +/*! r: number of overload samples */ uint32_t check_clipping( - const float x[], /* i : input vector */ - const int16_t n /* i : vector size */ + const float x[], /* i : input vector */ + const int16_t n, /* i : vector size */ + float *maxOverload, /* i/o: max overload value */ + float *minOverload /* i/o: max overload value */ ) { int16_t i; @@ -387,17 +389,22 @@ uint32_t check_clipping( for ( i = 0; i < n; i++ ) { temp = x[i]; - temp = (float) floor( temp + 0.5f ); - if ( temp > MAX16B_FLT ) + if ( temp >= ( MAX16B_FLT + 0.5f ) ) { - temp = MAX16B_FLT; noClipping++; + if ( temp > *maxOverload ) + { + *maxOverload = temp; + } } - else if ( temp < MIN16B_FLT ) + else if ( temp <= ( MIN16B_FLT - 0.5f ) ) { - temp = MIN16B_FLT; noClipping++; + if ( temp < *minOverload ) + { + *minOverload = temp; + } } } diff --git a/lib_enc/ivas_cpe_enc.c b/lib_enc/ivas_cpe_enc.c index 5ab323c39d..292cbd6a5f 100644 --- a/lib_enc/ivas_cpe_enc.c +++ b/lib_enc/ivas_cpe_enc.c @@ -472,7 +472,7 @@ ivas_error ivas_cpe_enc( for ( n = 0; n < n_CoreChannels; n++ ) { #ifdef DEBUGGING - st_ivas->noClipping += check_clipping( hCPE->hCoreCoder[n]->input, input_frame ); + st_ivas->noClipping += check_clipping( hCPE->hCoreCoder[n]->input, input_frame, &st_ivas->maxOverload, &st_ivas->minOverload ); #endif error = pre_proc_front_ivas( NULL, hCPE, hCPE->element_brate, nb_bits_metadata, input_frame, n, old_inp_12k8[n], old_inp_16k[n], diff --git a/lib_enc/ivas_init_enc.c b/lib_enc/ivas_init_enc.c index 5a014fbfae..48a3286a4e 100644 --- a/lib_enc/ivas_init_enc.c +++ b/lib_enc/ivas_init_enc.c @@ -412,6 +412,8 @@ ivas_error ivas_init_encoder( #ifdef DEBUGGING st_ivas->noClipping = 0; + st_ivas->maxOverload = 0; + st_ivas->minOverload = 0; #endif /*-----------------------------------------------------------------* * Allocate floating-point input audio buffers diff --git a/lib_enc/ivas_ism_enc.c b/lib_enc/ivas_ism_enc.c index e3569f44ac..c72431bd0b 100644 --- a/lib_enc/ivas_ism_enc.c +++ b/lib_enc/ivas_ism_enc.c @@ -173,7 +173,7 @@ ivas_error ivas_ism_enc( *----------------------------------------------------------------*/ #ifdef DEBUGGING - st_ivas->noClipping += check_clipping( hSCE->hCoreCoder[0]->input, input_frame ); + st_ivas->noClipping += check_clipping( hSCE->hCoreCoder[0]->input, input_frame, &st_ivas->maxOverload, &st_ivas->minOverload ); #endif error = pre_proc_front_ivas( hSCE, NULL, hSCE->element_brate, nb_bits_metadata[sce_id], input_frame, 0, old_inp_12k8[sce_id][0], old_inp_16k[sce_id][0], diff --git a/lib_enc/ivas_sce_enc.c b/lib_enc/ivas_sce_enc.c index cb3d8dea6e..257e806195 100644 --- a/lib_enc/ivas_sce_enc.c +++ b/lib_enc/ivas_sce_enc.c @@ -181,7 +181,7 @@ ivas_error ivas_sce_enc( *----------------------------------------------------------------*/ #ifdef DEBUGGING - st_ivas->noClipping += check_clipping( hSCE->hCoreCoder[0]->input, input_frame ); + st_ivas->noClipping += check_clipping( hSCE->hCoreCoder[0]->input, input_frame, &st_ivas->maxOverload, &st_ivas->minOverload ); #endif error = pre_proc_front_ivas( hSCE, NULL, hSCE->element_brate, nb_bits_metadata, input_frame, 0, old_inp_12k8[0], old_inp_16k[0], diff --git a/lib_enc/ivas_stat_enc.h b/lib_enc/ivas_stat_enc.h index 5a0dd72e6f..ad844b95d5 100644 --- a/lib_enc/ivas_stat_enc.h +++ b/lib_enc/ivas_stat_enc.h @@ -1248,6 +1248,8 @@ typedef struct CPE_ENC_HANDLE hCPE[MCT_MAX_BLOCKS]; /* CPE handles */ #ifdef DEBUGGING int32_t noClipping; /* number of clipped samples */ + float maxOverload; /* Maximum overload value */ + float minOverload; /* Maximum overload value */ #endif /* multichannel modules */ diff --git a/lib_enc/lib_enc.c b/lib_enc/lib_enc.c index dc62941199..acc1358db6 100644 --- a/lib_enc/lib_enc.c +++ b/lib_enc/lib_enc.c @@ -2427,9 +2427,13 @@ static void init_encoder_config( *---------------------------------------------------------------------*/ int32_t IVAS_ENC_GetNoCLipping( - IVAS_ENC_HANDLE hIvasEnc /* i : IVAS encoder handle */ + IVAS_ENC_HANDLE hIvasEnc, /* i : IVAS encoder handle */ + float *maxOverload, /* o : Max overload value */ + float *minOverload /* o : Min overload value */ ) { + *maxOverload = hIvasEnc->st_ivas->maxOverload; + *minOverload = hIvasEnc->st_ivas->minOverload; return hIvasEnc->st_ivas->noClipping; } #endif diff --git a/lib_enc/lib_enc.h b/lib_enc/lib_enc.h index 1b1929e755..7ece666652 100644 --- a/lib_enc/lib_enc.h +++ b/lib_enc/lib_enc.h @@ -385,7 +385,9 @@ void IVAS_ENC_PrintDisclaimer( #ifdef DEBUGGING int32_t IVAS_ENC_GetNoCLipping( - IVAS_ENC_HANDLE hIvasEnc /* i : IVAS encoder handle */ + IVAS_ENC_HANDLE hIvasEnc, /* i : IVAS encoder handle */ + float *maxOverload, /* o : Max overload value */ + float *minOverload /* o : Min overload value */ ); #endif /* clang-format on */ diff --git a/scripts/parse_xml_report.py b/scripts/parse_xml_report.py index 41004d2363..ef748e1c6f 100644 --- a/scripts/parse_xml_report.py +++ b/scripts/parse_xml_report.py @@ -79,7 +79,7 @@ if __name__ == "__main__": FORMATS = IVAS_FORMATS CATEGORIES = IVAS_CATEGORIES if args.clipping: - PROPERTIES += ["ENC_CORE_CLIPPING"] + PROPERTIES += ["ENC_CORE_OVL","MAX_OVL","MIN_OVL"] tree = ElementTree.parse(xml_report) testsuite = tree.find(".//testsuite") diff --git a/tests/conftest.py b/tests/conftest.py index c9422bb285..d460de646f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,8 +57,12 @@ from .constants import ( ODG, MAX_ENC_DIFF, MAX_ENC_DIFF_PATTERN, - ENC_CORE_CLIPPING, - CLIPPING_PATTERN, + ENC_CORE_OVL, + ENC_CORE_OVL_PATTERN, + MAX_OVL, + MAX_OVL_PATTERN, + MIN_OVL, + MIN_OVL_PATTERN, SCRIPTS_DIR, ) @@ -531,11 +535,12 @@ class EncoderFrontend: self.stdout = result.stdout.decode("ascii") # Record core encoder clipping - clipping = 0 - search_result = re.search(CLIPPING_PATTERN, self.stdout) - if search_result: - clipping = search_result.group(1) - record_property( ENC_CORE_CLIPPING, clipping ) + for (prop, pattern) in [(ENC_CORE_OVL,ENC_CORE_OVL_PATTERN),(MAX_OVL,MAX_OVL_PATTERN),(MIN_OVL,MIN_OVL_PATTERN)]: + val = 0 + search_result = re.search(pattern, self.stdout) + if search_result: + val = search_result.group(1) + record_property( prop, val ) if self.stdout: stdout_str = textwrap.indent(self.stdout, prefix="\t") diff --git a/tests/constants.py b/tests/constants.py index 8b6bd6beed..36dabd0584 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -12,7 +12,9 @@ MAX_ABS_DIFF = "MAXIMUM ABS DIFF" SSNR = "SSNR" ODG = "ODG" MAX_ENC_DIFF = "MAXIMUM ENC DIFF" -ENC_CORE_CLIPPING = "ENC_CORE_CLIPPING" +ENC_CORE_OVL = "ENC_CORE_OVL" +MAX_OVL = "MAX_OVL" +MIN_OVL = "MIN_OVL" # regex patterns for parsing the output from comparisons -> mainly for BASOP ci MLD_PATTERN = r"MLD: ([\d\.]*)" @@ -21,7 +23,9 @@ ODG_PATTERN_PQEVALAUDIO = r"Objective Difference Grade: (-*\d*\.\d*)" ODG_PATTERN = r"ODG: (-*\d*\.\d*)" SSNR_PATTERN = r"Channel \d* SSNR: (nan|[+-]*inf|[-*\d\.]*)" MAX_ENC_DIFF_PATTERN = r"MAXIMUM ENC DIFF: (\d+) \((\d+\.\d+)%\)" -CLIPPING_PATTERN = r"Clipping \(saturation\) detected: (\d+)" +ENC_CORE_OVL_PATTERN = r"Core input overload detected: (\d+)" +MAX_OVL_PATTERN = r"Max overload value: (\d+\.\d+)" +MIN_OVL_PATTERN = r"Min overload value: (-*\d+\.\d+)" # minimum difference between ref and dut encoder file lengths -- GitLab From ead242c639af8732069aeefd591b84389e7b637d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 7 Oct 2024 14:01:27 +0200 Subject: [PATCH 10/15] Add record_property for encoder tests. Remove check for DuT encoder if Ref is run. --- .gitlab-ci.yml | 2 +- tests/conftest.py | 38 ++++++++++++-------- tests/test_26444.py | 1 + tests/test_be_for_ext_outputs.py | 2 ++ tests/test_be_for_jbm_neutral_dly_profile.py | 3 +- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00181130a5..67655a5731 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1665,7 +1665,7 @@ check-clipping: - *enable-debugging-macro - make -j - tests/scale_pcm.py ./scripts/testv/ 3.162 # +10 dB level - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping artifacts: name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" diff --git a/tests/conftest.py b/tests/conftest.py index d460de646f..517ac03c52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -357,6 +357,8 @@ def dut_encoder_path(request) -> str: """ if request.config.option.dut_encoder_path: return request.config.option.dut_encoder_path + if request.config.option.update_ref == "1": + return None here = Path(__file__).parent.resolve() system = platform.system() @@ -568,16 +570,20 @@ class EncoderFrontend: @pytest.fixture(scope="function") -def dut_encoder_frontend(dut_encoder_path, request) -> EncoderFrontend: +def dut_encoder_frontend(dut_encoder_path, request) -> Union[None, EncoderFrontend]: """ Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session. """ - timeout = request.config.getoption("--testcase_timeout") - encoder = EncoderFrontend(dut_encoder_path, "DUT", timeout=timeout) + encoder = None + + if dut_encoder_path: + timeout = request.config.getoption("--testcase_timeout") + encoder = EncoderFrontend(dut_encoder_path, "DUT", timeout=timeout) yield encoder - # Fixture teardown - encoder._check_run() + if encoder is not None: + # Fixture teardown + encoder._check_run() @pytest.fixture(scope="session") @@ -636,6 +642,9 @@ def dut_decoder_path(request) -> str: if request.config.option.dut_decoder_path: return request.config.option.dut_decoder_path + if request.config.option.update_ref == "1": + return None + here = Path(__file__).parent.resolve() system = platform.system() @@ -830,20 +839,21 @@ class DecoderFrontend: @pytest.fixture(scope="function") -def dut_decoder_frontend(dut_decoder_path, request) -> DecoderFrontend: +def dut_decoder_frontend(dut_decoder_path, request) -> Union[None, DecoderFrontend]: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ - decoder = DecoderFrontend( - dut_decoder_path, - "DUT", - timeout=request.config.getoption("--testcase_timeout"), - fr=request.config.option.dut_fr, - ) + decoder = None + + if dut_decoder_path: + timeout = request.config.getoption("--testcase_timeout") + decoder = DecoderFrontend(dut_decoder_path, "DUT", timeout=timeout, fr=request.config.option.dut_fr) + yield decoder - # Fixture teardown - decoder._check_run() + if decoder is not None: + # Fixture teardown + decoder._check_run() @pytest.fixture(scope="session") diff --git a/tests/test_26444.py b/tests/test_26444.py index fa6214c66a..16367b6d21 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -104,6 +104,7 @@ def test_evs_26444( add_option_list = args[:-4] dut_encoder_frontend.run( + record_property, bitrate, sampling_rate, in_file, diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py index 9be0966acd..edca277ec6 100644 --- a/tests/test_be_for_ext_outputs.py +++ b/tests/test_be_for_ext_outputs.py @@ -74,6 +74,7 @@ def get_options(inp_format): @pytest.mark.parametrize("inp_format,inp_file", zip(INPUT_FORMATS, INPUT_FILES)) @pytest.mark.parametrize("bitrate", BITRATES) def test_be_for_ext_output( + record_property, inp_format, inp_file, bitrate, @@ -92,6 +93,7 @@ def test_be_for_ext_output( input_file = TESTV_DIR.joinpath(inp_file) options = get_options(inp_format) dut_encoder_frontend.run( + record_property, bitrate, sampling_rate, input_file, diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index 61bb168a2f..e7855d021f 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -150,7 +150,7 @@ INPUT_FILES = { @pytest.mark.parametrize("in_format,bitrate,out_format,dtx", TESTCASES) def test_be_for_jbm_neutral_dly_profile( - in_format, bitrate, out_format, dtx, dut_encoder_frontend, dut_decoder_frontend + record_property, in_format, bitrate, out_format, dtx, dut_encoder_frontend, dut_decoder_frontend ): with TemporaryDirectory() as tmp_dir: tmp_dir = pathlib.Path(tmp_dir) @@ -161,6 +161,7 @@ def test_be_for_jbm_neutral_dly_profile( input_file = TESTV_DIR.joinpath(INPUT_FILES[in_format]) options = get_options(in_format, bitrate, dtx == DTX_ON) dut_encoder_frontend.run( + record_property, bitrate, sampling_rate_khz, input_file, -- GitLab From e85b95895ee78774e070578774098107a1550ef9 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 8 Oct 2024 08:45:08 +0200 Subject: [PATCH 11/15] Added record_property to EncoderFrontend/DecoderFrontend to reduce clutter --- .../test_param_file.py | 9 ++---- tests/codec_be_on_mr_nonselection/test_sba.py | 32 +++---------------- tests/conftest.py | 25 ++++++++------- tests/test_be_for_ext_outputs.py | 2 -- tests/test_be_for_jbm_neutral_dly_profile.py | 3 +- 5 files changed, 21 insertions(+), 50 deletions(-) diff --git a/tests/codec_be_on_mr_nonselection/test_param_file.py b/tests/codec_be_on_mr_nonselection/test_param_file.py index ce49b52998..3fe21898e4 100644 --- a/tests/codec_be_on_mr_nonselection/test_param_file.py +++ b/tests/codec_be_on_mr_nonselection/test_param_file.py @@ -139,7 +139,6 @@ def convert_test_string_to_tag(test_string): # hack to have stv/ltv/evs in the test name @pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID]) def test_param_file_tests( - record_property, props_to_record, encoder_only, decoder_only, @@ -200,7 +199,6 @@ def test_param_file_tests( if not decoder_only: encode( - record_property, dut_encoder_frontend, ref_encoder_frontend, reference_path, @@ -235,7 +233,7 @@ def test_param_file_tests( cmp_result_msg += enc_test_result_msg props = parse_properties(cmp_result_msg, False, props_to_record) for k, v in props.items(): - record_property(k, v) + dut_encoder_frontend.record_property(k, v) if enc_test_result: pytest.fail("Too high difference in encoder statistics found.") @@ -421,7 +419,7 @@ def test_param_file_tests( props = parse_properties(cmp_result_msg, output_differs, props_to_record) for k, v in props.items(): - record_property(k, v) + dut_decoder_frontend.record_property(k, v) metadata_differs = False for md_file in md_out_files: @@ -477,7 +475,6 @@ def test_param_file_tests( def encode( - record_property, dut_encoder_frontend, ref_encoder_frontend, reference_path, @@ -513,7 +510,6 @@ def encode( # call REF encoder ref_encoder_frontend.run( - record_property, bitrate, sampling_rate, testv_file, @@ -527,7 +523,6 @@ def encode( # call DUT encoder dut_encoder_frontend.run( - record_property, bitrate, sampling_rate, testv_file, diff --git a/tests/codec_be_on_mr_nonselection/test_sba.py b/tests/codec_be_on_mr_nonselection/test_sba.py index d0adb52d31..e5b78cea93 100644 --- a/tests/codec_be_on_mr_nonselection/test_sba.py +++ b/tests/codec_be_on_mr_nonselection/test_sba.py @@ -92,7 +92,6 @@ def check_and_makedir(dir_path): @pytest.mark.parametrize("tag", tag_list) @pytest.mark.parametrize("sampling_rate", sample_rate_list) def test_pca_enc( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -138,7 +137,6 @@ def test_pca_enc( if not decoder_only: sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -164,7 +162,6 @@ def test_pca_enc( if not encoder_only: sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -198,7 +195,6 @@ def test_pca_enc( @pytest.mark.parametrize("gain_flag", gain_list) @pytest.mark.parametrize("sid", SID_list) def test_sba_enc_system( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -272,7 +268,6 @@ def test_sba_enc_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -325,7 +320,7 @@ def test_sba_enc_system( enc_test_result_msg, enc_test_result != 0, props_to_record ) for k, v in props.items(): - record_property(k, v) + dut_encoder_frontend.record_property(k, v) # report compare result if enc_test_result != 0: @@ -333,7 +328,6 @@ def test_sba_enc_system( if not encoder_only: sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -363,7 +357,6 @@ def test_sba_enc_system( @pytest.mark.parametrize("bitrate", ivas_br_HOA2) @pytest.mark.parametrize("tag", tag_list_HOA2) def test_spar_hoa2_enc_system( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -407,7 +400,6 @@ def test_spar_hoa2_enc_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -460,7 +452,7 @@ def test_spar_hoa2_enc_system( enc_test_result_msg, enc_test_result != 0, props_to_record ) for k, v in props.items(): - record_property(k, v) + dut_encoder_frontend.record_property(k, v) # report compare result if enc_test_result != 0: @@ -468,7 +460,6 @@ def test_spar_hoa2_enc_system( if not encoder_only: sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -498,7 +489,6 @@ def test_spar_hoa2_enc_system( @pytest.mark.parametrize("bitrate", ivas_br_HOA3) @pytest.mark.parametrize("tag", tag_list_HOA3) def test_spar_hoa3_enc_system( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -542,7 +532,6 @@ def test_spar_hoa3_enc_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -589,7 +578,7 @@ def test_spar_hoa3_enc_system( enc_test_result_msg, enc_test_result != 0, props_to_record ) for k, v in props.items(): - record_property(k, v) + dut_encoder_frontend.record_property(k, v) # report compare result if enc_test_result != 0: @@ -597,7 +586,6 @@ def test_spar_hoa3_enc_system( if not encoder_only: sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -629,7 +617,6 @@ def test_spar_hoa3_enc_system( @pytest.mark.parametrize("tag", tag_list_bw_force) @pytest.mark.parametrize("sample_rate_bw_idx", sample_rate_bw_idx_list) def test_sba_enc_BWforce_system( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -685,7 +672,6 @@ def test_sba_enc_BWforce_system( if not decoder_only: ref_stats_file, dut_stats_file = sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -738,7 +724,7 @@ def test_sba_enc_BWforce_system( enc_test_result_msg, enc_test_result != 0, props_to_record ) for k, v in props.items(): - record_property(k, v) + dut_encoder_frontend.record_property(k, v) # report compare result if enc_test_result != 0: @@ -746,7 +732,6 @@ def test_sba_enc_BWforce_system( if not encoder_only: sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -780,7 +765,6 @@ def test_sba_enc_BWforce_system( @pytest.mark.parametrize("sampling_rate", sample_rate_list) @pytest.mark.parametrize("gain_flag", gain_list) def test_sba_plc_system( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -850,7 +834,6 @@ def test_sba_plc_system( if not decoder_only: sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -876,7 +859,6 @@ def test_sba_plc_system( if not encoder_only: sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -907,7 +889,6 @@ def test_sba_plc_system( def sba_enc( - record_property, dut_encoder_frontend, test_vector_path, ref_encoder_frontend, @@ -984,7 +965,6 @@ def sba_enc( if update_ref == 1: # call REF encoder ref_encoder_frontend.run( - record_property, bitrate, sampling_rate, input_path, @@ -999,7 +979,6 @@ def sba_enc( if update_ref == 0: # call DUT encoder dut_encoder_frontend.run( - record_property, bitrate, sampling_rate, input_path, @@ -1033,7 +1012,6 @@ def sba_enc( def sba_dec( - record_property, props_to_record, dut_decoder_frontend, ref_decoder_frontend, @@ -1140,7 +1118,7 @@ def sba_dec( text_to_parse = reason props = parse_properties(text_to_parse, cmp_result != 0, props_to_record) for k, v in props.items(): - record_property(k, v) + dut_decoder_frontend.record_property(k, v) # report compare result if cmp_result != 0: diff --git a/tests/conftest.py b/tests/conftest.py index 517ac03c52..b7a0c1158e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -379,13 +379,14 @@ def dut_encoder_path(request) -> str: class EncoderFrontend: - def __init__(self, path, enc_type, timeout=None) -> None: + def __init__(self, path, enc_type, record_property, timeout=None) -> None: self._path = Path(path).absolute() self._type = enc_type self.returncode = None self.stdout = None self.stderr = None self.timeout = timeout + self.record_property = record_property def extract_enc_stats( self, @@ -463,7 +464,6 @@ class EncoderFrontend: def run( self, - record_property, bitrate: Union[int, Path], input_sampling_rate: int, input_path: Path, @@ -542,7 +542,7 @@ class EncoderFrontend: search_result = re.search(pattern, self.stdout) if search_result: val = search_result.group(1) - record_property( prop, val ) + self.record_property( prop, val ) if self.stdout: stdout_str = textwrap.indent(self.stdout, prefix="\t") @@ -570,7 +570,7 @@ class EncoderFrontend: @pytest.fixture(scope="function") -def dut_encoder_frontend(dut_encoder_path, request) -> Union[None, EncoderFrontend]: +def dut_encoder_frontend(dut_encoder_path, request, record_property) -> Union[None, EncoderFrontend]: """ Return a :class:`conftest.EncoderFrontend` instance as DUT for the test session. """ @@ -578,7 +578,7 @@ def dut_encoder_frontend(dut_encoder_path, request) -> Union[None, EncoderFronte if dut_encoder_path: timeout = request.config.getoption("--testcase_timeout") - encoder = EncoderFrontend(dut_encoder_path, "DUT", timeout=timeout) + encoder = EncoderFrontend(dut_encoder_path, "DUT", record_property, timeout=timeout) yield encoder if encoder is not None: @@ -617,7 +617,7 @@ def ref_encoder_path(request) -> str: @pytest.fixture(scope="function") -def ref_encoder_frontend(ref_encoder_path, request) -> Union[None, EncoderFrontend]: +def ref_encoder_frontend(ref_encoder_path, request, record_property) -> Union[None, EncoderFrontend]: """ Return a :class:`conftest.EncoderFrontend` instance as REF for the test session. """ @@ -625,7 +625,7 @@ def ref_encoder_frontend(ref_encoder_path, request) -> Union[None, EncoderFronte if ref_encoder_path: timeout = request.config.getoption("--testcase_timeout") - encoder = EncoderFrontend(ref_encoder_path, "REF", timeout=timeout) + encoder = EncoderFrontend(ref_encoder_path, "REF", record_property, timeout=timeout) yield encoder @@ -664,7 +664,7 @@ def dut_decoder_path(request) -> str: class DecoderFrontend: - def __init__(self, path, dec_type, timeout=None, fr=20) -> None: + def __init__(self, path, dec_type, record_property, timeout=None, fr=20) -> None: self._path = str(Path(path).absolute()) self._type = dec_type self.returncode = None @@ -672,6 +672,7 @@ class DecoderFrontend: self.stderr = None self.timeout = timeout self.fr = fr + self.record_property = record_property def run( self, @@ -839,7 +840,7 @@ class DecoderFrontend: @pytest.fixture(scope="function") -def dut_decoder_frontend(dut_decoder_path, request) -> Union[None, DecoderFrontend]: +def dut_decoder_frontend(dut_decoder_path, request, record_property) -> Union[None, DecoderFrontend]: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ @@ -847,7 +848,7 @@ def dut_decoder_frontend(dut_decoder_path, request) -> Union[None, DecoderFronte if dut_decoder_path: timeout = request.config.getoption("--testcase_timeout") - decoder = DecoderFrontend(dut_decoder_path, "DUT", timeout=timeout, fr=request.config.option.dut_fr) + decoder = DecoderFrontend(dut_decoder_path, "DUT", record_property, timeout=timeout, fr=request.config.option.dut_fr) yield decoder @@ -887,7 +888,7 @@ def ref_decoder_path(request) -> str: @pytest.fixture(scope="function") -def ref_decoder_frontend(ref_decoder_path, request) -> Union[None, DecoderFrontend]: +def ref_decoder_frontend(ref_decoder_path, request, record_property) -> Union[None, DecoderFrontend]: """ Return a :class:`conftest.DecoderFrontend` instance as DUT for the test session. """ @@ -895,7 +896,7 @@ def ref_decoder_frontend(ref_decoder_path, request) -> Union[None, DecoderFronte if ref_decoder_path: timeout = request.config.getoption("--testcase_timeout") - decoder = DecoderFrontend(ref_decoder_path, "REF", timeout=timeout) + decoder = DecoderFrontend(ref_decoder_path, "REF", record_property, timeout=timeout) yield decoder diff --git a/tests/test_be_for_ext_outputs.py b/tests/test_be_for_ext_outputs.py index edca277ec6..9be0966acd 100644 --- a/tests/test_be_for_ext_outputs.py +++ b/tests/test_be_for_ext_outputs.py @@ -74,7 +74,6 @@ def get_options(inp_format): @pytest.mark.parametrize("inp_format,inp_file", zip(INPUT_FORMATS, INPUT_FILES)) @pytest.mark.parametrize("bitrate", BITRATES) def test_be_for_ext_output( - record_property, inp_format, inp_file, bitrate, @@ -93,7 +92,6 @@ def test_be_for_ext_output( input_file = TESTV_DIR.joinpath(inp_file) options = get_options(inp_format) dut_encoder_frontend.run( - record_property, bitrate, sampling_rate, input_file, diff --git a/tests/test_be_for_jbm_neutral_dly_profile.py b/tests/test_be_for_jbm_neutral_dly_profile.py index e7855d021f..61bb168a2f 100644 --- a/tests/test_be_for_jbm_neutral_dly_profile.py +++ b/tests/test_be_for_jbm_neutral_dly_profile.py @@ -150,7 +150,7 @@ INPUT_FILES = { @pytest.mark.parametrize("in_format,bitrate,out_format,dtx", TESTCASES) def test_be_for_jbm_neutral_dly_profile( - record_property, in_format, bitrate, out_format, dtx, dut_encoder_frontend, dut_decoder_frontend + in_format, bitrate, out_format, dtx, dut_encoder_frontend, dut_decoder_frontend ): with TemporaryDirectory() as tmp_dir: tmp_dir = pathlib.Path(tmp_dir) @@ -161,7 +161,6 @@ def test_be_for_jbm_neutral_dly_profile( input_file = TESTV_DIR.joinpath(INPUT_FILES[in_format]) options = get_options(in_format, bitrate, dtx == DTX_ON) dut_encoder_frontend.run( - record_property, bitrate, sampling_rate_khz, input_file, -- GitLab From 723afe0c5d63e0678eaff8013a2514328cf83b03 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 8 Oct 2024 08:49:45 +0200 Subject: [PATCH 12/15] Cleanup in test_26444.py --- tests/test_26444.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_26444.py b/tests/test_26444.py index 16367b6d21..08d0ff77c5 100644 --- a/tests/test_26444.py +++ b/tests/test_26444.py @@ -78,7 +78,6 @@ for s in scripts: @pytest.mark.parametrize("test_tag", list(test_dict.keys())) def test_evs_26444( - record_property, props_to_record, dut_encoder_frontend: EncoderFrontend, dut_decoder_frontend: DecoderFrontend, @@ -104,7 +103,6 @@ def test_evs_26444( add_option_list = args[:-4] dut_encoder_frontend.run( - record_property, bitrate, sampling_rate, in_file, @@ -181,7 +179,7 @@ def test_evs_26444( props = parse_properties(reason, output_differs, props_to_record) for k, v in props.items(): - record_property(k, v) + ref_decoder_frontend.record_property(k, v) equal &= not output_differs else: -- GitLab From ef5eb1fc9e4fb97b155e727b569a4425871bb480 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 15 Oct 2024 09:10:15 +0200 Subject: [PATCH 13/15] Added SCALE_FACTOR variable for check-clipping job --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42b4548652..8410760881 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ variables: TESTCASE_TIMEOUT_STV_SANITIZERS: 180 TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 BASOP_REFERENCE_BRANCH: "ivas-float-update" + SCALE_FACTOR: 3.162 # Default scaling for check-clipping: +10 dB. Can be set when triggered from web. default: @@ -1669,7 +1670,7 @@ check-clipping: - *print-common-info - *enable-debugging-macro - make -j - - tests/scale_pcm.py ./scripts/testv/ 3.162 # +10 dB level + - tests/scale_pcm.py ./scripts/testv/ $SCALE_FACTOR # Default: 3.162 (+10 dB). Can be set in manual trigger. - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping artifacts: -- GitLab From bcfdc868f23253c7e13126322715dac6ed79d287 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 15 Oct 2024 09:15:11 +0200 Subject: [PATCH 14/15] Fix for SCALE_FACTOR definition, removed comment --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8410760881..366e66403a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,7 +31,7 @@ variables: TESTCASE_TIMEOUT_STV_SANITIZERS: 180 TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 BASOP_REFERENCE_BRANCH: "ivas-float-update" - SCALE_FACTOR: 3.162 # Default scaling for check-clipping: +10 dB. Can be set when triggered from web. + SCALE_FACTOR: 3.162 default: -- GitLab From 50fdbfb084f82c0e72ce06859f6f7cecb9e004c8 Mon Sep 17 00:00:00 2001 From: norvell Date: Tue, 15 Oct 2024 07:17:14 +0000 Subject: [PATCH 15/15] Update .gitlab-ci.yml: SCALE_FACTOR: "3.162" --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 366e66403a..e8099bb83d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,7 +31,7 @@ variables: TESTCASE_TIMEOUT_STV_SANITIZERS: 180 TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 BASOP_REFERENCE_BRANCH: "ivas-float-update" - SCALE_FACTOR: 3.162 + SCALE_FACTOR: "3.162" default: -- GitLab