Commit baeef46b authored by Jan Kiene's avatar Jan Kiene
Browse files

add cmdl arg for tolerance to have better error reporting

parent d5e1ad76

.gitlab-ci.yml

+4 −14
Original line number Diff line number Diff line
@@ -206,26 +206,16 @@ stages: 
    - exit_code20=0

    - exit_code10=0

    - exit_code5=0

    - errors20=0

    - errors10=0

    - errors5=0



    - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi



    # ignore exit codes, instead search in the report for errors

    - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    - errors20=$(cat report-20ms.html | grep -c "$sanitizer_type") || true

    # run with high enough tolerance to have no failures due to non-BE -> the only detected failures will be crashes from the sanitizers

    - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$?

    - errors10=$(cat report-10ms.html | grep -c "$sanitizer_type") || true



    - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$?

    - errors5=$(cat report-5ms.html | grep -c "sanitizer_type") || true



    - if [ $errors20 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 20ms framesize decoding"; fi

    - if [ $errors10 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 10ms framesize decoding"; fi

    - if [ $errors5 -ne 0 ]; then echo "CLANG $sanitizer_type found errors in 5ms framesize decoding"; fi

    - if [ $errors20 -ne 0 ] || [ $errors10 -ne 0 ] || [ $errors5 -ne 0 ]; then exit 1; fi

    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi





# ---------------------------------------------------------------

tests/cmp_pcm.py

+3 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ import pyaudio3dtools 
import pyivastest





def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str):

def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0, abs_tol=0) -> (int, str):

    """

    Compare 2 PCM files for bitexactness

    """
@@ -51,6 +51,8 @@ def cmp_pcm(file1, file2, out_config, fs, get_mld=False, mld_lim=0) -> (int, str 


    if cmp_result["bitexact"]:

        return 0, "SUCCESS: Files are bitexact"

    elif cmp_result["max_abs_diff"] <= abs_tol:

        return 0, "SUCCESS: Maximum absolute diff below threshold"

    else:

        diff_msg = f"MAXIMUM ABS DIFF ==> {cmp_result['max_abs_diff']} at sample num {cmp_result['max_abs_diff_pos_sample']} (assuming {nchannels} channels)"

        first_msg = f"First diff found at sample num {cmp_result['first_diff_pos_sample']} in channel {cmp_result['first_diff_pos_channel']}, frame {cmp_result['first_diff_pos_frame']} (assuming {nchannels} channels, {fs} sampling rate)"

tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

+3 −0
Original line number Diff line number Diff line
@@ -110,6 +110,7 @@ def test_masa_enc_dec( 
    get_mld,

    get_mld_lim,

    decoder_only,

    abs_tol,

):

    # Input parameters

    in_fs = 48
@@ -214,6 +215,7 @@ def test_masa_enc_dec( 
                int(out_fs * 1000),

                get_mld=get_mld,

                mld_lim=get_mld_lim,

                abs_tol=abs_tol,

            )

            if get_mld:

                mld = 0
@@ -246,6 +248,7 @@ def test_masa_enc_dec( 
                    int(out_fs * 1000),

                    get_mld=get_mld,

                    mld_lim=get_mld_lim,

                    abs_tol=abs_tol,

                )

                if get_mld:

                    mld = 0

tests/codec_be_on_mr_nonselection/test_param_file.py

+2 −0
Original line number Diff line number Diff line
@@ -141,6 +141,7 @@ def test_param_file_tests( 
    test_tag,

    get_mld,

    get_mld_lim,

    abs_tol,

):

    enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]
@@ -338,6 +339,7 @@ def test_param_file_tests( 
            fs,

            get_mld=get_mld,

            mld_lim=get_mld_lim,

            abs_tol=abs_tol,

        )

        md_out_files = get_expected_md_files(ref_output_file, enc_split, output_config)

tests/codec_be_on_mr_nonselection/test_sba_bs_dec_plc.py

+4 −0
Original line number Diff line number Diff line
@@ -90,6 +90,7 @@ def test_sba_plc_system( 
    gain_flag,

    get_mld,

    get_mld_lim,

    abs_tol,

):

    SID = 0

    if dtx == "1" and ivas_br not in ["13200", "16400", "24400", "32000", "64000"]:
@@ -128,6 +129,7 @@ def test_sba_plc_system( 
        keep_files,

        get_mld=get_mld,

        get_mld_lim=get_mld_lim,

        abs_tol=abs_tol,

    )
@@ -151,6 +153,7 @@ def sba_dec_plc( 
    keep_files,

    get_mld=False,

    get_mld_lim=0,

    abs_tol=0,

):

    # ------------  run cmd  ------------
@@ -207,6 +210,7 @@ def sba_dec_plc( 
            fs,

            get_mld=get_mld,

            mld_lim=get_mld_lim,

            abs_tol=abs_tol,

        )

        if get_mld:

            mld = 0