Commit 119c20d1 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'basop-ci/remove-test-param-file-ltv' into 'main' 

[BASOP CI] Remove test param file ltv

See merge request !1603
parents f46604c1 0565a640

.gitlab-ci.yml

+9 −9
Original line number Diff line number Diff line
@@ -227,12 +227,12 @@ stages: 
    - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi



    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi
@@ -561,7 +561,7 @@ codec-msan: 
  before_script:

    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with address-sanitizer binaries
@@ -571,7 +571,7 @@ codec-asan: 
  before_script:

    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with undefined-behaviour-sanitizer binaries
@@ -583,7 +583,7 @@ codec-usan: 
  before_script:

    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

  <<: *sanitizer-selftest-anchor



# compare bit-exactness between 5ms and 20 on the branch
@@ -1667,7 +1667,7 @@ ltv-msan: 
  before_script:

    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"

    - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"

  <<: *sanitizer-selftest-anchor



# code selftest long testvectors with address-sanitizer binaries
@@ -1684,7 +1684,7 @@ ltv-asan: 
  before_script:

    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"

    - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"

  <<: *sanitizer-selftest-anchor



# code selftest long testvectors with undefined-behaviour-sanitizer binaries
@@ -1701,7 +1701,7 @@ ltv-usan: 
  before_script:

    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"

    - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"

  <<: *sanitizer-selftest-anchor



.sanitizer-test-template:

tests/codec_be_on_mr_nonselection/test_param_file.py

+5 −0
Original line number Diff line number Diff line
@@ -63,6 +63,8 @@ VALID_DEC_OUTPUT_CONF = [ 
    "EXT",

]



PARAM_FILE_ID = "stv" if PARAM_FILE.stem == "self_test" else PARAM_FILE.stem.replace("self_test_", "")



param_file_test_dict = {}

with open(PARAM_FILE, "r", encoding="UTF-8") as fp:

    data = fp.read()
@@ -125,6 +127,8 @@ def convert_test_string_to_tag(test_string): 


@pytest.mark.create_ref

@pytest.mark.parametrize("test_tag", list(param_file_test_dict.keys()))

# hack to have stv/ltv/evs in the test name

@pytest.mark.parametrize("param_file_id", [PARAM_FILE_ID])

def test_param_file_tests(

    record_property,

    decoder_only,
@@ -139,6 +143,7 @@ def test_param_file_tests( 
    rootdir,

    keep_files,

    test_tag,

    param_file_id,

    get_mld,

    get_mld_lim,

    abs_tol,

tests/conftest.py

+1 −0
Original line number Diff line number Diff line
@@ -128,6 +128,7 @@ def pytest_addoption(parser): 
    parser.addoption(

        "--param_file",

        action="store",

        type=Path,

        help="If specified, use given param file in test_param_file.",

    )

tests/test_param_file_ltv.py

deleted100644 → 0
+0 −610

File deleted.

Preview size limit exceeded, changes collapsed.

tests/testconfig.py

+3 −1
Original line number Diff line number Diff line
@@ -32,6 +32,8 @@ __doc__ = """ 
To configure test modules.

"""



PARAM_FILE = "scripts/config/self_test.prm"

from pathlib import Path



PARAM_FILE = Path("scripts/config/self_test.prm")



MD5_REF_DICT = dict()