Commit fd6dda87 authored by Jan Kiene's avatar Jan Kiene
Browse files

Revert "[revert-me] reduce number of tests for faster iteration" 

This reverts commit 8345f5f0.
parent 3cd7de85

.gitlab-ci.yml

+43 −43
Original line number Diff line number Diff line
@@ -1389,10 +1389,10 @@ ivas-conformance-linux: 
    # Reference creation

    - python3 scripts/prepare_combined_format_inputs.py

    - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py"

    - python3 -m pytest -k stereo -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files

    - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files



    # Output creation

    - python3 -m pytest -k stereo -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html

    - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html

    - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



    # Copy input data and output ref data
@@ -1440,7 +1440,7 @@ ivas-conformance-linux: 
    # Test run generated scripts in testvec

    - cd testvec

    - exit_code=0

    - python3 -m pytest -k stereo -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$?

    - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$?

    - mv report.html ..

    - mv report-junit.xml ..
@@ -2263,33 +2263,33 @@ coverage-test-on-main-scheduled-stv: 
  script:

    # -- Run short test vector suite to check coverage of that first

    # codec smoke test

    # - bash ci/smoke_test.sh coverage

    - bash ci/smoke_test.sh coverage

    # default renderer framesize is 20ms

    - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true

    # # run renderer smoke test - needed for REFVEC tests

    # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py

    # # run renderer test only with REF creation stage - no CUT needed since executables are identical

    # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref

    # # run split rendering smoke test and REF creation

    # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py

    # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref

    # # VoIP BE test

    # - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py

    # # split rendering VoIP BE test

    # - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py

    # # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted

    # # - bash ci/run-first-frame-is-sid-test.sh

    # # TODO consider HRTF binary loading test, or probably already covered by PRM files

    # # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading

    # # run BE vs EXT test

    # - python3 tests/create_short_testvectors.py --cut_len 1.0

    # - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py

    # # run LC3plus wrapper unit test

    # - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true

    # run renderer smoke test - needed for REFVEC tests

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py

    # run renderer test only with REF creation stage - no CUT needed since executables are identical

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref

    # run split rendering smoke test and REF creation

    - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py

    - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref

    # VoIP BE test

    - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py

    # split rendering VoIP BE test

    - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py

    # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted

    # - bash ci/run-first-frame-is-sid-test.sh

    # TODO consider HRTF binary loading test, or probably already covered by PRM files

    # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading

    # run BE vs EXT test

    - python3 tests/create_short_testvectors.py --cut_len 1.0

    - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py

    # run LC3plus wrapper unit test

    - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test

    # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job

  artifacts:

    paths:
@@ -2304,20 +2304,20 @@ coverage-test-on-main-scheduled-ltv: 
    COVERAGE_TITLE: "long test vectors"

  script:

    # -- Add extended coverage of EVS test and long test vectors

    - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # # default renderer framesize is 20ms

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true

    # - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true

    # # run renderer smoke test - needed for REFVEC tests

    # - ls -altr scripts/testv

    # # LTV smoketest is the same as REF creation stage for renderer

    # - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # default renderer framesize is 20ms

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true

    # run renderer smoke test - needed for REFVEC tests

    - ls -altr scripts/testv

    # LTV smoketest is the same as REF creation stage for renderer

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv

    # split rendering test doesn't support LTV - skipped here

    # VoIP BE tests don't support LTV - skipped here

  artifacts: