Commit f1990759 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/selection-be-test' into 'main' 

Implement BE test for selection testing operating points

See merge request !802
parents c13c31e9 90010405

.gitignore

+3 −4
Original line number Diff line number Diff line
@@ -42,10 +42,9 @@ scripts/out/ 
scripts/self_test_summary.txt

scripts/cppp/

binary/

tests/renderer/cut

tests/renderer/ref

tests/dut

tests/ref

tests/**/[c|d]ut

tests/**/ref

tests/*/testv

scripts/testv/*_cut*.pcm

# default reference binary name

IVAS_cod_ref

.gitlab-ci.yml

+12 −11
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ variables: 
  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1

  PROCESSING_SCRIPTS_BIN_DIR: "/test-bin"

  TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection"



default:

  interruptible: true # Make all jobs by default interruptible
@@ -531,15 +532,15 @@ ivas-pytest-on-merge-request: 
    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -579,14 +580,14 @@ evs-pytest-on-merge-request: 


    ### prepare pytest

    # create references

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest for EVS cases

    - exit_code=0

    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$?

    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -840,15 +841,15 @@ codec-comparison-on-main-push: 
    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2



    ### re-checkout the latest commit here, if ref_using_main is set

    - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
@@ -1126,9 +1127,9 @@ coverage-test-on-main-scheduled: 
    - make GCOV=1 -j

    - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script  

    - python3 tests/create_short_testvectors.py

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - bash ci/smoke_test.sh coverage

    - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py

    - bash ci/ivas_voip_be_test.sh coverage

tests/__init__.py

0 → 100644
+0 −0

Empty file added.

tests/codec_be_on_mr_nonselection/__init__.py

0 → 100644
+0 −0

Empty file added.

tests/test_masa_enc_dec.py→tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py

+2 −2
Original line number Diff line number Diff line
@@ -41,8 +41,8 @@ import pytest 
from typing import Optional

from filecmp import cmp



from cmp_pcm import cmp_pcm

from conftest import EncoderFrontend, DecoderFrontend

from tests.conftest import EncoderFrontend, DecoderFrontend

from tests.cmp_pcm import cmp_pcm



# params

#output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT']