Commit 079480b4 authored by Tapani Pihlajakuja's avatar Tapani Pihlajakuja
Browse files

Merge branch 'main' into 128-add-a-test-to-merge-pipeline-for-commits-behind-target-branch

parents 6fb2fa21 52f6c004

.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ scripts/ref/ 
scripts/test/

scripts/out/

scripts/self_test_summary.txt

tests/dut

tests/ref



# Python files that pop up when running scripts

__pycache__/

.gitlab-ci.yml

+59 −72
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ variables: 
  OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"

  OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"

  OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM"

  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1





# This sets when pipelines are created. Jobs have more specific rules to restrict them.
@@ -273,7 +275,7 @@ asan-on-merge-request-linux: 




# compare bit exactness between target and source branch

self-test-on-merge-request:

pytest-on-merge-request:

  extends: 

    - .test-job-linux

    - .rules-merge-request
@@ -315,54 +317,51 @@ self-test-on-merge-request: 
    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..



    ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)

    ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything)

    - git checkout $source_branch_commit_sha



    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt

    ### analyse test output



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1

    - expected_nonbe_1=0

    - expected_nonbe_2=0

    - expected_nonbe_3=0

    - fail_1=0

    - fail_2=0

    - fail_3=0



    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi

    ### prepare pytest

    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi

    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi



    # check for non bitexact EVS output

    - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi

    - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi



    ### run SBA pytest

    ### run pytest for EVS cases

    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$?

    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi

    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi



    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi

    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi

    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi



    # Check results from all three tests

    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi

    # Check results from both tests

    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi

    - exit 0



  allow_failure:
@@ -372,14 +371,13 @@ self-test-on-merge-request: 
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - test_output.txt

      - test_output_evs.txt

      - scripts/test/logs/

      - scripts/ref/logs/

      - report-junit.xml

    expose_as: 'Self test results'

      - report-junit-evs.xml

    expose_as: 'pytest results'

    reports:

      junit: report-junit.xml

      junit:

        - report-junit.xml

        - report-junit-evs.xml





# ---------------------------------------------------------------
@@ -456,37 +454,26 @@ codec-comparison-on-main-push: 
    ### re-checkout the latest commit in the main branch

    - git checkout $latest_commit



    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    # helper variable - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[non[ -]*be\]") || true



    ### analyse test output

    ### prepare pytest

    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1



    - selftest_exit_code=0



    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi



    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi



    ### run SBA pytest

    ### run pytest

    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

    # return exit code from selftest if everything went well with the pytest run

    - exit $selftest_exit_code

  allow_failure:

    exit_codes:

      - 123
@@ -494,9 +481,6 @@ codec-comparison-on-main-push: 
    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - test_output.txt

      - scripts/test/logs/

      - scripts/ref/logs/

      - report-junit.xml

    expose_as: 'Results of comparison to previous merge commit'

    reports:
@@ -517,6 +501,8 @@ codec-comparison-on-main-push: 
    when: always

    paths:

      - ep_015.g192

      # second wildcard is necessary to get encoder and no-PLC run logs

      - "CLANG*/logs*"



sanitizer-test-mono:

  extends: .sanitizer-test-template
@@ -665,9 +651,10 @@ coverage-test-on-main-scheduled: 
  script:

    - *print-common-info

    - make GCOV=1 -j

    - ./scripts/self_test.py --create -t 1

    - ./scripts/self_test.py --create -t 1 scripts/config/self_test_evs.prm

    - ./scripts/ivas_pytests/self_test_b.py --create_only --numprocesses 1 --encref IVAS_cod --decref IVAS_dec --encdut IVAS_cod --decdut IVAS_dec

    - python3 tests/create_short_testvectors.py

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - lcov -c -d obj -o coverage.info

    - genhtml coverage.info -o coverage

  artifacts:

Workspace_msvc/lib_dec.vcxproj

+1 −0
Original line number Diff line number Diff line
@@ -267,6 +267,7 @@ 
    <ClCompile Include="..\lib_dec\ivas_allrad_dec.c" />

    <ClCompile Include="..\lib_dec\ivas_binauralRenderer.c" />

    <ClCompile Include="..\lib_dec\ivas_binaural_reverb.c" />

    <ClCompile Include="..\lib_dec\ivas_corecoder_dec_reconfig.c" />

    <ClCompile Include="..\lib_dec\ivas_core_dec.c" />

    <ClCompile Include="..\lib_dec\ivas_cpe_dec.c" />

    <ClCompile Include="..\lib_dec\ivas_crend.c" />

Workspace_msvc/lib_dec.vcxproj.filters

+3 −0
Original line number Diff line number Diff line
@@ -581,6 +581,9 @@ 
    <ClCompile Include="..\lib_dec\ivas_spar_md_dec.c">

      <Filter>dec_ivas_c</Filter>

    </ClCompile>

    <ClCompile Include="..\lib_dec\ivas_corecoder_dec_reconfig.c">

      <Filter>dec_ivas_c</Filter>

    </ClCompile>

  </ItemGroup>

  <ItemGroup>

    <ClInclude Include="..\lib_dec\jbm_jb4_inputbuffer.h">

Workspace_msvc/lib_enc.vcxproj

+1 −0
Original line number Diff line number Diff line
@@ -206,6 +206,7 @@ 
  </ItemDefinitionGroup>

  <ItemGroup>

    <ClCompile Include="..\lib_enc\ivas_agc_enc.c" />

    <ClCompile Include="..\lib_enc\ivas_corecoder_enc_reconfig.c" />

    <ClCompile Include="..\lib_enc\ivas_core_pre_proc.c" />

    <ClCompile Include="..\lib_enc\ivas_core_pre_proc_front.c" />

    <ClCompile Include="..\lib_enc\ivas_enc_cov_handler.c" />