- removed call to self_test.py - now covered by pytest (test_param_file)

variables:

  TESTV_DIR: "/usr/local/testv"

  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1

# prevent running two pipelines on pushes to merge request branches

# compare bit exactness between target and source branch

self-test-on-merge-request:

pytest-on-merge-request:

  extends: .test-job-linux

  stage: compare

  needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]

    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..

    ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)

    ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything)

    - git checkout $source_branch_commit_sha

    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    ### analyse test output

    # some helper variables - "|| true" to prevent failures from grep not finding anything

    # helper variable - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1

    - selftest_exit_code=0

    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi

    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi

    ### run SBA pytest

    ### prepare pytest

    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -n auto --update_ref 1 -m create_ref

    - python3 -m pytest tests -n auto --update_ref 1 -m create_ref_part2

    ### run pytest

    - exit_code=0

    - python3 ./tests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - python3 -m pytest tests -n auto --junit-xml=report-junit.xml || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'testsuite errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

    # return exit code from selftest if everything went well with the pytest run

    - exit $selftest_exit_code

  allow_failure:

    exit_codes:

      - 123

      - $EXIT_CODE_NON_BE

  artifacts:

    paths:

      - test_output.txt

      - scripts/test/logs

      - scripts/ref/logs

      - report-junit.xml

    reports:

      junit: report-junit.xml