Commit 3799c444 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/use-pytest-for-mr-sanitizer-tests' into 'main' 

[CI] use pytest for mr sanitizer tests

See merge request !1356
parents 0d466098 5813c4a0

.gitlab-ci.yml

+27 −15
Original line number Diff line number Diff line
@@ -449,17 +449,21 @@ codec-msan: 
    - *print-common-info

    - make clean

    - make -j CLANG=1

    - python3 scripts/self_test.py -z console --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi

    - testcase_timeout=180

    - exit_code=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?

    - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - scripts/ref/logs/

      - test_output.txt

      - report-junit.xml

      - report.html

    expose_as: "msan selftest results"

    reports:

      junit:

        - report-junit.xml



# code selftest testvectors with address-sanitizer binaries

codec-asan:
@@ -472,17 +476,21 @@ codec-asan: 
    - *print-common-info

    - make clean

    - make -j CLANG=2

    - python3 scripts/self_test.py -z console --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi

    - testcase_timeout=180

    - exit_code=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?

    - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - scripts/ref/logs/

      - test_output.txt

      - report-junit.xml

      - report.html

    expose_as: "asan selftest results"

    reports:

      junit:

        - report-junit.xml



# code selftest testvectors with address-sanitizer binaries

codec-usan:
@@ -495,17 +503,21 @@ codec-usan: 
    - *print-common-info

    - make clean

    - make -j CLANG=3

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 scripts/self_test.py -z console --create

    - grep_exit_code=0

    - grep UndefinedBehaviorSanitizer scripts/ref/logs/* || grep_exit_code=$?

    - if [ $grep_exit_code != 1 ] ; then echo "Run errors in self_test.py with Clang undefined-behavior-sanitizer"; exit 1; fi

    - testcase_timeout=180

    - exit_code=0

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?

    - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - scripts/ref/logs/

      - report-junit.xml

      - report.html

    expose_as: "usan selftest results"

    reports:

      junit:

        - report-junit.xml



# test renderer executable

renderer-smoke-test:

tests/conftest.py

+4 −0
Original line number Diff line number Diff line
@@ -292,6 +292,8 @@ class EncoderFrontend: 
            pytest.fail(

                f"{self._type} encoder terminated with a non-0 return code: {self.returncode}"

            )

        if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:

            pytest.fail("Undefined Behaviour runtime error encountered")



    def _check_run(self):

        if self.returncode is not None:
@@ -485,6 +487,8 @@ class DecoderFrontend: 
            pytest.fail(

                f"{self._type} decoder terminated with a non-0 return code: {self.returncode}"

            )

        if self.stderr and "UndefinedBehaviorSanitizer" in self.stderr:

            pytest.fail("Undefined Behaviour runtime error encountered")



    def _check_run(self):

        if self.returncode is not None: