From e6124230ddfeece8121536e7f10e079d9319ac11 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 13:54:01 +0100 Subject: [PATCH 01/10] add first test job for sanitizer run with short testvectors --- .gitlab-ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1067b210..7113a8ec6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,7 @@ variables: - 'pytest-mld' - 'pytest-mld-long' - 'evs-26444' + - 'msan-stv' default: @@ -311,6 +312,36 @@ ivas-pytest-mld-dec-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor +ivas-pytest-dec-msan: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "msan-stv" + script: + - *print-common-info + - *update-scripts-repo + - *remove-unsupported-testcases + - *mld-test-setup-codec + - make clean + - make -j CLANG=1 + - testcase_timeout=900 + - python3 -m pytest $SHORT_TEST_SUITE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec + artifacts: + name: "ivas-pytest-dec-msan--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: "2 weeks" + paths: + - report-junit.xml + - report.html + reports: + junit: + - report-junit.xml + # --------------------------------------------------------------- # Long test jobs # --------------------------------------------------------------- -- GitLab From b23d683f337329f74b9fccacac7bd7b20716785a Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 13:59:33 +0100 Subject: [PATCH 02/10] change workflow to allow manual run --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7113a8ec6..04d1de876 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,6 +51,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' variables: IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'msan-stv' + variables: + IVAS_PIPELINE_NAME: 'Short testvecor MSAN' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' -- GitLab From 91a89e4831e3c35c01516a3956eb6bf3f09f2baa Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 14:14:52 +0100 Subject: [PATCH 03/10] split setup anchor --- .gitlab-ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04d1de876..e85063a65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,9 +77,8 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" -.mld-test-setup-codec: &mld-test-setup-codec +.setup-codec: &setup-codec - current_commit_sha=$(git rev-parse HEAD) - ### build reference binaries - git checkout $REFERENCE_TAG - make clean @@ -92,6 +91,10 @@ stages: - make clean - make -j + +.mld-test-setup-codec: &mld-test-setup-codec + - *setup-codec + ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py @@ -131,7 +134,7 @@ stages: .copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir - cp "$LTV_DIR"/*.wav scripts/testv/ - cp "$LTV_DIR"/*.met scripts/testv/ - - cp "$LTV_DIR"/*.csv scripts/testv/ + - cp "$LTV_DIR"/*.csv scripts/testv/ .rules-pytest-mld: rules: @@ -189,7 +192,7 @@ stages: - fi - *remove-unsupported-testcases - if [ $LEVEL_SCALING != "1.0" ];then - - *apply-testv-scaling + - *apply-testv-scaling - fi - *mld-test-setup-codec @@ -203,7 +206,7 @@ stages: - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto --testcase_timeout $testcase_timeout || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - + - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -329,7 +332,7 @@ ivas-pytest-dec-msan: - *print-common-info - *update-scripts-repo - *remove-unsupported-testcases - - *mld-test-setup-codec + - *setup-codec - make clean - make -j CLANG=1 - testcase_timeout=900 -- GitLab From 824b379ec5ac4b90aa4a0154ec10fca7ba38d251 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 14:21:02 +0100 Subject: [PATCH 04/10] introduce anchor for sanitizers --- .gitlab-ci.yml | 61 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e85063a65..b823ab694 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -228,6 +228,38 @@ stages: reports: junit: - report-junit.xml + + +.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor: + stage: test + needs: ["build-codec-linux-make"] + timeout: "90 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "msan-stv" + script: + - *print-common-info + - *update-scripts-repo + - *remove-unsupported-testcases + - *setup-codec + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=900 + - python3 -m pytest $SHORT_TEST_SUITE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec + artifacts: + name: "ivas-pytest-dec-msan--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: "2 weeks" + paths: + - report-junit.xml + - report.html + reports: + junit: + - report-junit.xml # --------------------------------------------------------------- # Build jobs @@ -321,32 +353,9 @@ ivas-pytest-mld-dec-lev+10: ivas-pytest-dec-msan: extends: - .test-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "msan-stv" - script: - - *print-common-info - - *update-scripts-repo - - *remove-unsupported-testcases - - *setup-codec - - make clean - - make -j CLANG=1 - - testcase_timeout=900 - - python3 -m pytest $SHORT_TEST_SUITE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec - artifacts: - name: "ivas-pytest-dec-msan--sha-$CI_COMMIT_SHORT_SHA--results" - when: always - expire_in: "2 weeks" - paths: - - report-junit.xml - - report.html - reports: - junit: - - report-junit.xml + before_script: + - CLANG_NUM=1 + <<: *ivas-pytest-sanitizers-anchor # --------------------------------------------------------------- # Long test jobs -- GitLab From 9ea1e08cec60118dc61946b11789a795e52e7d9c Mon Sep 17 00:00:00 2001 From: kiene Date: Tue, 20 Feb 2024 13:22:17 +0000 Subject: [PATCH 05/10] fix typo in ci file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b823ab694..7354fbb9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -230,7 +230,7 @@ stages: - report-junit.xml -.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor: +.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor stage: test needs: ["build-codec-linux-make"] timeout: "90 minutes" -- GitLab From 141b5c431dafc41b073febe7102509885ad0ea38 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 15:21:43 +0100 Subject: [PATCH 06/10] add asan job --- .gitlab-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7354fbb9d..5bb77e400 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,6 +22,7 @@ variables: - 'pytest-mld-long' - 'evs-26444' - 'msan-stv' + - 'asan-stv' default: @@ -51,7 +52,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' variables: IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'msan-stv' + - if: $CI_PIPELINE_SOURCE == 'web' && ( $MANUAL_PIPELINE_TYPE == 'msan-stv' || $MANUAL_PIPELINE_TYPE == 'asan-stv' ) variables: IVAS_PIPELINE_NAME: 'Short testvecor MSAN' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch @@ -240,7 +241,7 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "msan-stv" + - if: $CI_PIPELINE_SOURCE == 'web' && ( $MANUAL_PIPELINE_TYPE == "msan-stv" || $MANUAL_PIPELINE_TYPE == 'asan-stv' ) script: - *print-common-info - *update-scripts-repo @@ -357,6 +358,13 @@ ivas-pytest-dec-msan: - CLANG_NUM=1 <<: *ivas-pytest-sanitizers-anchor +ivas-pytest-dec-asan: + extends: + - .test-job-linux + before_script: + - CLANG_NUM=2 + <<: *ivas-pytest-sanitizers-anchor + # --------------------------------------------------------------- # Long test jobs # --------------------------------------------------------------- -- GitLab From e07adc61004d1409c1b4c490a75bb7ea1d485825 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 15:25:26 +0100 Subject: [PATCH 07/10] one value for both sanitizers --- .gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bb77e400..13767865b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,8 +21,7 @@ variables: - 'pytest-mld' - 'pytest-mld-long' - 'evs-26444' - - 'msan-stv' - - 'asan-stv' + - 'sanitizer-stv' default: @@ -52,7 +51,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' variables: IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && ( $MANUAL_PIPELINE_TYPE == 'msan-stv' || $MANUAL_PIPELINE_TYPE == 'asan-stv' ) + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer-stv' variables: IVAS_PIPELINE_NAME: 'Short testvecor MSAN' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch @@ -241,7 +240,7 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - - if: $CI_PIPELINE_SOURCE == 'web' && ( $MANUAL_PIPELINE_TYPE == "msan-stv" || $MANUAL_PIPELINE_TYPE == 'asan-stv' ) + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer-stv" script: - *print-common-info - *update-scripts-repo @@ -249,7 +248,7 @@ stages: - *setup-codec - make clean - make -j CLANG=$CLANG_NUM - - testcase_timeout=900 + - testcase_timeout=300 - python3 -m pytest $SHORT_TEST_SUITE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec artifacts: name: "ivas-pytest-dec-msan--sha-$CI_COMMIT_SHORT_SHA--results" -- GitLab From 782fc240c3185108ed38ba2c47a421353deaa5fc Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 15:34:17 +0100 Subject: [PATCH 08/10] set no traceback as too long for console log --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13767865b..453f68bdf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -249,7 +249,7 @@ stages: - make clean - make -j CLANG=$CLANG_NUM - testcase_timeout=300 - - python3 -m pytest $SHORT_TEST_SUITE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec + - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec artifacts: name: "ivas-pytest-dec-msan--sha-$CI_COMMIT_SHORT_SHA--results" when: always -- GitLab From 0f8b34c92db965030bcac468cf4bcc9dbc252cc0 Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 15:58:12 +0100 Subject: [PATCH 09/10] add undef behaviour sanitizer job --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 453f68bdf..4ca671e7d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -248,6 +248,7 @@ stages: - *setup-codec - make clean - make -j CLANG=$CLANG_NUM + - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi - testcase_timeout=300 - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec artifacts: @@ -364,6 +365,13 @@ ivas-pytest-dec-asan: - CLANG_NUM=2 <<: *ivas-pytest-sanitizers-anchor +ivas-pytest-dec-usan: + extends: + - .test-job-linux + before_script: + - CLANG_NUM=3 + <<: *ivas-pytest-sanitizers-anchor + # --------------------------------------------------------------- # Long test jobs # --------------------------------------------------------------- -- GitLab From 3b5e2cbe19a9715099dab4b5534d0f4dae4f572b Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 20 Feb 2024 15:58:59 +0100 Subject: [PATCH 10/10] change pipeline name for sanitizer manual pipeline --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ca671e7d..6e483173e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,7 +53,7 @@ workflow: IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer-stv' variables: - IVAS_PIPELINE_NAME: 'Short testvecor MSAN' + IVAS_PIPELINE_NAME: 'Short testvectors sanitizers' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' -- GitLab