From de8b742c5f115afab96311e9b31ca1ee141936ec Mon Sep 17 00:00:00 2001 From: Sandesh Venkatesh Date: Tue, 2 Jul 2024 14:16:54 +0530 Subject: [PATCH 1/4] Re-enable encoder pytests in CI --- .gitlab-ci.yml | 54 +++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fb5645e8c..5c1f73e4c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -410,15 +410,15 @@ build-codec-linux-instrumented-make: # Short test jobs # --------------------------------------------------------------- -#ivas-pytest-mld-enc-dec: -# extends: -# - .rules-pytest-mld -# - .test-job-linux -# before_script: -# - USE_LTV=0 -# - TEST_SUITE="$SHORT_TEST_SUITE" -# - LEVEL_SCALING=1.0 -# <<: *ivas-pytest-anchor +ivas-pytest-mld-enc-dec: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-anchor ivas-pytest-mld-dec: extends: @@ -431,15 +431,15 @@ ivas-pytest-mld-dec: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -#ivas-pytest-mld-enc-dec-lev-10: -# extends: -# - .rules-pytest-mld -# - .test-job-linux -# before_script: -# - USE_LTV=0 -# - TEST_SUITE="$SHORT_TEST_SUITE" -# - LEVEL_SCALING=0.3162 -# <<: *ivas-pytest-anchor +ivas-pytest-mld-enc-dec-lev-10: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-anchor ivas-pytest-mld-dec-lev-10: extends: @@ -452,15 +452,15 @@ ivas-pytest-mld-dec-lev-10: - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor -#ivas-pytest-mld-enc-dec-lev+10: -# extends: -# - .rules-pytest-mld -# - .test-job-linux -# before_script: -# - USE_LTV=0 -# - TEST_SUITE="$SHORT_TEST_SUITE" -# - LEVEL_SCALING=3.162 -# <<: *ivas-pytest-anchor +ivas-pytest-mld-enc-dec-lev+10: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-anchor ivas-pytest-mld-dec-lev+10: extends: -- GitLab From 65452ddff95b32615bd8a1e5345423c75d34da19 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 2 Jul 2024 15:08:54 +0200 Subject: [PATCH 2/4] refactor ci file --- .gitlab-ci.yml | 114 ++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 63 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c1f73e4c..fc5c7edfd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,8 @@ variables: TEST_SUITE: "" DUT_ENCODER_PATH: "./IVAS_cod" DUT_DECODER_PATH: "./IVAS_dec" + REF_ENCODER_PATH: "./IVAS_cod_ref" + REF_DECODER_PATH: "./IVAS_dec_ref" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" @@ -24,7 +26,6 @@ variables: options: - 'default' - 'pytest-mld' - - 'pytest-mld-interop' - 'pytest-mld-long' - 'pytest-saturation-smoke-test' - 'evs-26444' @@ -53,9 +54,6 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-interop' - variables: - IVAS_PIPELINE_NAME: 'Run MLD tool agains float ref - interop: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' @@ -172,15 +170,6 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never - -.rules-pytest-mld-interop: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-interop" - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - .rules-pytest-saturation-smoke-test: rules: - if: $PYTEST_SMOKE_TEST # Set by scheduled pipeline @@ -240,7 +229,7 @@ stages: ### run pytest - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld_xml.py report-junit.xml $MLD_ARTIFACT_NAME @@ -302,7 +291,7 @@ stages: - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi - testcase_timeout=$TESTCASE_TIMEOUT_STV - - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec + - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -410,56 +399,85 @@ build-codec-linux-instrumented-make: # Short test jobs # --------------------------------------------------------------- -ivas-pytest-mld-enc-dec: +### jobs that test fx encoder -> flt decoder +ivas-pytest-mld-enc: extends: - .rules-pytest-mld - .test-job-linux before_script: - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor - -ivas-pytest-mld-dec: + +ivas-pytest-mld-enc-lev-10: extends: - .rules-pytest-mld - .test-job-linux before_script: - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref + - DUT_DECODER_PATH=./IVAS_dec - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=1.0 + - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor - -ivas-pytest-mld-enc-dec-lev-10: + +ivas-pytest-mld-enc-lev+10: extends: - .rules-pytest-mld - .test-job-linux before_script: - - USE_LTV=0 + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 + - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor + +ivas-pytest-enc-msan: + extends: + - .test-job-linux + before_script: + - CLANG_NUM=1 + - REF_ENCODER_PATH=./IVAS_cod + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-enc-asan: + extends: + - .test-job-linux + before_script: + - CLANG_NUM=2 + - REF_ENCODER_PATH=./IVAS_cod + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-enc-usan: + extends: + - .test-job-linux + before_script: + - CLANG_NUM=3 + - REF_ENCODER_PATH=./IVAS_cod + <<: *ivas-pytest-sanitizers-anchor -ivas-pytest-mld-dec-lev-10: +### jobs that test flt encoder -> fx decoder +ivas-pytest-mld-dec: extends: - .rules-pytest-mld - .test-job-linux before_script: - - USE_LTV=0 + - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 + - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor - -ivas-pytest-mld-enc-dec-lev+10: + +ivas-pytest-mld-dec-lev-10: extends: - .rules-pytest-mld - .test-job-linux before_script: - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=3.162 + - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor ivas-pytest-mld-dec-lev+10: @@ -473,44 +491,12 @@ ivas-pytest-mld-dec-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor -ivas-pytest-mld-long-enc-fx-dec-flt-interop: - extends: - - .rules-pytest-mld-interop - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=1.0 - <<: *ivas-pytest-anchor - -ivas-pytest-mld-long-enc-fx-dec-flt-lev-10-interop: - extends: - - .rules-pytest-mld-interop - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 - <<: *ivas-pytest-anchor - -ivas-pytest-mld-long-enc-fx-dec-flt-lev+10-interop: - extends: - - .rules-pytest-mld-interop - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=3.162 - <<: *ivas-pytest-anchor - ivas-pytest-dec-msan: extends: - .test-job-linux before_script: - CLANG_NUM=1 + - REF_DECODER_PATH=./IVAS_dec <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: @@ -518,6 +504,7 @@ ivas-pytest-dec-asan: - .test-job-linux before_script: - CLANG_NUM=2 + - REF_DECODER_PATH=./IVAS_dec <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: @@ -525,6 +512,7 @@ ivas-pytest-dec-usan: - .test-job-linux before_script: - CLANG_NUM=3 + - REF_DECODER_PATH=./IVAS_dec <<: *ivas-pytest-sanitizers-anchor # --------------------------------------------------------------- -- GitLab From 26a3ad8fba60cb612ffec78a3aca4538d84b3258 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 2 Jul 2024 15:50:10 +0200 Subject: [PATCH 3/4] fix setting of decoder path in fxencoder test jobs --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc5c7edfd..809ace31e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -406,7 +406,7 @@ ivas-pytest-mld-enc: - .test-job-linux before_script: - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec + - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor @@ -417,7 +417,7 @@ ivas-pytest-mld-enc-lev-10: - .test-job-linux before_script: - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec + - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor @@ -428,7 +428,7 @@ ivas-pytest-mld-enc-lev+10: - .test-job-linux before_script: - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec + - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-anchor -- GitLab From e4d1ddd76fb8face6f728472c93ae6b374aaa9a7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 2 Jul 2024 15:53:58 +0200 Subject: [PATCH 4/4] use the DUT* vars in the sanitizer anchor, too this is now consistent with the other anchor --- .gitlab-ci.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 809ace31e..598173231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,8 +12,6 @@ variables: TEST_SUITE: "" DUT_ENCODER_PATH: "./IVAS_cod" DUT_DECODER_PATH: "./IVAS_dec" - REF_ENCODER_PATH: "./IVAS_cod_ref" - REF_DECODER_PATH: "./IVAS_dec_ref" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" @@ -291,7 +289,7 @@ stages: - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi - testcase_timeout=$TESTCASE_TIMEOUT_STV - - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH + - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -438,7 +436,7 @@ ivas-pytest-enc-msan: - .test-job-linux before_script: - CLANG_NUM=1 - - REF_ENCODER_PATH=./IVAS_cod + - DUT_DECODER_PATH=./IVAS_dec_ref <<: *ivas-pytest-sanitizers-anchor ivas-pytest-enc-asan: @@ -446,7 +444,7 @@ ivas-pytest-enc-asan: - .test-job-linux before_script: - CLANG_NUM=2 - - REF_ENCODER_PATH=./IVAS_cod + - DUT_DECODER_PATH=./IVAS_dec_ref <<: *ivas-pytest-sanitizers-anchor ivas-pytest-enc-usan: @@ -454,7 +452,7 @@ ivas-pytest-enc-usan: - .test-job-linux before_script: - CLANG_NUM=3 - - REF_ENCODER_PATH=./IVAS_cod + - DUT_DECODER_PATH=./IVAS_dec_ref <<: *ivas-pytest-sanitizers-anchor ### jobs that test flt encoder -> fx decoder @@ -496,7 +494,7 @@ ivas-pytest-dec-msan: - .test-job-linux before_script: - CLANG_NUM=1 - - REF_DECODER_PATH=./IVAS_dec + - DUT_ENCODER_PATH=./IVAS_cod_ref <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: @@ -504,7 +502,7 @@ ivas-pytest-dec-asan: - .test-job-linux before_script: - CLANG_NUM=2 - - REF_DECODER_PATH=./IVAS_dec + - DUT_ENCODER_PATH=./IVAS_cod_ref <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: @@ -512,7 +510,7 @@ ivas-pytest-dec-usan: - .test-job-linux before_script: - CLANG_NUM=3 - - REF_DECODER_PATH=./IVAS_dec + - DUT_ENCODER_PATH=./IVAS_cod_ref <<: *ivas-pytest-sanitizers-anchor # --------------------------------------------------------------- -- GitLab