From 9d9a917b015f9320d0cbccea60c8b24c4803459f Mon Sep 17 00:00:00 2001 From: knj Date: Tue, 30 Jan 2024 15:21:50 +0100 Subject: [PATCH 1/9] add interop test with fxp encoder and flt decoder --- .gitlab-ci.yml | 56 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0bcd5d7be..98cbce811 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,9 +73,8 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" -.mld-test-setup-codec: &mld-test-setup-codec +.setup-codec: &setup-codec - current_commit_sha=$(git rev-parse HEAD) - ### build reference binaries - git checkout $REFERENCE_TAG - make clean @@ -88,6 +87,10 @@ stages: - make clean - make -j + +.mld-test-setup-codec: &mld-test-setup-codec + - *setup-codec + ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py @@ -108,6 +111,12 @@ stages: - cp -r $SCRIPTS_DIR/tests . - cp $SCRIPTS_DIR/pytest.ini . + +.pytest-check-results: &pytest-check-results + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + + # TODO: this needs to be updated in case the reference is updated .remove-unsupported-testcases: &remove-unsupported-testcases - sed -i '1649,1655d' scripts/config/self_test.prm @@ -125,7 +134,7 @@ stages: .copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir - cp "$LTV_DIR"/*.wav scripts/testv/ - cp "$LTV_DIR"/*.met scripts/testv/ - - cp "$LTV_DIR"/*.csv scripts/testv/ + - cp "$LTV_DIR"/*.csv scripts/testv/ .rules-pytest-mld: rules: @@ -170,6 +179,27 @@ stages: - cp -r scripts/testv/* $TESTV_DIR/ +.ivas-pytest-interop-anchor: &ivas-pytest-interop-anchor + - *print-common-info + - *update-scripts-repo + - *remove-unsupported-testcases + + ### run pytest + # create short test vectors + - python3 tests/create_short_testvectors.py + - exit_code1=0 + - exit_code2=0 + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_1.html --self-contained-html --junit-xml=report-junit_1.xml || exit_code1=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_2.html --self-contained-html --junit-xml=report-junit_2.xml || exit_code2=$? + + - zero_errors=$(cat report-junit_1.xml report-junit2.xml | grep -c 'errors="0"') || true + + - exit_code=0 + - if [ $exit_code1 -ne 0 || $exit_code2 -ne 0 ]; then exit_code=1; fi + - *pytest-check-results + - exit 0 + + .ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor stage: test needs: ["build-codec-linux-make"] @@ -183,7 +213,7 @@ stages: - fi - *remove-unsupported-testcases - if [ $LEVEL_SCALING != "1.0" ];then - - *apply-testv-scaling + - *apply-testv-scaling - fi - *mld-test-setup-codec @@ -195,11 +225,10 @@ stages: - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - + - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - *pytest-check-results - exit 0 allow_failure: @@ -307,6 +336,19 @@ ivas-pytest-mld-dec-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor +ivas-pytest-interop-enc-fixed-dec-float: + extends: + - .rules-pytest-mld + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *setup-codec + - ENCODER=./IVAS_cod_ref + - DECODER=./IVAS_dec_ref + - *ivas-pytest-interop-anchor + # --------------------------------------------------------------- # Long test jobs # --------------------------------------------------------------- -- GitLab From 825c6beeaabee790169d9bc90396da7d8c6da525 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 5 Feb 2024 11:08:12 +0100 Subject: [PATCH 2/9] set TEST_SUITE variable in new test job --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b6ce6d10e..117eb4415 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -347,6 +347,7 @@ ivas-pytest-interop-enc-fixed-dec-float: - *setup-codec - ENCODER=./IVAS_cod_ref - DECODER=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE" - *ivas-pytest-interop-anchor # --------------------------------------------------------------- -- GitLab From e54668ce9bf486c0fb2aa6d873bc04959c55849c Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 5 Feb 2024 11:27:57 +0100 Subject: [PATCH 3/9] fix typo --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 117eb4415..0da864afc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -192,8 +192,7 @@ stages: - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_1.html --self-contained-html --junit-xml=report-junit_1.xml || exit_code1=$? - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_2.html --self-contained-html --junit-xml=report-junit_2.xml || exit_code2=$? - - zero_errors=$(cat report-junit_1.xml report-junit2.xml | grep -c 'errors="0"') || true - + - zero_errors=$(cat report-junit_1.xml report-junit_2.xml | grep -c 'errors="0"') || true - exit_code=0 - if [ $exit_code1 -ne 0 || $exit_code2 -ne 0 ]; then exit_code=1; fi - *pytest-check-results -- GitLab From cb1dcda81b238bcd2938d6132d8b0469643f3b82 Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 5 Feb 2024 14:47:47 +0100 Subject: [PATCH 4/9] fix comparison code --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0da864afc..119dc8b43 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -192,9 +192,12 @@ stages: - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_1.html --self-contained-html --junit-xml=report-junit_1.xml || exit_code1=$? - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_2.html --self-contained-html --junit-xml=report-junit_2.xml || exit_code2=$? - - zero_errors=$(cat report-junit_1.xml report-junit_2.xml | grep -c 'errors="0"') || true + - zero_errors_1=$(cat report-junit_1.xml | grep -c 'errors="0"') || true + - zero_errors_2=$(cat report-junit_2.xml | grep -c 'errors="0"') || true + - zero_errors=1 + - if [[ $zero_errors_1 != 1 || $zero_errors_2 != 1 ]]; then zero_errors=0; fi - exit_code=0 - - if [ $exit_code1 -ne 0 || $exit_code2 -ne 0 ]; then exit_code=1; fi + - if [[ $exit_code1 != 0 || $exit_code2 != 0 ]]; then exit_code=1; fi - *pytest-check-results - exit 0 -- GitLab From 0ffa71a2d69ef3a20cfaf51dd1bbcf651ce99e60 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 7 Feb 2024 09:03:44 +0100 Subject: [PATCH 5/9] use correct encoder binary --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 119dc8b43..d949936a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -347,7 +347,7 @@ ivas-pytest-interop-enc-fixed-dec-float: timeout: "30 minutes" script: - *setup-codec - - ENCODER=./IVAS_cod_ref + - ENCODER=./IVAS_cod - DECODER=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - *ivas-pytest-interop-anchor -- GitLab From 57d3fab019a8d3cb46ff24ff7848f9208edcf6c0 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 15 Mar 2024 11:50:06 +0100 Subject: [PATCH 6/9] implement fx enc -> flt dec interop test --- .gitlab-ci.yml | 74 ++++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a5820e29..27412226c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,7 @@ variables: options: - 'default' - 'pytest-mld' + - 'pytest-mld-interop' - 'pytest-mld-long' - 'evs-26444' - 'sanitizer-stv' @@ -158,6 +159,14 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never +.rules-pytest-mld-interop: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-interop" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -181,30 +190,6 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ - -.ivas-pytest-interop-anchor: &ivas-pytest-interop-anchor - - *print-common-info - - *update-scripts-repo - - python3 ci/remove_unsupported_testcases.py $PRM_FILES - - ### run pytest - # create short test vectors - - python3 tests/create_short_testvectors.py - - exit_code1=0 - - exit_code2=0 - - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_1.html --self-contained-html --junit-xml=report-junit_1.xml || exit_code1=$? - - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $ENCODER --ref_decoder_path $DECODER --html=report_2.html --self-contained-html --junit-xml=report-junit_2.xml || exit_code2=$? - - - zero_errors_1=$(cat report-junit_1.xml | grep -c 'errors="0"') || true - - zero_errors_2=$(cat report-junit_2.xml | grep -c 'errors="0"') || true - - zero_errors=1 - - if [[ $zero_errors_1 != 1 || $zero_errors_2 != 1 ]]; then zero_errors=0; fi - - exit_code=0 - - if [[ $exit_code1 != 0 || $exit_code2 != 0 ]]; then exit_code=1; fi - - *pytest-check-results - - exit 0 - - .ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor stage: test needs: ["build-codec-linux-make"] @@ -385,19 +370,38 @@ ivas-pytest-mld-dec-lev+10: - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor -ivas-pytest-interop-enc-fixed-dec-float: +ivas-pytest-mld-long-enc-fx-dec-flt-interop: extends: - - .rules-pytest-mld + - .rules-pytest-mld-interop - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *setup-codec - - ENCODER=./IVAS_cod - - DECODER=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - *ivas-pytest-interop-anchor + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-fx-dec-flt-lev-10-interop: + extends: + - .rules-pytest-mld-interop + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-fx-dec-flt-lev+10-interop: + extends: + - .rules-pytest-mld-interop + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor ivas-pytest-dec-msan: extends: -- GitLab From ab9e92b40b2162139ac97e3c5adc47615bb5ad54 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 15 Mar 2024 11:52:50 +0100 Subject: [PATCH 7/9] add workflow section to allow pipeline to run --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27412226c..0d88f6dbc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-interop' + variables: + IVAS_PIPELINE_NAME: 'Run MLD tool agains float ref - interop: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' -- GitLab From 67fdcd4459325c188bf211f3d7cc8b62fad92456 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 15 Mar 2024 13:04:14 +0100 Subject: [PATCH 8/9] use correct test suite --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d88f6dbc..a48749351 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -380,7 +380,7 @@ ivas-pytest-mld-long-enc-fx-dec-flt-interop: before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$LONG_TEST_SUITE" + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -391,7 +391,7 @@ ivas-pytest-mld-long-enc-fx-dec-flt-lev-10-interop: before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$LONG_TEST_SUITE" + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -402,7 +402,7 @@ ivas-pytest-mld-long-enc-fx-dec-flt-lev+10-interop: before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$LONG_TEST_SUITE" + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor -- GitLab From 472431cac4274e98f6a2be13e6edd2427b679bb7 Mon Sep 17 00:00:00 2001 From: knj Date: Fri, 15 Mar 2024 13:17:44 +0100 Subject: [PATCH 9/9] cleanup --- .gitlab-ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a48749351..23715d6a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -125,10 +125,6 @@ stages: - cp -r $SCRIPTS_DIR/tests . - cp $SCRIPTS_DIR/pytest.ini . -.pytest-check-results: &pytest-check-results - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - .apply-testv-scaling: &apply-testv-scaling - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING @@ -219,7 +215,8 @@ stages: - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv - - *pytest-check-results + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: -- GitLab