From f05a508ac40c1fc3698d1bde06b6084ed5cb8b12 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 09:43:06 +0100 Subject: [PATCH 01/47] Added ivas-pytest-mld-dec-lev-10 to run MLD test with -10 dB input scaling --- .gitlab-ci.yml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bc603618..9eae1469a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" + LEVEL_SCALING: 1.0 IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger @@ -69,6 +70,10 @@ stages: .remove-unsupported-testcases: &remove-unsupported-testcases - sed -i '1629,1635d' scripts/config/self_test.prm +.apply-testv-scaling: &apply-testv-scaling + - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" + - for file in `ls ./scripts/testv/*.wav`; do tests/scale_pcm.py $file tmp.wav $LEVEL_SCALING; mv tmp.wav $file;done + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -186,3 +191,45 @@ ivas-pytest-mld-dec: reports: junit: - report-junit.xml + + +ivas-pytest-mld-dec-lev-10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *remove-unsupported-testcases + - *mld-test-setup-codec + - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) + - *apply-testv-scaling + + ### run pytest + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml -- GitLab From 979a2957f64f7d295cbbe7ec0a7ac65a85b8c053 Mon Sep 17 00:00:00 2001 From: norvell Date: Thu, 21 Dec 2023 08:45:56 +0000 Subject: [PATCH 02/47] Fix LEVEL_SCALING init (use string) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9eae1469a..5ee6b26dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" - LEVEL_SCALING: 1.0 + LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger -- GitLab From 4d581683c974e35d67796e9127d509d976239a99 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 11:18:10 +0100 Subject: [PATCH 03/47] Fix order of level scaling. Add anchors for LTV --- .gitlab-ci.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9eae1469a..864ca9c03 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,11 +1,12 @@ variables: + LTV_DIR: "/usr/local/ltv" REFERENCE_TAG: "20231128_Update_Ittiam" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" - LEVEL_SCALING: 1.0 + LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger @@ -74,6 +75,16 @@ stages: - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - for file in `ls ./scripts/testv/*.wav`; do tests/scale_pcm.py $file tmp.wav $LEVEL_SCALING; mv tmp.wav $file;done +.update-ltv-repo: &update-ltv-repo + - cd $LTV_DIR + - git pull + - cd - + +.copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir + - cp "$LTV_DIR"/*.wav scripts/testv/ + - cp "$LTV_DIR"/*.met scripts/testv/ + - cp "$LTV_DIR"/*.csv scripts/testv/ + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -203,9 +214,9 @@ ivas-pytest-mld-dec-lev-10: - *print-common-info - *update-scripts-repo - *remove-unsupported-testcases - - *mld-test-setup-codec - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - - *apply-testv-scaling + - *apply-testv-scaling + - *mld-test-setup-codec ### run pytest - exit_code=0 -- GitLab From f02ea709a5d382047aa29ac2b906503588adce30 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 13:59:13 +0100 Subject: [PATCH 04/47] Added long test vectors and renderer tests --- .gitlab-ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 864ca9c03..ad77149f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" + TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer tests/" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: @@ -56,8 +56,8 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -133,12 +133,14 @@ ivas-pytest-mld-enc-dec: script: - *print-common-info - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir - *remove-unsupported-testcases - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv @@ -173,12 +175,14 @@ ivas-pytest-mld-dec: script: - *print-common-info - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir - *remove-unsupported-testcases - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv @@ -213,6 +217,8 @@ ivas-pytest-mld-dec-lev-10: script: - *print-common-info - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir - *remove-unsupported-testcases - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - *apply-testv-scaling @@ -220,7 +226,7 @@ ivas-pytest-mld-dec-lev-10: ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv -- GitLab From 9cc7b910aa5024f88907f56967cb1200e019be38 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 14:27:35 +0100 Subject: [PATCH 05/47] Revert extension of test suite --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad77149f3..185367bfe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer tests/" + TEST_SUITE: "tests/codec_be_on_mr_nonselection" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: -- GitLab From 55119d7a55f521d38a6c96dd6803df12df272eca Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 14:37:28 +0100 Subject: [PATCH 06/47] Add back extension of tests, now without typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 185367bfe..c5b78b9b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TEST_SUITE: "tests/codec_be_on_mr_nonselection" + TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: -- GitLab From c4bbd8c355348a4b2fa344477e9a6f11cb54cdfe Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 14:46:41 +0100 Subject: [PATCH 07/47] Added remaining level scaling jobs --- .gitlab-ci.yml | 130 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ae2ab9c6..bf5dda211 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -288,3 +288,133 @@ ivas-pytest-mld-dec-lev-10: reports: junit: - report-junit.xml + +ivas-pytest-mld-dec-lev+10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *remove-unsupported-testcases + - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) + - *apply-testv-scaling + - *mld-test-setup-codec + + ### run pytest + - exit_code=0 + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml + +ivas-pytest-mld-enc-dec-lev-10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *remove-unsupported-testcases + - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) + - *apply-testv-scaling + - *mld-test-setup-codec + + ### run pytest + - exit_code=0 + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml + +ivas-pytest-mld-enc-dec-lev+10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *remove-unsupported-testcases + - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) + - *apply-testv-scaling + - *mld-test-setup-codec + + ### run pytest + - exit_code=0 + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml + -- GitLab From 6f0a3a4d4afa3f82ac2c5dbcb199ef936dfe48e3 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 15:59:52 +0100 Subject: [PATCH 08/47] Trying to simplify jobs with anchor --- .gitlab-ci.yml | 275 +++++++------------------------------------------ 1 file changed, 39 insertions(+), 236 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf5dda211..9d7a9d8fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,10 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" + LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + USE_REF_ENC: "1" + TEST_SUITE: "" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: @@ -138,29 +141,8 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ -# --------------------------------------------------------------- -# Build jobs -# --------------------------------------------------------------- - -# ensure that codec builds on linux -build-codec-linux-make: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - extends: - - .build-job-linux - script: - - *print-common-info - - make -j - -# --------------------------------------------------------------- -# Test jobs -# --------------------------------------------------------------- -ivas-pytest-mld-enc-dec: +.ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor extends: - .test-job-linux - .rules-pytest-mld @@ -170,14 +152,23 @@ ivas-pytest-mld-enc-dec: script: - *print-common-info - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - fi - *remove-unsupported-testcases + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - if [ $USE_REF_ENC -eq 1 ]; then + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - else + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv @@ -202,219 +193,31 @@ ivas-pytest-mld-enc-dec: junit: - report-junit.xml +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- -ivas-pytest-mld-dec: - extends: - - .test-job-linux - - .rules-pytest-mld - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - - -ivas-pytest-mld-dec-lev-10: - extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - -ivas-pytest-mld-dec-lev+10: - extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - -ivas-pytest-mld-enc-dec-lev-10: - extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - -ivas-pytest-mld-enc-dec-lev+10: +# ensure that codec builds on linux +build-codec-linux-make: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'push' + when: never extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" + - .build-job-linux script: - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 + - make -j - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml +# --------------------------------------------------------------- +# Test jobs +# --------------------------------------------------------------- +ivas-pytest-mld-enc-dec: + before_script: + - USE_REF_ENC=0 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=1.0 + *ivas-pytest-mld-anchor \ No newline at end of file -- GitLab From f9342451b624515c20c5aca0e9d2fc65fc2daba8 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 16:02:39 +0100 Subject: [PATCH 09/47] Fix syntax in .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9d7a9d8fc..9e7f6973d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -220,4 +220,4 @@ ivas-pytest-mld-enc-dec: - USE_REF_ENC=0 - TEST_SUITE=$SHORT_TEST_SUITE - LEVEL_SCALING=1.0 - *ivas-pytest-mld-anchor \ No newline at end of file + <<: *ivas-pytest-mld-anchor \ No newline at end of file -- GitLab From 78219d992e30b754258256f0dedd5372ad7a82a6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 16:08:58 +0100 Subject: [PATCH 10/47] Expand test set using anchor -- to be verified --- .gitlab-ci.yml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9e7f6973d..d02b20e64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -220,4 +220,39 @@ ivas-pytest-mld-enc-dec: - USE_REF_ENC=0 - TEST_SUITE=$SHORT_TEST_SUITE - LEVEL_SCALING=1.0 - <<: *ivas-pytest-mld-anchor \ No newline at end of file + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec: + before_script: + - USE_REF_ENC=1 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-enc-dec-lev-10: + before_script: + - USE_REF_ENC=0 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec-lev-10: + before_script: + - USE_REF_ENC=1 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-enc-dec-lev+10: + before_script: + - USE_REF_ENC=0 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec-lev+10: + before_script: + - USE_REF_ENC=1 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor -- GitLab From e1a6b25811b27f4ebe03378699629b37cf80e366 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:38:39 +0100 Subject: [PATCH 11/47] Added short test set for testing, new scale_pcm.py --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d02b20e64..1400756b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,8 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::\"test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]\"" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" @@ -97,7 +98,7 @@ stages: .apply-testv-scaling: &apply-testv-scaling - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - - for file in `ls ./scripts/testv/*.wav`; do tests/scale_pcm.py $file tmp.wav $LEVEL_SCALING; mv tmp.wav $file;done + - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING .update-ltv-repo: &update-ltv-repo - cd $LTV_DIR -- GitLab From 911972881250ef80c57834bec5efd596fe8b979e Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:42:27 +0100 Subject: [PATCH 12/47] Fix test suite --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1400756b5..a6d189ce2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: EXIT_CODE_FAIL: 1 LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::\"test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]\"" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" -- GitLab From 11bf04b0eeb4dbb14f59c7efc14b6ef8153495a6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:46:06 +0100 Subject: [PATCH 13/47] Fix quotation marks, hopefully --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6d189ce2..354ee8c66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -166,9 +166,9 @@ stages: ### run pytest - exit_code=0 - if [ $USE_REF_ENC -eq 1 ]; then - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - else - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true -- GitLab From 25f119703db4f847b5b0b47c5a37ff0ac0bc5392 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:49:42 +0100 Subject: [PATCH 14/47] Fix quotation marks again --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 354ee8c66..0b009b7b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,8 +79,8 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -219,41 +219,41 @@ build-codec-linux-make: ivas-pytest-mld-enc-dec: before_script: - USE_REF_ENC=0 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec: before_script: - USE_REF_ENC=1 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-enc-dec-lev-10: before_script: - USE_REF_ENC=0 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec-lev-10: before_script: - USE_REF_ENC=1 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-enc-dec-lev+10: before_script: - USE_REF_ENC=0 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec-lev+10: before_script: - USE_REF_ENC=1 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor -- GitLab From 7f3107687a57247d1f2087b19028d9bdd903b5eb Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 19:03:42 +0100 Subject: [PATCH 15/47] Run with set that allows create_part_2 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b009b7b8..aa8d93176 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,8 +6,8 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" - #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" -- GitLab From 2841b821ccb7d16aad312775826e7af2f76b07a9 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 09:21:01 +0100 Subject: [PATCH 16/47] Added scheduled long test vector jobs --- .gitlab-ci.yml | 106 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa8d93176..4552613bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,9 +5,8 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" @@ -18,6 +17,7 @@ variables: options: - 'default' - 'pytest-mld' + - 'pytest-mld-long' default: @@ -41,6 +41,13 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' + variables: + IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch + variables: + IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + stages: - build @@ -79,8 +86,10 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto + # - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto + # - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto -k 13200 + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto -k 13200 .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -117,7 +126,17 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'push' when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never +.rules-pytest-mld-long: + rules: + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never # --------------------------------------------------------------- # Job templates @@ -146,7 +165,6 @@ stages: .ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor extends: - .test-job-linux - - .rules-pytest-mld stage: test needs: ["build-codec-linux-make"] timeout: "30 minutes" @@ -166,9 +184,11 @@ stages: ### run pytest - exit_code=0 - if [ $USE_REF_ENC -eq 1 ]; then - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? +# - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto -k 13200 || exit_code=$? - else - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? +# - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto -k 13200 || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true @@ -213,10 +233,12 @@ build-codec-linux-make: - make -j # --------------------------------------------------------------- -# Test jobs +# Short test jobs # --------------------------------------------------------------- ivas-pytest-mld-enc-dec: + extends: + - .rules-pytest-mld before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -224,6 +246,8 @@ ivas-pytest-mld-enc-dec: <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec: + extends: + - .rules-pytest-mld before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -231,6 +255,8 @@ ivas-pytest-mld-dec: <<: *ivas-pytest-mld-anchor ivas-pytest-mld-enc-dec-lev-10: + extends: + - .rules-pytest-mld before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -238,6 +264,8 @@ ivas-pytest-mld-enc-dec-lev-10: <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec-lev-10: + extends: + - .rules-pytest-mld before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -245,6 +273,8 @@ ivas-pytest-mld-dec-lev-10: <<: *ivas-pytest-mld-anchor ivas-pytest-mld-enc-dec-lev+10: + extends: + - .rules-pytest-mld before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -252,6 +282,66 @@ ivas-pytest-mld-enc-dec-lev+10: <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec-lev+10: + extends: + - .rules-pytest-mld + before_script: + - USE_REF_ENC=1 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +# --------------------------------------------------------------- +# Long test jobs +# --------------------------------------------------------------- + +ivas-pytest-mld-long-enc-dec: + extends: + - .rules-pytest-mld-long + before_script: + - USE_REF_ENC=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-dec: + extends: + - .rules-pytest-mld-long + before_script: + - USE_REF_ENC=1 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-dec-lev-10: + extends: + - .rules-pytest-mld-long + before_script: + - USE_REF_ENC=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-dec-lev-10: + extends: + - .rules-pytest-mld-long + before_script: + - USE_REF_ENC=1 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-dec-lev+10: + extends: + - .rules-pytest-mld-long + before_script: + - USE_REF_ENC=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-dec-lev+10: + extends: + - .rules-pytest-mld-long before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" -- GitLab From a9a943cf5bd4b2a4715b6689cc137b263c61c5df Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 09:24:34 +0100 Subject: [PATCH 17/47] Try correcting rules in rules-pytest-mld --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4552613bc..5cfe6de58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,7 +127,9 @@ stages: - if: $CI_PIPELINE_SOURCE == 'push' when: never - if: $CI_PIPELINE_SOURCE == 'schedule' - when: never + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + when: never .rules-pytest-mld-long: rules: -- GitLab From ec9c7ec25998f01ac6d17bb8d03c990005342eea Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 09:30:49 +0100 Subject: [PATCH 18/47] Try correcting rules --- .gitlab-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cfe6de58..c3fa08de9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,8 +128,6 @@ stages: when: never - if: $CI_PIPELINE_SOURCE == 'schedule' when: never - - if: $CI_PIPELINE_SOURCE == 'web' - when: never .rules-pytest-mld-long: rules: @@ -165,8 +163,6 @@ stages: .ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor - extends: - - .test-job-linux stage: test needs: ["build-codec-linux-make"] timeout: "30 minutes" @@ -241,6 +237,7 @@ build-codec-linux-make: ivas-pytest-mld-enc-dec: extends: - .rules-pytest-mld + - .test-job-linux before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -250,6 +247,7 @@ ivas-pytest-mld-enc-dec: ivas-pytest-mld-dec: extends: - .rules-pytest-mld + - .test-job-linux before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -259,6 +257,7 @@ ivas-pytest-mld-dec: ivas-pytest-mld-enc-dec-lev-10: extends: - .rules-pytest-mld + - .test-job-linux before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -268,6 +267,7 @@ ivas-pytest-mld-enc-dec-lev-10: ivas-pytest-mld-dec-lev-10: extends: - .rules-pytest-mld + - .test-job-linux before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -277,6 +277,7 @@ ivas-pytest-mld-dec-lev-10: ivas-pytest-mld-enc-dec-lev+10: extends: - .rules-pytest-mld + - .test-job-linux before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -286,6 +287,7 @@ ivas-pytest-mld-enc-dec-lev+10: ivas-pytest-mld-dec-lev+10: extends: - .rules-pytest-mld + - .test-job-linux before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -299,6 +301,7 @@ ivas-pytest-mld-dec-lev+10: ivas-pytest-mld-long-enc-dec: extends: - .rules-pytest-mld-long + - .test-job-linux before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -308,6 +311,7 @@ ivas-pytest-mld-long-enc-dec: ivas-pytest-mld-long-dec: extends: - .rules-pytest-mld-long + - .test-job-linux before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -317,6 +321,7 @@ ivas-pytest-mld-long-dec: ivas-pytest-mld-long-enc-dec-lev-10: extends: - .rules-pytest-mld-long + - .test-job-linux before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -326,6 +331,7 @@ ivas-pytest-mld-long-enc-dec-lev-10: ivas-pytest-mld-long-dec-lev-10: extends: - .rules-pytest-mld-long + - .test-job-linux before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -335,6 +341,7 @@ ivas-pytest-mld-long-dec-lev-10: ivas-pytest-mld-long-enc-dec-lev+10: extends: - .rules-pytest-mld-long + - .test-job-linux before_script: - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" @@ -344,6 +351,7 @@ ivas-pytest-mld-long-enc-dec-lev+10: ivas-pytest-mld-long-dec-lev+10: extends: - .rules-pytest-mld-long + - .test-job-linux before_script: - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" -- GitLab From 6f99d268a506bb394dbfb7677e2006642edccba6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 09:40:03 +0100 Subject: [PATCH 19/47] Fix for scheduled pipeline --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3fa08de9..13403a47b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -131,7 +131,7 @@ stages: .rules-pytest-mld-long: rules: - - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $PYTEST_MLD_LONG # Set by scheduled pipeline - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long" - if: $CI_PIPELINE_SOURCE == 'push' when: never -- GitLab From 0ea7784f2a9e3321ecad50f26b50b2fa2156fa74 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 10:01:45 +0100 Subject: [PATCH 20/47] Add dependent job build-codec-linux-make for scheduled jobs --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13403a47b..5b4b8d7e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -222,6 +222,7 @@ build-codec-linux-make: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: -- GitLab From ce47654a9ba07423b14a81ab2d360b36d3985e1a Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 10:35:26 +0100 Subject: [PATCH 21/47] Enable long test suite and remove debugging limitation with -k 13200 --- .gitlab-ci.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b4b8d7e1..7d4004363 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -86,10 +86,9 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - # - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto - # - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto -k 13200 - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto -k 13200 + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto + .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -182,11 +181,9 @@ stages: ### run pytest - exit_code=0 - if [ $USE_REF_ENC -eq 1 ]; then -# - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto -k 13200 || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - else -# - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto -k 13200 || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true @@ -305,7 +302,7 @@ ivas-pytest-mld-long-enc-dec: - .test-job-linux before_script: - USE_REF_ENC=0 - - TEST_SUITE="$SHORT_TEST_SUITE" + - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -315,7 +312,7 @@ ivas-pytest-mld-long-dec: - .test-job-linux before_script: - USE_REF_ENC=1 - - TEST_SUITE="$SHORT_TEST_SUITE" + - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -325,7 +322,7 @@ ivas-pytest-mld-long-enc-dec-lev-10: - .test-job-linux before_script: - USE_REF_ENC=0 - - TEST_SUITE="$SHORT_TEST_SUITE" + - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -335,7 +332,7 @@ ivas-pytest-mld-long-dec-lev-10: - .test-job-linux before_script: - USE_REF_ENC=1 - - TEST_SUITE="$SHORT_TEST_SUITE" + - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -345,7 +342,7 @@ ivas-pytest-mld-long-enc-dec-lev+10: - .test-job-linux before_script: - USE_REF_ENC=0 - - TEST_SUITE="$SHORT_TEST_SUITE" + - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor @@ -355,6 +352,6 @@ ivas-pytest-mld-long-dec-lev+10: - .test-job-linux before_script: - USE_REF_ENC=1 - - TEST_SUITE="$SHORT_TEST_SUITE" + - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor -- GitLab From 61af8bed85976d4773fcb4f7a2a3dd1ff3f51e3b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 13:18:30 +0100 Subject: [PATCH 22/47] Adding 26.444 test job --- .gitlab-ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d4004363..b83e0dac4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,6 @@ variables: LTV_DIR: "/usr/local/ltv" + EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" REFERENCE_TAG: "20231128_Update_Ittiam" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" @@ -18,6 +19,7 @@ variables: - 'default' - 'pytest-mld' - 'pytest-mld-long' + - 'evs-26444' default: @@ -44,6 +46,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' + variables: + IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' @@ -355,3 +360,31 @@ ivas-pytest-mld-long-dec-lev+10: - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor + +# --------------------------------------------------------------- +# EVS 26.444 test job +# --------------------------------------------------------------- + +# check bitexactness to EVS +be-2-evs-26444: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" + tags: + - be-2-evs-basop + stage: test + needs: ["build-codec-linux-cmake"] + timeout: "120 minutes" # To be revisited + script: + - *print-common-info + - *update-scripts-repo + - make -j + + # copy over to never change the testvector dir + - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test + - cp ./IVAS_cod ./evs_be_test/bin/EVS_cod + - cp ./IVAS_dec ./evs_be_test/bin/EVS_dec + + - cd evs_be_test + - python3 ../ci/run_evs_be_test.py -- GitLab From ef489eec912e376fe9a1cc8eceeebce5b2814691 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 13:21:07 +0100 Subject: [PATCH 23/47] Remove incorrect job dependency --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b83e0dac4..fb5b0ea58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -374,7 +374,6 @@ be-2-evs-26444: tags: - be-2-evs-basop stage: test - needs: ["build-codec-linux-cmake"] timeout: "120 minutes" # To be revisited script: - *print-common-info -- GitLab From cb92e5168ef9ed998e716e1d4d5188af1b9d28ed Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 13:46:12 +0100 Subject: [PATCH 24/47] Added output folders for be-2-evs-26444 --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fb5b0ea58..e3630d04a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -384,6 +384,7 @@ be-2-evs-26444: - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test - cp ./IVAS_cod ./evs_be_test/bin/EVS_cod - cp ./IVAS_dec ./evs_be_test/bin/EVS_dec + - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams - cd evs_be_test - python3 ../ci/run_evs_be_test.py -- GitLab From caef1ba4e22f40ce7f792fab17664c0c1087510b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 17:25:35 +0100 Subject: [PATCH 25/47] Updated 26.444 test to use pytest-based test --- .gitlab-ci.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e3630d04a..1e904be54 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -386,5 +386,18 @@ be-2-evs-26444: - cp ./IVAS_dec ./evs_be_test/bin/EVS_dec - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams - - cd evs_be_test - - python3 ../ci/run_evs_be_test.py + - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi + - exit 0 + + artifacts: + name: "evs-26444--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "EVS 26.444 result" + reports: + junit: + - report-junit.xml -- GitLab From 7bb65a407a33c43838e1b82da9f0df902d229c43 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 17:28:06 +0100 Subject: [PATCH 26/47] Fix expose_as in evs-26444 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e904be54..1bb682f52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -397,7 +397,7 @@ be-2-evs-26444: paths: - report-junit.xml - report.html - expose_as: "EVS 26.444 result" + expose_as: "EVS 26444 result" reports: junit: - report-junit.xml -- GitLab From bd8bc747f60ba9b1ec43d870f9838707271164d7 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 18:29:26 +0100 Subject: [PATCH 27/47] Add disabling of EVS_FLOAT for 26.444 test --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bb682f52..b3e0f64b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -378,12 +378,11 @@ be-2-evs-26444: script: - *print-common-info - *update-scripts-repo + - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h - make -j # copy over to never change the testvector dir - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test - - cp ./IVAS_cod ./evs_be_test/bin/EVS_cod - - cp ./IVAS_dec ./evs_be_test/bin/EVS_dec - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? -- GitLab From 058f4b21d719882ae62ae8cc6731e25befe97d3f Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jan 2024 14:02:40 +0100 Subject: [PATCH 28/47] Remove quotation marks from pytest calls --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3e0f64b8..3da7a9bb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,8 +91,8 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto .update-scripts-repo: &update-scripts-repo @@ -186,9 +186,9 @@ stages: ### run pytest - exit_code=0 - if [ $USE_REF_ENC -eq 1 ]; then - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - else - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true -- GitLab From eb44b243ac944fcf043f3ba4e12a99b25238e6f6 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jan 2024 14:09:42 +0100 Subject: [PATCH 29/47] Set USE_LTV for long jobs --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3da7a9bb3..1cad89f00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -242,6 +242,7 @@ ivas-pytest-mld-enc-dec: - .rules-pytest-mld - .test-job-linux before_script: + - USE_LTV=0 - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 @@ -252,6 +253,7 @@ ivas-pytest-mld-dec: - .rules-pytest-mld - .test-job-linux before_script: + - USE_LTV=0 - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 @@ -262,6 +264,7 @@ ivas-pytest-mld-enc-dec-lev-10: - .rules-pytest-mld - .test-job-linux before_script: + - USE_LTV=0 - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 @@ -272,6 +275,7 @@ ivas-pytest-mld-dec-lev-10: - .rules-pytest-mld - .test-job-linux before_script: + - USE_LTV=0 - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 @@ -282,6 +286,7 @@ ivas-pytest-mld-enc-dec-lev+10: - .rules-pytest-mld - .test-job-linux before_script: + - USE_LTV=0 - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 @@ -292,6 +297,7 @@ ivas-pytest-mld-dec-lev+10: - .rules-pytest-mld - .test-job-linux before_script: + - USE_LTV=0 - USE_REF_ENC=1 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 @@ -306,6 +312,7 @@ ivas-pytest-mld-long-enc-dec: - .rules-pytest-mld-long - .test-job-linux before_script: + - USE_LTV=1 - USE_REF_ENC=0 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 @@ -316,6 +323,7 @@ ivas-pytest-mld-long-dec: - .rules-pytest-mld-long - .test-job-linux before_script: + - USE_LTV=1 - USE_REF_ENC=1 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 @@ -326,6 +334,7 @@ ivas-pytest-mld-long-enc-dec-lev-10: - .rules-pytest-mld-long - .test-job-linux before_script: + - USE_LTV=1 - USE_REF_ENC=0 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 @@ -336,6 +345,7 @@ ivas-pytest-mld-long-dec-lev-10: - .rules-pytest-mld-long - .test-job-linux before_script: + - USE_LTV=1 - USE_REF_ENC=1 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 @@ -346,6 +356,7 @@ ivas-pytest-mld-long-enc-dec-lev+10: - .rules-pytest-mld-long - .test-job-linux before_script: + - USE_LTV=1 - USE_REF_ENC=0 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 @@ -356,6 +367,7 @@ ivas-pytest-mld-long-dec-lev+10: - .rules-pytest-mld-long - .test-job-linux before_script: + - USE_LTV=1 - USE_REF_ENC=1 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 -- GitLab From 7c5f51515b55eee690df4ecc5795d2ea2572e38b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 5 Jan 2024 11:28:36 +0100 Subject: [PATCH 30/47] Changed MLD parsing to use report-junit.xml and extended test name to full name --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1cad89f00..2e1044453 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -192,7 +192,7 @@ stages: - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_mld.py report.html mld.csv + - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi -- GitLab From 6c7dd673407607a66176af30c77678d606bb37b3 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 09:43:06 +0100 Subject: [PATCH 31/47] Added ivas-pytest-mld-dec-lev-10 to run MLD test with -10 dB input scaling --- .gitlab-ci.yml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 632760c1c..8ebd51eb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" + LEVEL_SCALING: 1.0 IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger @@ -97,6 +98,10 @@ stages: - if: $CI_PIPELINE_SOURCE == 'push' when: never +.apply-testv-scaling: &apply-testv-scaling + - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" + - for file in `ls ./scripts/testv/*.wav`; do tests/scale_pcm.py $file tmp.wav $LEVEL_SCALING; mv tmp.wav $file;done + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -222,3 +227,45 @@ ivas-pytest-mld-dec: reports: junit: - report-junit.xml + + +ivas-pytest-mld-dec-lev-10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *remove-unsupported-testcases + - *mld-test-setup-codec + - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) + - *apply-testv-scaling + + ### run pytest + - exit_code=0 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml -- GitLab From 2519f46a28ed7c807118cfdcd7d5022267215a61 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 11:18:10 +0100 Subject: [PATCH 32/47] Fix order of level scaling. Add anchors for LTV --- .gitlab-ci.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ebd51eb8..74ec15085 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,11 +1,12 @@ variables: + LTV_DIR: "/usr/local/ltv" REFERENCE_TAG: "20231128_Update_Ittiam" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" - LEVEL_SCALING: 1.0 + LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger @@ -102,6 +103,16 @@ stages: - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - for file in `ls ./scripts/testv/*.wav`; do tests/scale_pcm.py $file tmp.wav $LEVEL_SCALING; mv tmp.wav $file;done +.update-ltv-repo: &update-ltv-repo + - cd $LTV_DIR + - git pull + - cd - + +.copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir + - cp "$LTV_DIR"/*.wav scripts/testv/ + - cp "$LTV_DIR"/*.met scripts/testv/ + - cp "$LTV_DIR"/*.csv scripts/testv/ + # --------------------------------------------------------------- # Job templates # --------------------------------------------------------------- @@ -239,9 +250,9 @@ ivas-pytest-mld-dec-lev-10: - *print-common-info - *update-scripts-repo - *remove-unsupported-testcases - - *mld-test-setup-codec - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - - *apply-testv-scaling + - *apply-testv-scaling + - *mld-test-setup-codec ### run pytest - exit_code=0 -- GitLab From b947025309284ec72da3654d36378bccc8ddaef9 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 13:59:13 +0100 Subject: [PATCH 33/47] Added long test vectors and renderer tests --- .gitlab-ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74ec15085..292a7a76c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" + TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer tests/" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: @@ -75,8 +75,8 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -168,12 +168,14 @@ ivas-pytest-mld-enc-dec: script: - *print-common-info - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir - *remove-unsupported-testcases - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv @@ -209,12 +211,14 @@ ivas-pytest-mld-dec: script: - *print-common-info - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir - *remove-unsupported-testcases - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv @@ -249,6 +253,8 @@ ivas-pytest-mld-dec-lev-10: script: - *print-common-info - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir - *remove-unsupported-testcases - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - *apply-testv-scaling @@ -256,7 +262,7 @@ ivas-pytest-mld-dec-lev-10: ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv -- GitLab From 46cc1c2b856e9f72e31801f98807d0d575e79771 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 14:27:35 +0100 Subject: [PATCH 34/47] Revert extension of test suite --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 292a7a76c..eaacdd41e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer tests/" + TEST_SUITE: "tests/codec_be_on_mr_nonselection" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: -- GitLab From af3b1a6aebf9e36c875eb9a3b824e9f9f11f1eda Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 14:37:28 +0100 Subject: [PATCH 35/47] Add back extension of tests, now without typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eaacdd41e..d576d0465 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TEST_SUITE: "tests/codec_be_on_mr_nonselection" + TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: -- GitLab From 6d4c52eff6f58f940585274e4a073b4df4943784 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 14:46:41 +0100 Subject: [PATCH 36/47] Added remaining level scaling jobs --- .gitlab-ci.yml | 130 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d576d0465..c53ebd6fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -286,3 +286,133 @@ ivas-pytest-mld-dec-lev-10: reports: junit: - report-junit.xml + +ivas-pytest-mld-dec-lev+10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *remove-unsupported-testcases + - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) + - *apply-testv-scaling + - *mld-test-setup-codec + + ### run pytest + - exit_code=0 + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml + +ivas-pytest-mld-enc-dec-lev-10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *remove-unsupported-testcases + - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) + - *apply-testv-scaling + - *mld-test-setup-codec + + ### run pytest + - exit_code=0 + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml + +ivas-pytest-mld-enc-dec-lev+10: + extends: + - .test-job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "30 minutes" + script: + - *print-common-info + - *update-scripts-repo + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - *remove-unsupported-testcases + - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) + - *apply-testv-scaling + - *mld-test-setup-codec + + ### run pytest + - exit_code=0 + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - python3 scripts/parse_mld.py report.html mld.csv + + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - mld.csv + expose_as: "pytest mld results" + reports: + junit: + - report-junit.xml + -- GitLab From 11ff544fae5acf396a264a520661335c0ae4710f Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 15:59:52 +0100 Subject: [PATCH 37/47] Trying to simplify jobs with anchor --- .gitlab-ci.yml | 275 +++++++------------------------------------------ 1 file changed, 39 insertions(+), 236 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c53ebd6fa..53b75a664 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,10 @@ variables: SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" + LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + USE_REF_ENC: "1" + TEST_SUITE: "" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: @@ -136,29 +139,8 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ -# --------------------------------------------------------------- -# Build jobs -# --------------------------------------------------------------- - -# ensure that codec builds on linux -build-codec-linux-make: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - extends: - - .build-job-linux - script: - - *print-common-info - - make -j - -# --------------------------------------------------------------- -# Test jobs -# --------------------------------------------------------------- -ivas-pytest-mld-enc-dec: +.ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor extends: - .test-job-linux - .rules-pytest-mld @@ -168,14 +150,23 @@ ivas-pytest-mld-enc-dec: script: - *print-common-info - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - fi - *remove-unsupported-testcases + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - if [ $USE_REF_ENC -eq 1 ]; then + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - else + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld.py report.html mld.csv @@ -200,219 +191,31 @@ ivas-pytest-mld-enc-dec: junit: - report-junit.xml +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- -ivas-pytest-mld-dec: - extends: - - .test-job-linux - - .rules-pytest-mld - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - - -ivas-pytest-mld-dec-lev-10: - extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - -ivas-pytest-mld-dec-lev+10: - extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - -ivas-pytest-mld-enc-dec-lev-10: - extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" - script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=0.3162 # -10 dB, 10^(-10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml - -ivas-pytest-mld-enc-dec-lev+10: +# ensure that codec builds on linux +build-codec-linux-make: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'push' + when: never extends: - - .test-job-linux - stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" + - .build-job-linux script: - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *remove-unsupported-testcases - - LEVEL_SCALING=3.162 # +10 dB, 10^(10/20) - - *apply-testv-scaling - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - - python3 scripts/parse_mld.py report.html mld.csv - - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 + - make -j - allow_failure: - exit_codes: - - 123 - artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" - expire_in: 1 week - when: always - paths: - - report-junit.xml - - report.html - - mld.csv - expose_as: "pytest mld results" - reports: - junit: - - report-junit.xml +# --------------------------------------------------------------- +# Test jobs +# --------------------------------------------------------------- +ivas-pytest-mld-enc-dec: + before_script: + - USE_REF_ENC=0 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=1.0 + *ivas-pytest-mld-anchor \ No newline at end of file -- GitLab From f6369818893a7886b06c50abc54e57fbed1934bc Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 16:02:39 +0100 Subject: [PATCH 38/47] Fix syntax in .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53b75a664..751872b5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,4 +218,4 @@ ivas-pytest-mld-enc-dec: - USE_REF_ENC=0 - TEST_SUITE=$SHORT_TEST_SUITE - LEVEL_SCALING=1.0 - *ivas-pytest-mld-anchor \ No newline at end of file + <<: *ivas-pytest-mld-anchor \ No newline at end of file -- GitLab From cb02bb1195267e2283ed51f58e1e7d8f003d0289 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 16:08:58 +0100 Subject: [PATCH 39/47] Expand test set using anchor -- to be verified --- .gitlab-ci.yml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 751872b5a..93ed8d1c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,4 +218,39 @@ ivas-pytest-mld-enc-dec: - USE_REF_ENC=0 - TEST_SUITE=$SHORT_TEST_SUITE - LEVEL_SCALING=1.0 - <<: *ivas-pytest-mld-anchor \ No newline at end of file + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec: + before_script: + - USE_REF_ENC=1 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-enc-dec-lev-10: + before_script: + - USE_REF_ENC=0 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec-lev-10: + before_script: + - USE_REF_ENC=1 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-enc-dec-lev+10: + before_script: + - USE_REF_ENC=0 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec-lev+10: + before_script: + - USE_REF_ENC=1 + - TEST_SUITE=$SHORT_TEST_SUITE + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor -- GitLab From 1f46d6685fed2a40761787cff058bcbf84869ce1 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:38:39 +0100 Subject: [PATCH 40/47] Added short test set for testing, new scale_pcm.py --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 93ed8d1c9..f569c199d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,8 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::\"test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]\"" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" @@ -104,7 +105,7 @@ stages: .apply-testv-scaling: &apply-testv-scaling - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - - for file in `ls ./scripts/testv/*.wav`; do tests/scale_pcm.py $file tmp.wav $LEVEL_SCALING; mv tmp.wav $file;done + - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING .update-ltv-repo: &update-ltv-repo - cd $LTV_DIR -- GitLab From 22aad55ec43925cdc4f5fb7f30bbd4f6f426e931 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:42:27 +0100 Subject: [PATCH 41/47] Fix test suite --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f569c199d..8587bbb68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: EXIT_CODE_FAIL: 1 LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::\"test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]\"" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" -- GitLab From fdea4918ab07ba109f0853cff19e6e4087b9c4c3 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:46:06 +0100 Subject: [PATCH 42/47] Fix quotation marks, hopefully --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8587bbb68..0baeac5dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -164,9 +164,9 @@ stages: ### run pytest - exit_code=0 - if [ $USE_REF_ENC -eq 1 ]; then - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - else - - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true -- GitLab From 204a6a8d05961008becc30d28f934ea96578c7bc Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 18:49:42 +0100 Subject: [PATCH 43/47] Fix quotation marks again --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0baeac5dc..fa93e180e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,8 +79,8 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -217,41 +217,41 @@ build-codec-linux-make: ivas-pytest-mld-enc-dec: before_script: - USE_REF_ENC=0 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec: before_script: - USE_REF_ENC=1 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-enc-dec-lev-10: before_script: - USE_REF_ENC=0 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec-lev-10: before_script: - USE_REF_ENC=1 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-enc-dec-lev+10: before_script: - USE_REF_ENC=0 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor ivas-pytest-mld-dec-lev+10: before_script: - USE_REF_ENC=1 - - TEST_SUITE=$SHORT_TEST_SUITE + - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor -- GitLab From d783b6d3a74e9f6dc595bddd3f9a875a15b12f18 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Dec 2023 19:03:42 +0100 Subject: [PATCH 44/47] Run with set that allows create_part_2 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa93e180e..38f366015 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,8 +6,8 @@ variables: EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 LARGE_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" - #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + #SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection/test_param_file.py::test_param_file_tests[stereo at 16.4 kbps, 32kHz in, 16kHz out, DTX on]" USE_REF_ENC: "1" TEST_SUITE: "" LEVEL_SCALING: "1.0" -- GitLab From 7f210c02ba5d790f0bc3f04239f9ff0cb02e4e8d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 2 Jan 2024 10:01:45 +0100 Subject: [PATCH 45/47] Add dependent job build-codec-linux-make for scheduled jobs --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38f366015..ee11859f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,6 +202,7 @@ build-codec-linux-make: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: -- GitLab From 78a1c085a7c6c2c8a1904d9f0cc89817815667c5 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 3 Jan 2024 14:02:40 +0100 Subject: [PATCH 46/47] Remove quotation marks from pytest calls --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee11859f9..c92fb251f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,8 +79,8 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest "$TEST_SUITE" -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -164,9 +164,9 @@ stages: ### run pytest - exit_code=0 - if [ $USE_REF_ENC -eq 1 ]; then - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - else - - python3 -m pytest "$TEST_SUITE" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true -- GitLab From 93ce1cb75ecf1bb6c76bf5e3aaf7e270b7d7ef82 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 5 Jan 2024 11:28:36 +0100 Subject: [PATCH 47/47] Changed MLD parsing to use report-junit.xml and extended test name to full name --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c92fb251f..d10c1db71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,7 +170,7 @@ stages: - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_mld.py report.html mld.csv + - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi -- GitLab