diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 632760c1c380e82cef0c7c9d90b46d0b67385e82..7d7a0b726cb603d6032c28c1fbac925229394cbb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,16 @@ variables: + LTV_DIR: "/usr/local/ltv" + EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" REFERENCE_TAG: "20231128_Update_Ittiam" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 EXIT_CODE_FAIL: 1 - TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + USE_REF_ENC: "1" + TEST_SUITE: "" + LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger @@ -12,6 +18,8 @@ variables: options: - 'default' - 'pytest-mld' + - 'pytest-mld-long' + - 'evs-26444' default: @@ -35,6 +43,16 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' + variables: + IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' + variables: + IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch + variables: + IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + stages: - build @@ -73,8 +91,9 @@ stages: # create short test vectors - python3 tests/create_short_testvectors.py # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref -n auto - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto + - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto + .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -89,6 +108,21 @@ stages: .remove-unsupported-testcases: &remove-unsupported-testcases - sed -i '1649,1655d' scripts/config/self_test.prm + +.apply-testv-scaling: &apply-testv-scaling + - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" + - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING + +.update-ltv-repo: &update-ltv-repo + - cd $LTV_DIR + - git pull + - cd - + +.copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir + - cp "$LTV_DIR"/*.wav scripts/testv/ + - cp "$LTV_DIR"/*.met scripts/testv/ + - cp "$LTV_DIR"/*.csv scripts/testv/ + .rules-pytest-mld: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld" @@ -96,6 +130,17 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'push' when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + +.rules-pytest-mld-long: + rules: + - if: $PYTEST_MLD_LONG # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never # --------------------------------------------------------------- # Job templates @@ -120,47 +165,34 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ -# --------------------------------------------------------------- -# Build jobs -# --------------------------------------------------------------- - -# ensure that codec builds on linux -build-codec-linux-make: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - extends: - - .build-job-linux - script: - - *print-common-info - - make -j - -# --------------------------------------------------------------- -# Test jobs -# --------------------------------------------------------------- -ivas-pytest-mld-enc-dec: - extends: - - .test-job-linux - - .rules-pytest-mld +.ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor stage: test needs: ["build-codec-linux-make"] timeout: "30 minutes" script: - *print-common-info - *update-scripts-repo + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - fi - *remove-unsupported-testcases + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi - *mld-test-setup-codec ### run pytest - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - if [ $USE_REF_ENC -eq 1 ]; then + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? + - else + - python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto || exit_code=$? + - fi - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - python3 scripts/parse_mld.py report.html mld.csv + - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi @@ -182,43 +214,201 @@ ivas-pytest-mld-enc-dec: junit: - report-junit.xml +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- -ivas-pytest-mld-dec: +# ensure that codec builds on linux +build-codec-linux-make: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + script: + - *print-common-info + - make -j + +# --------------------------------------------------------------- +# Short test jobs +# --------------------------------------------------------------- + +ivas-pytest-mld-enc-dec: extends: + - .rules-pytest-mld - .test-job-linux + before_script: + - USE_LTV=0 + - USE_REF_ENC=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - USE_REF_ENC=1 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-enc-dec-lev-10: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - USE_REF_ENC=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec-lev-10: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - USE_REF_ENC=1 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-enc-dec-lev+10: + extends: + - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - USE_REF_ENC=0 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-dec-lev+10: + extends: - .rules-pytest-mld + - .test-job-linux + before_script: + - USE_LTV=0 + - USE_REF_ENC=1 + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +# --------------------------------------------------------------- +# Long test jobs +# --------------------------------------------------------------- + +ivas-pytest-mld-long-enc-dec: + extends: + - .rules-pytest-mld-long + - .test-job-linux + before_script: + - USE_LTV=1 + - USE_REF_ENC=0 + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-dec: + extends: + - .rules-pytest-mld-long + - .test-job-linux + before_script: + - USE_LTV=1 + - USE_REF_ENC=1 + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-dec-lev-10: + extends: + - .rules-pytest-mld-long + - .test-job-linux + before_script: + - USE_LTV=1 + - USE_REF_ENC=0 + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-dec-lev-10: + extends: + - .rules-pytest-mld-long + - .test-job-linux + before_script: + - USE_LTV=1 + - USE_REF_ENC=1 + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-dec-lev+10: + extends: + - .rules-pytest-mld-long + - .test-job-linux + before_script: + - USE_LTV=1 + - USE_REF_ENC=0 + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-dec-lev+10: + extends: + - .rules-pytest-mld-long + - .test-job-linux + before_script: + - USE_LTV=1 + - USE_REF_ENC=1 + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +# --------------------------------------------------------------- +# EVS 26.444 test job +# --------------------------------------------------------------- + +# check bitexactness to EVS +be-2-evs-26444: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" + tags: + - be-2-evs-basop stage: test - needs: ["build-codec-linux-make"] - timeout: "30 minutes" + timeout: "120 minutes" # To be revisited script: - *print-common-info - - *update-scripts-repo - - *remove-unsupported-testcases - - *mld-test-setup-codec - - ### run pytest - - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - *update-scripts-repo + - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h + - make -j - - python3 scripts/parse_mld.py report.html mld.csv + # copy over to never change the testvector dir + - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test + - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi - exit 0 - - allow_failure: - exit_codes: - - 123 + artifacts: - name: "mld--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + name: "evs-26444--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always paths: - report-junit.xml - report.html - - mld.csv - expose_as: "pytest mld results" + expose_as: "EVS 26444 result" reports: junit: - - report-junit.xml + - report-junit.xml