diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a49643a725c2df8053a3282b54912e9df1a6a80..2aada60ca33da2eaa4a086afa15ba7296ebd3646 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,9 @@ variables: EXIT_CODE_FAIL: 1 LONG_TEST_SUITE: "tests/test_param_file_ltv.py tests/renderer" SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - USE_REF_ENC: "1" TEST_SUITE: "" + DUT_ENCODER_PATH: "./IVAS_cod" + DUT_DECODER_PATH: "./IVAS_dec" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" @@ -21,6 +22,7 @@ variables: options: - 'default' - 'pytest-mld' + - 'pytest-mld-interop' - 'pytest-mld-long' - 'pytest-saturation-smoke-test' - 'evs-26444' @@ -48,6 +50,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-interop' + variables: + IVAS_PIPELINE_NAME: 'Run MLD tool agains float ref - interop: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long' variables: IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH' @@ -158,6 +163,15 @@ stages: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' when: never + +.rules-pytest-mld-interop: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-interop" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + .rules-pytest-saturation-smoke-test: rules: - if: $PYTEST_SMOKE_TEST # Set by scheduled pipeline @@ -191,7 +205,6 @@ stages: - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ - .ivas-pytest-mld-anchor: &ivas-pytest-mld-anchor stage: test needs: ["build-codec-linux-make"] @@ -213,11 +226,7 @@ stages: - exit_code=0 # timeout of 15 min per individual testcase - hopefully too much, but better be safe for now - testcase_timeout=900 - - if [ $USE_REF_ENC -eq 1 ]; then - - python3 -m pytest $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto --testcase_timeout $testcase_timeout || exit_code=$? - - else - - python3 -m pytest $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto --testcase_timeout $testcase_timeout || exit_code=$? - - fi + - python3 -m pytest $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv @@ -320,7 +329,6 @@ ivas-pytest-mld-enc-dec: - .test-job-linux before_script: - USE_LTV=0 - - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -331,7 +339,7 @@ ivas-pytest-mld-dec: - .test-job-linux before_script: - USE_LTV=0 - - USE_REF_ENC=1 + - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -342,7 +350,6 @@ ivas-pytest-mld-enc-dec-lev-10: - .test-job-linux before_script: - USE_LTV=0 - - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -353,7 +360,7 @@ ivas-pytest-mld-dec-lev-10: - .test-job-linux before_script: - USE_LTV=0 - - USE_REF_ENC=1 + - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -364,7 +371,6 @@ ivas-pytest-mld-enc-dec-lev+10: - .test-job-linux before_script: - USE_LTV=0 - - USE_REF_ENC=0 - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor @@ -375,7 +381,40 @@ ivas-pytest-mld-dec-lev+10: - .test-job-linux before_script: - USE_LTV=0 - - USE_REF_ENC=1 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-fx-dec-flt-interop: + extends: + - .rules-pytest-mld-interop + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-fx-dec-flt-lev-10-interop: + extends: + - .rules-pytest-mld-interop + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-mld-anchor + +ivas-pytest-mld-long-enc-fx-dec-flt-lev+10-interop: + extends: + - .rules-pytest-mld-interop + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor @@ -411,7 +450,6 @@ ivas-pytest-mld-long-enc-dec: - .test-job-linux before_script: - USE_LTV=1 - - USE_REF_ENC=0 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -422,7 +460,7 @@ ivas-pytest-mld-long-dec: - .test-job-linux before_script: - USE_LTV=1 - - USE_REF_ENC=1 + - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-mld-anchor @@ -433,7 +471,6 @@ ivas-pytest-mld-long-enc-dec-lev-10: - .test-job-linux before_script: - USE_LTV=1 - - USE_REF_ENC=0 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -444,7 +481,7 @@ ivas-pytest-mld-long-dec-lev-10: - .test-job-linux before_script: - USE_LTV=1 - - USE_REF_ENC=1 + - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-mld-anchor @@ -455,7 +492,6 @@ ivas-pytest-mld-long-enc-dec-lev+10: - .test-job-linux before_script: - USE_LTV=1 - - USE_REF_ENC=0 - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor @@ -466,7 +502,7 @@ ivas-pytest-mld-long-dec-lev+10: - .test-job-linux before_script: - USE_LTV=1 - - USE_REF_ENC=1 + - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 <<: *ivas-pytest-mld-anchor