diff --git a/main-basop.yml b/main-basop.yml index 4394b4968ca6f7db4ce1488e4fedc71b3f2c285c..ccb837f412eade3899a8557566cc948e325cb4bc 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -259,7 +259,6 @@ workflow: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -678,7 +677,9 @@ workflow: - if [[ $CLANG_NUM = 1 ]]; then - testcase_timeout_arg="" - fi - - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH + + # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) + - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -714,7 +715,8 @@ workflow: - if [ $LEVEL_SCALING != "1.0" ];then - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -722,7 +724,7 @@ workflow: ### run pytest - exit_code=0 - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv @@ -2181,10 +2183,10 @@ coverage-test-on-main-scheduled: - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 - exit_code_enc=0 - - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path IVAS_dec || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer - - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_dec || exit_code_enc=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer # remove apps and lib_util files from coverage @@ -2314,21 +2316,23 @@ check-be-between-renderer-framesizes: stage: test needs: ["build-codec-linux-make"] script: + - echo "$SHORT_TEST_SUITE" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh - make clean - make -j - - echo "$SHORT_TEST_SUITE" - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? + # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? - exit_code5=0 - exit_code10=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true @@ -2569,7 +2573,6 @@ backup-long-term-job-logs: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh - *complexity-measurements-setup # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present) - rm -rf COMPLEXITY/logs diff --git a/snippets/basop/build-and-create-float-ref-outputs.sh b/snippets/basop/build-and-create-float-ref-outputs.sh index 0db6483a028971d8702c59a16bfce3b74a8fc42b..a3f8244eaa540343130fb8d600314c3a229b4677 100644 --- a/snippets/basop/build-and-create-float-ref-outputs.sh +++ b/snippets/basop/build-and-create-float-ref-outputs.sh @@ -4,7 +4,9 @@ set -euxo pipefail cd "${CI_PROJECT_DIR}" -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut ### prepare pytest # create short test vectors diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 67c07f9c8e88e255aeed88785ff7b0781b266928..00c680d5c0aad6754a7728702d2a470a1dcf782b 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -39,6 +39,16 @@ merge-source) isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$CUT_COMMIT_FILE" ;; +dut) + # This produces the same binary names as merge-source, but sets branch_to_check_out differently + # so that this is usable in non-merge-request pipelines + branch_to_check_out="$CI_COMMIT_REF_NAME" + ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + commit_file="$CUT_COMMIT_FILE" + ;; *) echo "Error: invalid mode: $mode" exit 2 @@ -47,11 +57,6 @@ esac echo "Building $mode binaries from branch $branch_to_check_out" -# TODO: could probably always be run -if [[ "$mode" == "float-ref" ]]; then - git rev-parse HEAD >"$CUT_COMMIT_FILE" -fi - current_commit_sha="$(git rev-parse HEAD)" restore_stash() { @@ -65,9 +70,7 @@ trap restore_stash EXIT # should never fail, even if there is nothing to be stored git stash -if [[ "$mode" != "merge-source" ]]; then - git checkout "$branch_to_check_out" -fi +git checkout "$branch_to_check_out" bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh diff --git a/snippets/basop/build-float-ref-and-dut-binaries.sh b/snippets/basop/build-float-ref-and-dut-binaries.sh deleted file mode 100755 index 9f1746a1d691cd9a31eaea7176a5729af6c04320..0000000000000000000000000000000000000000 --- a/snippets/basop/build-float-ref-and-dut-binaries.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - -set -euxo pipefail - -cd "${CI_PROJECT_DIR}" - -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh - -make clean -make -j "$(nproc)"