From 98323f9aa9f25ddc1c0388de21e90410a033cf3e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 1 Aug 2025 11:36:09 +0200 Subject: [PATCH 1/3] fixes to building and naming of binaries for schedule jobs --- main-basop.yml | 27 ++++++++++--------- .../build-and-create-float-ref-outputs.sh | 7 +++-- .../basop/build-float-ref-and-dut-binaries.sh | 12 --------- 3 files changed, 19 insertions(+), 27 deletions(-) delete mode 100755 snippets/basop/build-float-ref-and-dut-binaries.sh diff --git a/main-basop.yml b/main-basop.yml index 4394b49..684b0f0 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -259,7 +259,6 @@ workflow: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -280,7 +279,7 @@ workflow: ### run pytest - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path IVAS_cod --dut_decoder_path IVAS_dec || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms @@ -678,7 +677,7 @@ workflow: - if [[ $CLANG_NUM = 1 ]]; then - testcase_timeout_arg="" - fi - - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH + - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -714,7 +713,9 @@ workflow: - if [ $LEVEL_SCALING != "1.0" ];then - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - make clean + - make -j - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -722,7 +723,7 @@ workflow: ### run pytest - exit_code=0 - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_cod --dut_decoder_path IVAS_dec --compare_to_input || exit_code=$? - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv @@ -2181,10 +2182,10 @@ coverage-test-on-main-scheduled: - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code_dec=0 - exit_code_enc=0 - - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path IVAS_dec || exit_code_dec=$? - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer - - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_dec || exit_code_enc=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer # remove apps and lib_util files from coverage @@ -2314,21 +2315,22 @@ check-be-between-renderer-framesizes: stage: test needs: ["build-codec-linux-make"] script: + - echo "$SHORT_TEST_SUITE" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh - make clean - make -j - - echo "$SHORT_TEST_SUITE" - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? - exit_code5=0 - exit_code10=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true @@ -2569,7 +2571,6 @@ backup-long-term-job-logs: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh - *complexity-measurements-setup # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present) - rm -rf COMPLEXITY/logs diff --git a/snippets/basop/build-and-create-float-ref-outputs.sh b/snippets/basop/build-and-create-float-ref-outputs.sh index 0db6483..b27dd82 100644 --- a/snippets/basop/build-and-create-float-ref-outputs.sh +++ b/snippets/basop/build-and-create-float-ref-outputs.sh @@ -4,7 +4,10 @@ set -euxo pipefail cd "${CI_PROJECT_DIR}" -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + +make clean +make -j ### prepare pytest # create short test vectors @@ -23,4 +26,4 @@ if [ "$COMPARE_DMX" = "true" ]; then enc_dmx_arg="--compare_enc_dmx" fi -python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? +python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path IVAS_cod --dut_decoder_path IVAS_dec || exit_code=$? diff --git a/snippets/basop/build-float-ref-and-dut-binaries.sh b/snippets/basop/build-float-ref-and-dut-binaries.sh deleted file mode 100755 index 9f1746a..0000000 --- a/snippets/basop/build-float-ref-and-dut-binaries.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - -set -euxo pipefail - -cd "${CI_PROJECT_DIR}" - -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh - -make clean -make -j "$(nproc)" -- GitLab From 569a3382eefa86546e1fa6312ab72e6f19a173e6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 1 Aug 2025 13:57:00 +0200 Subject: [PATCH 2/3] add dut arg to build-binaries.sh --- snippets/basop/build-binaries.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 67c07f9..00c680d 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -39,6 +39,16 @@ merge-source) isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$CUT_COMMIT_FILE" ;; +dut) + # This produces the same binary names as merge-source, but sets branch_to_check_out differently + # so that this is usable in non-merge-request pipelines + branch_to_check_out="$CI_COMMIT_REF_NAME" + ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + commit_file="$CUT_COMMIT_FILE" + ;; *) echo "Error: invalid mode: $mode" exit 2 @@ -47,11 +57,6 @@ esac echo "Building $mode binaries from branch $branch_to_check_out" -# TODO: could probably always be run -if [[ "$mode" == "float-ref" ]]; then - git rev-parse HEAD >"$CUT_COMMIT_FILE" -fi - current_commit_sha="$(git rev-parse HEAD)" restore_stash() { @@ -65,9 +70,7 @@ trap restore_stash EXIT # should never fail, even if there is nothing to be stored git stash -if [[ "$mode" != "merge-source" ]]; then - git checkout "$branch_to_check_out" -fi +git checkout "$branch_to_check_out" bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh -- GitLab From 89b46f25b21910b1b2c0fd45b6d713d7c9c1eb7a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 1 Aug 2025 14:13:06 +0200 Subject: [PATCH 3/3] restore some of the DUT_..CODEC_PATH -> IVAS_... changes --- main-basop.yml | 10 ++++++---- snippets/basop/build-and-create-float-ref-outputs.sh | 5 ++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 684b0f0..ccb837f 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -279,7 +279,7 @@ workflow: ### run pytest - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path IVAS_cod --dut_decoder_path IVAS_dec || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms @@ -677,6 +677,8 @@ workflow: - if [[ $CLANG_NUM = 1 ]]; then - testcase_timeout_arg="" - fi + + # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" @@ -714,8 +716,7 @@ workflow: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - - make clean - - make -j + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -723,7 +724,7 @@ workflow: ### run pytest - exit_code=0 - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_cod --dut_decoder_path IVAS_dec --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv @@ -2325,6 +2326,7 @@ check-be-between-renderer-framesizes: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code=0 + # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? - exit_code5=0 diff --git a/snippets/basop/build-and-create-float-ref-outputs.sh b/snippets/basop/build-and-create-float-ref-outputs.sh index b27dd82..a3f8244 100644 --- a/snippets/basop/build-and-create-float-ref-outputs.sh +++ b/snippets/basop/build-and-create-float-ref-outputs.sh @@ -6,8 +6,7 @@ cd "${CI_PROJECT_DIR}" bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref -make clean -make -j +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut ### prepare pytest # create short test vectors @@ -26,4 +25,4 @@ if [ "$COMPARE_DMX" = "true" ]; then enc_dmx_arg="--compare_enc_dmx" fi -python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path IVAS_cod --dut_decoder_path IVAS_dec || exit_code=$? +python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? -- GitLab