From 8c0f552aab94fe99564f436754a03a4f040b2f6a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 14:27:52 +0200 Subject: [PATCH 01/13] rework branch-is-up-to-date-with-target-pre job - unshallow repo to always allow commits behind count to work - store not only target commit, but also float ref branches --- includes/basop-default-variables.yml | 3 + main-basop.yml | 19 +++---- ...ranch-follows-porting-naming-convention.sh | 51 +++++++++++++++++ .../branch-is-up-to-date-with-target-pre.sh | 57 +++++++++++++++++++ snippets/basop/get-float-ref-branch-name.sh | 18 +----- .../basop/set-float-reference-branches.sh | 2 +- 6 files changed, 123 insertions(+), 27 deletions(-) create mode 100644 includes/basop-default-variables.yml create mode 100644 snippets/basop/branch-follows-porting-naming-convention.sh create mode 100644 snippets/basop/branch-is-up-to-date-with-target-pre.sh diff --git a/includes/basop-default-variables.yml b/includes/basop-default-variables.yml new file mode 100644 index 0000000..bb52ac7 --- /dev/null +++ b/includes/basop-default-variables.yml @@ -0,0 +1,3 @@ + FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" + MERGE_SOURCE_FLOAT_REF_COMMIT_FILE: "merge-source-float-ref-git-sha.txt" + MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" diff --git a/main-basop.yml b/main-basop.yml index 554e561..87f17ec 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1,5 +1,6 @@ include: - local: main.yml + - local: includes/basop-default-variables.yml stages: - .pre @@ -835,6 +836,11 @@ fail-pipeline-if-in-draft: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME - fi +# This job runs in MR pipelines and checks if your branch is up-to-date with your target +# If it is not, it fails the whole pipeline early (this should be the first job to run) +# It also stores the target branch's current commit as well as the float_ref branch's current commit +# to ensure that later jobs that need access to these branches all run on the same commit - which is +# important in case the branch is updated during the runtime of the pipeline branch-is-up-to-date-with-target-pre: extends: - .job-linux @@ -844,19 +850,12 @@ branch-is-up-to-date-with-target-pre: tags: - ivas-basop-linux script: - - | - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" - if [ $commits_behind_count -ne 0 ]; then - echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." - exit 1 - fi - - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" - - TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")" - - echo "TARGET_COMMIT is $TARGET_COMMIT" - - echo "$TARGET_COMMIT" > "$MERGE_TARGET_COMMIT_FILE" + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-is-up-to-date-with-target-pre.sh artifacts: paths: - $MERGE_TARGET_COMMIT_FILE + - $FLOAT_REF_COMMIT_FILE + - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE branch-is-up-to-date-with-target-post: extends: diff --git a/snippets/basop/branch-follows-porting-naming-convention.sh b/snippets/basop/branch-follows-porting-naming-convention.sh new file mode 100644 index 0000000..962c231 --- /dev/null +++ b/snippets/basop/branch-follows-porting-naming-convention.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +If this branch is porting a float MR with a corresponding float part, then +ticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec + +If this is not a porting branch, ignore above message. +" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +if ! [[ "$1" =~ $BASOP_PATTERN ]]; then + printf "$MSG_DOES_NOT_MATCH" "$1" + exit 1 +fi diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh new file mode 100644 index 0000000..796650b --- /dev/null +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +# fetch all needed branches +# we need a full repo here to always be sure we get enough commits so the commits_behind_count thingy below works (only relevant for switching back to fetch GIT_STRATEGY) +git fetch --unshallow +git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" +git fetch origin ivas-float-update +float_ref_branchname=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME) +git fetch origin "$float_ref_branchname" + +# then check if we are behind the merge target +commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" +if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 +fi + +# if we are up-to-date, store the commit SHA's for all branches +MERGE_TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")" +echo "MERGE_TARGET_COMMIT is $MERGE_TARGET_COMMIT" +echo "$MERGE_TARGET_COMMIT" >"$MERGE_TARGET_COMMIT_FILE" + +FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")" +echo "FLOAT_REF_COMMIT is $FLOAT_REF_COMMIT" +echo "$FLOAT_REF_COMMIT" >"$FLOAT_REF_COMMIT_FILE" + +IVAS_FLOAT_UPDATE_COMMIT="$(git rev-parse "origin/ivas-float-update")" +echo "IVAS_FLOAT_UPDATE_COMMIT is $IVAS_FLOAT_UPDATE_COMMIT" +echo "$IVAS_FLOAT_UPDATE_COMMIT" >"$IVAS_FLOAT_UPDATE_COMMIT_FILE" diff --git a/snippets/basop/get-float-ref-branch-name.sh b/snippets/basop/get-float-ref-branch-name.sh index 3c4574a..5720f55 100755 --- a/snippets/basop/get-float-ref-branch-name.sh +++ b/snippets/basop/get-float-ref-branch-name.sh @@ -28,17 +28,8 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" -MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. -If this branch is porting a float MR with a corresponding float part, then -ticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. -Please rename your branch. You can easily do this by creating a new branch from this branch: - - git checkout -b -You then also need to create a new merge request and update the links in your issue. -See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec - -If this is not a porting branch, ignore above message. -" +# NOTE: this does not check e.g. for float_ref_branchname actually existing and will silently return ivas-float-update if not +# Use branch-follows-porting-naming-convention.sh for that if [ $# -ne 1 ]; then echo "Usage: $0 " @@ -48,11 +39,6 @@ fi branchname_in="$1" branchname_out="ivas-float-update" -if ! [[ "$branchname_in" =~ $BASOP_PATTERN ]]; then - printf "$MSG_DOES_NOT_MATCH" "$branchname_in" - exit 1 -fi - # The float ref branch is just the same name, but with basop -> ref replacement # Replace only the first occurrence, as "basop" may be present in the later description # If the format is correct, then before "_basop", only numbers can occur diff --git a/snippets/basop/set-float-reference-branches.sh b/snippets/basop/set-float-reference-branches.sh index 37044d8..0b12a7e 100644 --- a/snippets/basop/set-float-reference-branches.sh +++ b/snippets/basop/set-float-reference-branches.sh @@ -9,7 +9,7 @@ FLOAT_REF_BRANCH_MERGE_SOURCE=$FLOAT_REF_BRANCH exit_code=0 # first check if branch matches the pattern -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME || exit_code=$? +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-follows-porting-naming-convention.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME || exit_code=$? # run again to actually set the branch name only if the naming pattern is there # TODO: this is a bit awkward, but needed as a quick fix for now -- GitLab From 939e758147557f4fdb8e4c2bbf5d0ce7d01c1ca3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 14:39:19 +0200 Subject: [PATCH 02/13] fix variables section syntax --- includes/basop-default-variables.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/includes/basop-default-variables.yml b/includes/basop-default-variables.yml index bb52ac7..e6075c7 100644 --- a/includes/basop-default-variables.yml +++ b/includes/basop-default-variables.yml @@ -1,3 +1,4 @@ +variables: FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" MERGE_SOURCE_FLOAT_REF_COMMIT_FILE: "merge-source-float-ref-git-sha.txt" MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" -- GitLab From cd903600a3bcc26d9e516b30f2f1a9cbf9a12ecd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 14:43:36 +0200 Subject: [PATCH 03/13] remove unshallow as it does not work with current clone strat --- snippets/basop/branch-is-up-to-date-with-target-pre.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index 796650b..d4ec2a4 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -29,8 +29,6 @@ # the United Nations Convention on Contracts on the International Sales of Goods. # fetch all needed branches -# we need a full repo here to always be sure we get enough commits so the commits_behind_count thingy below works (only relevant for switching back to fetch GIT_STRATEGY) -git fetch --unshallow git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" git fetch origin ivas-float-update float_ref_branchname=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME) -- GitLab From bf524f2a0d0dd32fbb2080c86a19f5da44236894 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 14:51:18 +0200 Subject: [PATCH 04/13] fix naming of variables --- snippets/basop/branch-is-up-to-date-with-target-pre.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index d4ec2a4..4f07443 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -46,10 +46,10 @@ MERGE_TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAM echo "MERGE_TARGET_COMMIT is $MERGE_TARGET_COMMIT" echo "$MERGE_TARGET_COMMIT" >"$MERGE_TARGET_COMMIT_FILE" -FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")" +MERGE_SOURCE_FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")" +echo "FLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT" +echo "$MERGE_SOURCE_FLOAT_REF_COMMIT" >"$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" + +FLOAT_REF_COMMIT="$(git rev-parse "origin/ivas-float-update")" echo "FLOAT_REF_COMMIT is $FLOAT_REF_COMMIT" echo "$FLOAT_REF_COMMIT" >"$FLOAT_REF_COMMIT_FILE" - -IVAS_FLOAT_UPDATE_COMMIT="$(git rev-parse "origin/ivas-float-update")" -echo "IVAS_FLOAT_UPDATE_COMMIT is $IVAS_FLOAT_UPDATE_COMMIT" -echo "$IVAS_FLOAT_UPDATE_COMMIT" >"$IVAS_FLOAT_UPDATE_COMMIT_FILE" -- GitLab From 6bc16129e5d16783e0108b151cb8d6f6054e4cdf Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 16:09:37 +0200 Subject: [PATCH 05/13] fix printout --- snippets/basop/branch-is-up-to-date-with-target-pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index 4f07443..7cc15e9 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -47,7 +47,7 @@ echo "MERGE_TARGET_COMMIT is $MERGE_TARGET_COMMIT" echo "$MERGE_TARGET_COMMIT" >"$MERGE_TARGET_COMMIT_FILE" MERGE_SOURCE_FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")" -echo "FLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT" +echo "MERGE_SOURCEFLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT" echo "$MERGE_SOURCE_FLOAT_REF_COMMIT" >"$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" FLOAT_REF_COMMIT="$(git rev-parse "origin/ivas-float-update")" -- GitLab From 8333171115541533998ce3edaddfc524c329b6e3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 16:10:07 +0200 Subject: [PATCH 06/13] [revert-me] shorten pytest-based tests for faster testing --- main-basop.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 87f17ec..cd0cd39 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -299,7 +299,7 @@ workflow: - mv $DUT_RENDERER_PATH IVAS_rend - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest -k stereo --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms @@ -429,11 +429,11 @@ workflow: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code_target=0 - - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? - exit_code=0 - rm -rf .pytest_cache || true - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE @@ -574,10 +574,10 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH # Store branch outputs for later comparison @@ -589,10 +589,10 @@ workflow: ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions @@ -1252,13 +1252,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1321,7 +1321,7 @@ ivas-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1329,7 +1329,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=600 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1389,7 +1389,7 @@ split-rendering-pytest-on-merge-request: # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181 - mv ISAR_post_rend ISAR_post_rend_tmp - cp $REF_POST_RENDERER_PATH ISAR_post_rend - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - mv ISAR_post_rend_tmp ISAR_post_rend ### Run test using scripts and input from main @@ -1400,7 +1400,7 @@ split-rendering-pytest-on-merge-request: - echo "$comp_args" # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1533,7 +1533,7 @@ ivas-interop-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1568,7 +1568,7 @@ split-rendering-smoke-test: - make -j - testcase_timeout=10 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1614,7 +1614,7 @@ renderer-smoke-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - make -j IVAS_rend - testcase_timeout=60 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1640,7 +1640,7 @@ renderer-asan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1669,7 +1669,7 @@ renderer-msan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1698,7 +1698,7 @@ renderer-usan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? @@ -1759,12 +1759,12 @@ renderer-usan: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi -- GitLab From 000dbcab359151056d62e3993a0a6b60b605595a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 22 Sep 2025 16:38:50 +0200 Subject: [PATCH 07/13] also use dotenv mechanism and print commits in jobs --- main-basop.yml | 10 ++++++++++ snippets/basop/branch-is-up-to-date-with-target-pre.sh | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/main-basop.yml b/main-basop.yml index cd0cd39..fd2f4df 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -381,6 +381,14 @@ workflow: - if [ "$DISABLE_HRTF" = "true" ]; then - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi + - echo "Commits from Pre job from file:" + - cat $MERGE_TARGET_COMMIT_FILE + - cat $FLOAT_REF_COMMIT_FILE + - cat $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE + - echo "Commits from Pre job from env vars:" + - echo "MERGE_TARGET_COMMIT=$MERGE_TARGET_COMMIT" + - echo "MERGE_SOURCE_FLOAT_REF_COMMIT=$MERGE_SOURCE_FLOAT_REF_COMMIT" + - echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" variables: USE_LTV: 0 @@ -856,6 +864,8 @@ branch-is-up-to-date-with-target-pre: - $MERGE_TARGET_COMMIT_FILE - $FLOAT_REF_COMMIT_FILE - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE + reports: + dotenv: commits.env branch-is-up-to-date-with-target-post: extends: diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index 7cc15e9..e3a16a7 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -53,3 +53,8 @@ echo "$MERGE_SOURCE_FLOAT_REF_COMMIT" >"$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" FLOAT_REF_COMMIT="$(git rev-parse "origin/ivas-float-update")" echo "FLOAT_REF_COMMIT is $FLOAT_REF_COMMIT" echo "$FLOAT_REF_COMMIT" >"$FLOAT_REF_COMMIT_FILE" + +# create commits.env file for passing the variables to other jobs +echo "MERGE_TARGET_COMMIT=$MERGE_TARGET_COMMIT" >>commits.env +echo "MERGE_SOURCE_FLOAT_REF_COMMIT=$MERGE_SOURCE_FLOAT_REF_COMMIT" >>commits.env +echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" >>commits.env -- GitLab From b39492972ca536534d1418cfdcdeb8bf9008f49a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 23 Sep 2025 08:54:13 +0200 Subject: [PATCH 08/13] use refs instead of branches in the build script - also no need to set up float ref branch anymore, the commit we need will come from the env var via the artifacts --- main-basop.yml | 8 ++----- snippets/basop/build-binaries.sh | 22 +++++------------ .../basop/set-float-reference-branches.sh | 24 ------------------- 3 files changed, 8 insertions(+), 46 deletions(-) delete mode 100644 snippets/basop/set-float-reference-branches.sh diff --git a/main-basop.yml b/main-basop.yml index fd2f4df..14e146f 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -423,15 +423,13 @@ workflow: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - # set branches for reference builds - - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/set-float-reference-branches.sh - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" # build merge target branch and correpsonding float reference - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" # build to-be-merged branch and corresponding float ref branch - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source - python3 ci/remove_unsupported_testcases.py $PRM_FILES @@ -559,13 +557,11 @@ workflow: - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" - # set branches for reference builds - - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/set-float-reference-branches.sh # build merge target branch and correpsonding float reference - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target # build to-be-merged branch and corresponding float ref branch - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source # set pytest args for ref creation diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 00c680d..624d701 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -8,7 +8,7 @@ mode="${1:-}" case "$mode" in float-ref) - branch_to_check_out="$FLOAT_REF_BRANCH" + ref_to_check_out="$FLOAT_REF_COMMIT" ivas_cod_filename="./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -16,7 +16,7 @@ float-ref) commit_file="$FLOAT_REF_COMMIT_FILE" ;; float-ref-merge-source) - branch_to_check_out="$FLOAT_REF_BRANCH_MERGE_SOURCE" + ref_to_check_out="$MERGE_SOURCE_FLOAT_REF_COMMIT" ivas_cod_filename="./$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$MERGE_SOURCE_FLOAT_REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -24,25 +24,15 @@ float-ref-merge-source) commit_file="$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" ;; merge-target) - branch_to_check_out="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" + ref_to_check_out="$MERGE_TARGET_COMMIT" ivas_cod_filename="./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$MERGE_TARGET_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" isar_post_rend_filename="./$MERGE_TARGET_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$MERGE_TARGET_COMMIT_FILE" ;; -merge-source) - branch_to_check_out="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" - ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" - ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" - ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" - isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" - commit_file="$CUT_COMMIT_FILE" - ;; dut) - # This produces the same binary names as merge-source, but sets branch_to_check_out differently - # so that this is usable in non-merge-request pipelines - branch_to_check_out="$CI_COMMIT_REF_NAME" + ref_to_check_out="$CI_COMMIT_REF_NAME" ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -55,7 +45,7 @@ dut) ;; esac -echo "Building $mode binaries from branch $branch_to_check_out" +echo "Building $mode binaries from branch $ref_to_check_out" current_commit_sha="$(git rev-parse HEAD)" @@ -70,7 +60,7 @@ trap restore_stash EXIT # should never fail, even if there is nothing to be stored git stash -git checkout "$branch_to_check_out" +git checkout "$ref_to_check_out" bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh diff --git a/snippets/basop/set-float-reference-branches.sh b/snippets/basop/set-float-reference-branches.sh deleted file mode 100644 index 0b12a7e..0000000 --- a/snippets/basop/set-float-reference-branches.sh +++ /dev/null @@ -1,24 +0,0 @@ -#! /bin/bash - -set -euxo pipefail - -cd "${CI_PROJECT_DIR}" - -# use ivas-float-update as reference for the merge source per default -FLOAT_REF_BRANCH_MERGE_SOURCE=$FLOAT_REF_BRANCH - -exit_code=0 -# first check if branch matches the pattern -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-follows-porting-naming-convention.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME || exit_code=$? - -# run again to actually set the branch name only if the naming pattern is there -# TODO: this is a bit awkward, but needed as a quick fix for now -if [[ "$exit_code" == "0" ]]; then - if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then - # a bit awkward: write to file + standard out first so that the error message is visible in case of failure. Then fill the variable from the file - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME | tee tmp_ref_branch.txt - FLOAT_REF_BRANCH_MERGE_SOURCE=$(cat tmp_ref_branch.txt) - fi -fi - -export FLOAT_REF_BRANCH_MERGE_SOURCE -- GitLab From b674229ad1830e703d14f42dc83aa749ef86e990 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 23 Sep 2025 10:11:41 +0200 Subject: [PATCH 09/13] for dut builds, build from commit, not from branch --- snippets/basop/build-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 624d701..d1eb687 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -32,7 +32,7 @@ merge-target) commit_file="$MERGE_TARGET_COMMIT_FILE" ;; dut) - ref_to_check_out="$CI_COMMIT_REF_NAME" + ref_to_check_out="$CI_COMMIT_SHA" ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" -- GitLab From 4a19327e9075b83ea891d656cf11370a8f65a8c0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 23 Sep 2025 11:28:51 +0200 Subject: [PATCH 10/13] try reference for needs in the regression jobs --- main-basop.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main-basop.yml b/main-basop.yml index ba820a5..dc790c0 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1138,6 +1138,7 @@ check-regressions-short-enc-0db: needs: - job: "check-be-to-target-short-enc-0db" artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] variables: TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" # +/-0dB @@ -1153,6 +1154,7 @@ check-regressions-short-enc-+10db: needs: - job: "check-be-to-target-short-enc-+10db" artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] variables: TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" # +10dB @@ -1168,6 +1170,7 @@ check-regressions-short-enc--10db: needs: - job: "check-be-to-target-short-enc--10db" artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] variables: TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" # -10dB @@ -1183,6 +1186,7 @@ check-regressions-short-dec-0db: needs: - job: "check-be-to-target-short-dec-0db" artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] variables: TEST_SUITE: "$SHORT_TEST_SUITE" # +/-0dB @@ -1198,6 +1202,7 @@ check-regressions-short-dec-+10db: needs: - job: "check-be-to-target-short-dec-+10db" artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] variables: TEST_SUITE: "$SHORT_TEST_SUITE" # +10dB @@ -1213,6 +1218,7 @@ check-regressions-short-dec--10db: needs: - job: "check-be-to-target-short-dec--10db" artifacts: true + - !reference [.test-job-linux-compares-to-fixed-target, needs] variables: TEST_SUITE: "$SHORT_TEST_SUITE" # -10dB -- GitLab From 47e4ef870f35380ea30b17e0721e015dad2d1404 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 23 Sep 2025 12:34:43 +0200 Subject: [PATCH 11/13] add printout of different git refs for all relevant jobs --- main-basop.yml | 13 +++---- .../merge-request-print-pinned-commits.sh | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 snippets/basop/merge-request-print-pinned-commits.sh diff --git a/main-basop.yml b/main-basop.yml index dc790c0..faeba10 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -381,14 +381,7 @@ workflow: - if [ "$DISABLE_HRTF" = "true" ]; then - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi - - echo "Commits from Pre job from file:" - - cat $MERGE_TARGET_COMMIT_FILE - - cat $FLOAT_REF_COMMIT_FILE - - cat $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE - - echo "Commits from Pre job from env vars:" - - echo "MERGE_TARGET_COMMIT=$MERGE_TARGET_COMMIT" - - echo "MERGE_SOURCE_FLOAT_REF_COMMIT=$MERGE_SOURCE_FLOAT_REF_COMMIT" - - echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh variables: USE_LTV: 0 @@ -1247,6 +1240,7 @@ renderer-pytest-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -1312,6 +1306,7 @@ ivas-pytest-on-merge-request: - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -1380,6 +1375,7 @@ split-rendering-pytest-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -1530,6 +1526,7 @@ ivas-interop-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES diff --git a/snippets/basop/merge-request-print-pinned-commits.sh b/snippets/basop/merge-request-print-pinned-commits.sh new file mode 100644 index 0000000..ff86427 --- /dev/null +++ b/snippets/basop/merge-request-print-pinned-commits.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +echo "====== Pinned commits in files: ======" +cat $MERGE_TARGET_COMMIT_FILE +cat $FLOAT_REF_COMMIT_FILE +cat $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE +echo "====== Pinned commits in env vars: ======" +echo "MERGE_TARGET_COMMIT=$MERGE_TARGET_COMMIT" +echo "MERGE_SOURCE_FLOAT_REF_COMMIT=$MERGE_SOURCE_FLOAT_REF_COMMIT" +echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" -- GitLab From f6a93bb5ccfcc1cc26f78560153ffccef6f679ee Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 23 Sep 2025 12:52:05 +0200 Subject: [PATCH 12/13] Revert "[revert-me] shorten pytest-based tests for faster testing" This reverts commit 8333171115541533998ce3edaddfc524c329b6e3. --- main-basop.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index faeba10..47a7b0b 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -299,7 +299,7 @@ workflow: - mv $DUT_RENDERER_PATH IVAS_rend - exit_code=0 - - python3 -m pytest -k stereo --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true ### create histograms @@ -428,11 +428,11 @@ workflow: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - exit_code_target=0 - - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? - exit_code=0 - rm -rf .pytest_cache || true - - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE @@ -571,10 +571,10 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH # Store branch outputs for later comparison @@ -586,10 +586,10 @@ workflow: ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - - python3 -m pytest -k stereo $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - - python3 -m pytest -k stereo --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions @@ -1258,13 +1258,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1328,7 +1328,7 @@ ivas-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1336,7 +1336,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=600 - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1397,7 +1397,7 @@ split-rendering-pytest-on-merge-request: # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181 - mv ISAR_post_rend ISAR_post_rend_tmp - cp $REF_POST_RENDERER_PATH ISAR_post_rend - - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - mv ISAR_post_rend_tmp ISAR_post_rend ### Run test using scripts and input from main @@ -1408,7 +1408,7 @@ split-rendering-pytest-on-merge-request: - echo "$comp_args" # run test - - python3 -m pytest -k mono -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1542,7 +1542,7 @@ ivas-interop-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest -k stereo $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1577,7 +1577,7 @@ split-rendering-smoke-test: - make -j - testcase_timeout=10 - - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1623,7 +1623,7 @@ renderer-smoke-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - make -j IVAS_rend - testcase_timeout=60 - - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1649,7 +1649,7 @@ renderer-asan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1678,7 +1678,7 @@ renderer-msan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1707,7 +1707,7 @@ renderer-usan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -k mono -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? @@ -1784,15 +1784,15 @@ renderer-usan: - make -j $make_args # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # check different renderer fremsizes only for float - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest -k stereo tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - fi - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi -- GitLab From c9521b53c11f0c562a00060a51e3b71a840e539d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 25 Sep 2025 09:42:31 +0200 Subject: [PATCH 13/13] correct printout --- snippets/basop/branch-is-up-to-date-with-target-pre.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index e3a16a7..69210f7 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -47,7 +47,7 @@ echo "MERGE_TARGET_COMMIT is $MERGE_TARGET_COMMIT" echo "$MERGE_TARGET_COMMIT" >"$MERGE_TARGET_COMMIT_FILE" MERGE_SOURCE_FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")" -echo "MERGE_SOURCEFLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT" +echo "MERGE_SOURCE_FLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT" echo "$MERGE_SOURCE_FLOAT_REF_COMMIT" >"$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" FLOAT_REF_COMMIT="$(git rev-parse "origin/ivas-float-update")" -- GitLab