From 6b041745a651413e4b97776ccf683459e0a992f6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Dec 2025 13:59:59 +0100 Subject: [PATCH 1/3] adapt scripts to new naming convention for branches --- ...ranch-follows-porting-naming-convention.sh | 10 +-- .../branch-is-up-to-date-with-target-pre.sh | 2 +- ...ranch-exists-for-porting-merge-requests.sh | 7 +- snippets/basop/get-float-ref-branch-name.sh | 65 ------------------- 4 files changed, 9 insertions(+), 75 deletions(-) delete mode 100755 snippets/basop/get-float-ref-branch-name.sh diff --git a/snippets/basop/branch-follows-porting-naming-convention.sh b/snippets/basop/branch-follows-porting-naming-convention.sh index a3811b0..3b4c035 100644 --- a/snippets/basop/branch-follows-porting-naming-convention.sh +++ b/snippets/basop/branch-follows-porting-naming-convention.sh @@ -30,16 +30,16 @@ set -euo pipefail -BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" -MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. -If this branch is porting a float MR with a corresponding float part, then +BASOP_PATTERN='^(float|basop)[_-][0-9]+[_-].+$' +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '[basop|float]--', e.g. 'basop-123-fix_this_one_bug-2'. +If this branch has a parallel companion MR in the float repo with a corresponding float part, then sticking to this naming convention is needed for the testing system to match this branch with its float-reference counterpart. Please rename your branch. You can easily do this by creating a new branch from this branch: - git checkout -b You then also need to create a new merge request and update the links in your issue. -See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec +See here for details on the naming conventions: https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/Software-development/Development-workflow#workflow-for-parallel-changes-in-fixed--and-floating-point-code -If this is not a porting branch, ignore above message. +If this branch does not have a float companion merge request, ignore above message. " if [ $# -ne 1 ]; then diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index fec2592..69c1c4e 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -37,7 +37,7 @@ bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-follows-porting-nam # if this branch follows the naming convetions, we can try to get the corresponding float MR branch # if this branch does NOT follow the porting naming convention, the float ref is just ivas-codec main, so skip everything if [[ "$exit_code_follows_naming_conventions" == "0" ]]; then - float_ref_branchname_in_ivas_codec=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME) + float_ref_branchname_in_ivas_codec=$CI_COMMIT_REF_NAME fi # fetch merge target and check if we are up-to-date diff --git a/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh b/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh index 7a4b921..04d1b61 100644 --- a/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh +++ b/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh @@ -28,7 +28,7 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -MSG_NO_FLOAT_REF_BRANCH_FOUND="Your branch name %s looks like it is a porting MR, but there is no corresponding float merge request (%s). +MSG_NO_FLOAT_REF_BRANCH_FOUND="Your branch name %s conforms to the naming convention for parallel work on float and basop, but there is no corresponding branch of the same name in ivas-codec. If this is intended (no changes on ivas-codec main), simply ignore this warning. If there should be a float ref branch for this MR, please check your spelling on your other branch and make sure that you pushed it.\n " @@ -44,11 +44,10 @@ if [ "$exit_code_is_porting_mr" == "1" ]; then fi # if this does indeed look like a porting branch, get the float companion branch - if that is the default, then warn about possible typo/missing branch -float_ref_branchname=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_COMMIT_REF_NAME) +float_ref_branchname=$CI_COMMIT_REF_NAME if [ "$float_ref_branchname" == "main" ]; then - expected_float_ref_branch="${CI_COMMIT_REF_NAME/basop/ref}" set +x - printf "$MSG_NO_FLOAT_REF_BRANCH_FOUND" "$CI_COMMIT_REF_NAME" "$expected_float_ref_branch" + printf "$MSG_NO_FLOAT_REF_BRANCH_FOUND" "$CI_COMMIT_REF_NAME" exit 123 fi diff --git a/snippets/basop/get-float-ref-branch-name.sh b/snippets/basop/get-float-ref-branch-name.sh deleted file mode 100755 index 6ba19f9..0000000 --- a/snippets/basop/get-float-ref-branch-name.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, -# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -# contributors to this repository. All Rights Reserved. -# -# This software is protected by copyright law and by international treaties. -# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, -# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., -# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, -# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other -# contributors to this repository retain full ownership rights in their respective contributions in -# the software. This notice grants no license of any kind, including but not limited to patent -# license, nor is any license granted by implication, estoppel or otherwise. -# -# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making -# contributions. -# -# This software is provided "AS IS", without any express or implied warranties. The software is in the -# development stage. It is intended exclusively for experts who have experience with such software and -# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability -# and fitness for a particular purpose are hereby disclaimed and excluded. -# -# Any dispute, controversy or claim arising under or in relation to providing this software shall be -# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in -# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and -# the United Nations Convention on Contracts on the International Sales of Goods. - -set -euo pipefail - -# in this snippet, always print stuff to stderr and ONLY the final echo with of branchname_out should go to stdout -exec 3>&1 # save stdout on file desc 3 -exec 1>&2 # send all stdout to stderr - -if [ $# -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -branchname_in="$1" -# Note: this is ivas-codec ("float") main -branchname_out="main" - -# The float ref branch is just the same name, but with basop -> ref replacement -# Replace only the first occurrence, as "basop" may be present in the later description -# If the format is correct, then before "_basop", only numbers can occur -float_ref_branchname="${branchname_in/basop/ref}" - -# now check in ivas-codec repository if a branch with this name exists -exit_code_ls_remote=0 -pushd "${SCRIPTS_DIR}" -git fetch origin -git ls-remote --exit-code --heads origin refs/heads/$float_ref_branchname || exit_code_ls_remote=$? -popd - -# If the branch exists, we use it as ref name instead of main -# NOTE: "git ls-remote --exit-code" returns 2 if the given ref was not found -if [ "$exit_code_ls_remote" -eq "0" ]; then - branchname_out="${float_ref_branchname}" -fi - -printf '%s\n' "$branchname_out" >&3 -exit 0 -- GitLab From a96844eab9d7de5ba9c6ecb96003c39140ae7354 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Dec 2025 15:08:55 +0100 Subject: [PATCH 2/3] fix/cleanup some !reference calls --- main-basop.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 3c168bc..9c4a2df 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -252,8 +252,7 @@ workflow: .test-job-linux-needs-testv-dir: extends: .test-job-linux before_script: - - !reference [ .job-linux, before_script ] - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - !reference [ .test-job-linux, before_script ] - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -273,7 +272,7 @@ workflow: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - - !reference [ .job-linux, before_script ] + - !reference [ .test-job-linux, before_script ] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -680,10 +679,9 @@ workflow: stage: test needs: ["build-codec-linux-make"] script: - - !reference [ .job-linux, before_script ] + - !reference [ .test-job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - set -euxo pipefail @@ -722,10 +720,9 @@ workflow: SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" script: - - !reference [ .job-linux, before_script ] + - !reference [ .test-job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh -- GitLab From 91b712eba1fd15c202d45e94055d84f0308b25f6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 4 Dec 2025 15:18:25 +0100 Subject: [PATCH 3/3] update redme --- README.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 60405aa..114d1a8 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,9 @@ This way, any CI change can be developed centrally in this repository and does n ### Adding new CI code 1. Implement your new CI code on a branch in this repo, e.g. `add-awesome-test` and ideally add a merge request for easier review. + +- Note: your job needs to inherit the `before_script:` section from `.test-job-linux` or `.test-job-windows`, respectively. Without that, the job will not clone this repo (i.e. ivas-codec-ci) and the snippets will be missing. If your job does not make use of any snippet, in theory you can do without this, but as a general rule, you should inherit from one of these. If your job uses a before_script: section as well, see [guidelines section below](#some-guidelines-hints). + 2. Test it by opening a branch in the respective ivas-* repository and changing IVAS_CODEC_CI_REF to your WIP branch, so the yaml there should read: ``` yaml @@ -61,6 +64,21 @@ This way, any CI change can be developed centrally in this repository and does n 3. If your change affects multiple repos/branches, you might need to use multiple test branches 4. Once you are done and review is complete, merge `add-awesome-test` to `main` here. The test branches in the other repos are not needed and SHOULD NOT BE MERGED. +### Some guidelines/hints + +- if your new job needs to overwrite the before_script: section, one can use this construct to still get the ci repo cloning: + +```yaml +before_script: + - !reference [.test-job-linux, before_script] + - ... +``` + +- use only one include section at the top of the main yml file to keep things clear, remember that later stuff overwrites earlier stuff +- in general prefer adding scripts here over using yaml anchors in a CI config file +- if your script shall modify some environment variable, you need to run it with `source` instead of calling it. +- the merge CI config in the ivas-* repos can be viewed in the pipeline editor under "full configuration" - this is especially useful when making changes which should not functionally affect the CI, then it is helpful to copy the full merged config with and without the changes and compare the files directly or as parsed yaml dict locally. + ### Overwriting CI variables in the ivas-* repos' CI configs Usually, everything should be done in the files here, but if there is the need to still overwrite some configuration/control CI variables - e.g. to switch off something temporarily or in a branch-specific manner - this should be done by in the variables: section at the top of the `.gitlab-ci.yml` file. @@ -87,10 +105,3 @@ Useful configuration variables: - `DISABLE_HRTF` - adds `-k not model` to `PYTEST_ADDOPTS` which skips the testcases with -hrtf in the selftest. Probably more for historic reasons. - `BASOP_CI_SCRIPTS_BRANCH` - used to change the branch from which testscripts are pulled, see [here](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/Software-development/Test-Setup-For-BASOP-Code#how-do-i-check-that-my-new-testcase-does-work-for-basop) for more info. - -### Some guidelines/hints - -- use only one include section at the top of the main yml file to keep things clear, remember that later stuff overwrites earlier stuff -- in general prefer adding scripts here over using yaml anchors in a CI config file -- if your script shall modify some environment variable, you need to run it with `source` instead of calling it. -- the merge CI config in the ivas-* repos can be viewed in the pipeline editor under "full configuration" - this is especially useful when making changes which should not functionally affect the CI, then it is helpful to copy the full merged config with and without the changes and compare the files directly or as parsed yaml dict locally. -- GitLab