From 485aa1f15a0c36626c489b9343f0665f004fea24 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 24 Jul 2025 15:26:27 +0200 Subject: [PATCH 01/30] add build-merge-target-binaries.sh scripadd --- snippets/basop/build-merge-target-binaries.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 snippets/basop/build-merge-target-binaries.sh diff --git a/snippets/basop/build-merge-target-binaries.sh b/snippets/basop/build-merge-target-binaries.sh new file mode 100644 index 0000000..6698350 --- /dev/null +++ b/snippets/basop/build-merge-target-binaries.sh @@ -0,0 +1,19 @@ +#! /bin/bash + +set -euxo pipefail + +cd "${CI_PROJECT_DIR}" + +if [ -f $MERGE_TARGET_COMMIT_FILE ]; then + MERGE_REQUEST_TARGET="$(cat "$MERGE_TARGET_COMMIT_FILE")" +else + echo "MERGE_TARGET_COMMIT_FILE ($MERGE_TARGET_COMMIT_FILE) does not exist, exiting!" + exit 1 +fi + +# overwrite CI variable temporarily so we can use the other script +CI_MERGE_REQUEST_TARGET_BRANCH_NAME_tmp=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME +CI_MERGE_REQUEST_TARGET_BRANCH_NAME=$MERGE_REQUEST_TARGET + +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh +bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target -- GitLab From df140fb22186365fde5d95167663668274354ada Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 24 Jul 2025 16:32:32 +0200 Subject: [PATCH 02/30] use checkout instead of switch --detach --- includes/job-templates.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/includes/job-templates.yml b/includes/job-templates.yml index 813714c..6d0740b 100644 --- a/includes/job-templates.yml +++ b/includes/job-templates.yml @@ -7,9 +7,10 @@ - ivas-linux before_script: - | - if [[ "$IVAS_CODEC_CI_REF" == "" ]]; then echo "Error: IVAS_CODEC_CI_REF not set" && exit 1; fi + if [[ "$IVAS_CODEC_CI_REF" == "" ]]; then echo "Error: IVAS_CODEC_CI_REF not set" && exit 1; else echo "IVAS_CODEC_CI_REF set to: $IVAS_CODEC_CI_REF"; fi # note: git clone -b does not work with SHA refs, hence using the safe way of a full clone and git switch - - git clone "$CI_SERVER_URL"/ivas-codec-pc/ivas-codec-ci.git && pushd ivas-codec-ci && git switch --detach "$IVAS_CODEC_CI_REF" && popd + # NOTE: set CUSTOM_CI_REPO_CLONE_URL in your gitlab setup + - git clone "$CUSTOM_CI_REPO_CLONE_URL" ivas-codec-ci && pushd ivas-codec-ci && git checkout "$IVAS_CODEC_CI_REF" && popd .job-windows: tags: @@ -19,7 +20,8 @@ if (-not $env:IVAS_CODEC_CI_REF) { echo "Error: IVAS_CODEC_CI_REF not set" ; exit 1 } # note: git clone -b does not work with SHA refs, hence using the safe way of a full clone and git switch # also, we cannot use && with powershell < 7 unfortunately, hence we cannot reliably detect if the snippet fails - - git clone $env:CI_SERVER_URL/ivas-codec-pc/ivas-codec-ci.git; pushd ivas-codec-ci ; git switch --detach $env:IVAS_CODEC_CI_REF ; popd + # NOTE: set CUSTOM_CI_REPO_CLONE_URL in your gitlab setup + - git clone $env:CUSTOM_CI_REPO_CLONE_URL ivas-codec-ci; pushd ivas-codec-ci ; git switch --detach $env:IVAS_CODEC_CI_REF ; popd .build-job-linux: stage: build -- GitLab From ee29ff0ecb4bda684f119e8c9d4d2b8805e8fb72 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 24 Jul 2025 16:37:32 +0200 Subject: [PATCH 03/30] remove unintentionally committed part --- includes/job-templates.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/includes/job-templates.yml b/includes/job-templates.yml index 6d0740b..87c58f1 100644 --- a/includes/job-templates.yml +++ b/includes/job-templates.yml @@ -9,8 +9,7 @@ - | if [[ "$IVAS_CODEC_CI_REF" == "" ]]; then echo "Error: IVAS_CODEC_CI_REF not set" && exit 1; else echo "IVAS_CODEC_CI_REF set to: $IVAS_CODEC_CI_REF"; fi # note: git clone -b does not work with SHA refs, hence using the safe way of a full clone and git switch - # NOTE: set CUSTOM_CI_REPO_CLONE_URL in your gitlab setup - - git clone "$CUSTOM_CI_REPO_CLONE_URL" ivas-codec-ci && pushd ivas-codec-ci && git checkout "$IVAS_CODEC_CI_REF" && popd + - git clone "$CI_SERVER_URL"/ivas-codec-pc/ivas-codec-ci.git ivas-codec-ci && pushd ivas-codec-ci && git checkout "$IVAS_CODEC_CI_REF" && popd .job-windows: tags: @@ -20,8 +19,7 @@ if (-not $env:IVAS_CODEC_CI_REF) { echo "Error: IVAS_CODEC_CI_REF not set" ; exit 1 } # note: git clone -b does not work with SHA refs, hence using the safe way of a full clone and git switch # also, we cannot use && with powershell < 7 unfortunately, hence we cannot reliably detect if the snippet fails - # NOTE: set CUSTOM_CI_REPO_CLONE_URL in your gitlab setup - - git clone $env:CUSTOM_CI_REPO_CLONE_URL ivas-codec-ci; pushd ivas-codec-ci ; git switch --detach $env:IVAS_CODEC_CI_REF ; popd + - git clone $env:CI_SERVER_URL/ivas-codec-pc/ivas-codec-ci.git ivas-codec-ci; pushd ivas-codec-ci ; git switch --detach $env:IVAS_CODEC_CI_REF ; popd .build-job-linux: stage: build -- GitLab From a24c577371c6b0667351fe659d5827f3f2ac597d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 24 Jul 2025 18:02:57 +0200 Subject: [PATCH 04/30] also rename post renderer executable --- snippets/basop/build-binaries.sh | 43 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index f1f8d81..d9732b6 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -7,31 +7,33 @@ cd "${CI_PROJECT_DIR}" mode="${1:-}" case "$mode" in - float-ref) - branch_to_check_out="$FLOAT_REF_BRANCH" - ivas_cod_filename="./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" - ivas_dec_filename="./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" - ivas_rend_filename="./IVAS_rend_ref" - commit_file="$FLOAT_REF_COMMIT_FILE" - ;; - merge-target) - branch_to_check_out="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" - ivas_cod_filename="./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" - ivas_dec_filename="./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" - ivas_rend_filename="./IVAS_rend_merge_target" - commit_file="$MERGE_TARGET_COMMIT_FILE" - ;; - *) - echo "Error: invalid mode: $mode" - exit 2 - ;; +float-ref) + branch_to_check_out="$FLOAT_REF_BRANCH" + ivas_cod_filename="./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_dec_filename="./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_rend_filename="./$REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + isar_post_rend_filename="./$REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + commit_file="$FLOAT_REF_COMMIT_FILE" + ;; +merge-target) + branch_to_check_out="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" + ivas_cod_filename="./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_dec_filename="./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_rend_filename="./$MERGE_TARGET_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + isar_post_rend_filename="./$MERGE_TARGET_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + commit_file="$MERGE_TARGET_COMMIT_FILE" + ;; +*) + echo "Error: invalid mode: $mode" + exit 2 + ;; esac echo "Building $mode binaries" # TODO: could probably always be run if [[ "$mode" == "float-ref" ]]; then - git rev-parse HEAD > "$CUT_COMMIT_FILE" + git rev-parse HEAD >"$CUT_COMMIT_FILE" fi current_commit_sha="$(git rev-parse HEAD)" @@ -57,8 +59,9 @@ make -j "$(nproc)" mv IVAS_cod "$ivas_cod_filename" mv IVAS_dec "$ivas_dec_filename" mv IVAS_rend "$ivas_rend_filename" +mv isar_post_rend "$isar_post_rend_filename" # return to current branch git restore . -git rev-parse HEAD > "$commit_file" +git rev-parse HEAD >"$commit_file" git checkout "$current_commit_sha" -- GitLab From 962984e9303847e3987d40768f7771bae3a1c948 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 24 Jul 2025 18:15:59 +0200 Subject: [PATCH 05/30] fix case --- snippets/basop/build-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index d9732b6..ee04e29 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -59,7 +59,7 @@ make -j "$(nproc)" mv IVAS_cod "$ivas_cod_filename" mv IVAS_dec "$ivas_dec_filename" mv IVAS_rend "$ivas_rend_filename" -mv isar_post_rend "$isar_post_rend_filename" +mv ISAR_post_rend "$isar_post_rend_filename" # return to current branch git restore . -- GitLab From 251e3e4402074f306979508b591d19c606ad1f6f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 09:51:24 +0200 Subject: [PATCH 06/30] temporary change for main-pc merge only rename post renderer if it is there --- snippets/basop/build-binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index ee04e29..3f4f3f4 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -59,7 +59,7 @@ make -j "$(nproc)" mv IVAS_cod "$ivas_cod_filename" mv IVAS_dec "$ivas_dec_filename" mv IVAS_rend "$ivas_rend_filename" -mv ISAR_post_rend "$isar_post_rend_filename" +if [ -f ISAR_post_rend ]; then mv ISAR_post_rend "$isar_post_rend_filename"; fi # return to current branch git restore . -- GitLab From 915667f65af84a5e6757ceb1e3f9c90d703984f7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 11:01:07 +0200 Subject: [PATCH 07/30] add new modes to build-binaries script --- snippets/basop/build-binaries.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 3f4f3f4..e77a71f 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -15,6 +15,14 @@ float-ref) isar_post_rend_filename="./$REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$FLOAT_REF_COMMIT_FILE" ;; +float-ref-merge-source) + branch_to_check_out="$FLOAT_REF_BRANCH_MERGE_SOURCE" + ivas_cod_filename="./$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_dec_filename="./$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_rend_filename="./$MERGE_SOURCE_FLOAT_REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + isar_post_rend_filename="./$MERGE_SOURCE_FLOAT_REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + commit_file="$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" + ;; merge-target) branch_to_check_out="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ivas_cod_filename="./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -23,6 +31,13 @@ merge-target) isar_post_rend_filename="./$MERGE_TARGET_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$MERGE_TARGET_COMMIT_FILE" ;; +merge-source) + branch_to_check_out="" + ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" + commit_file="$CUT_COMMIT_FILE" *) echo "Error: invalid mode: $mode" exit 2 @@ -49,7 +64,9 @@ trap restore_stash EXIT # should never fail, even if there is nothing to be stored git stash -git checkout "$branch_to_check_out" +if [[ "$mode" != "merge-source" ]]; then + git checkout "$branch_to_check_out" +fi bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh -- GitLab From 182bf8d35a667a43a09fefbf3a68c02f2b792b4b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 11:16:00 +0200 Subject: [PATCH 08/30] convert ref branch setting anchor to script --- .../basop/set-float-reference-branches.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 snippets/basop/set-float-reference-branches.sh diff --git a/snippets/basop/set-float-reference-branches.sh b/snippets/basop/set-float-reference-branches.sh new file mode 100644 index 0000000..8cb3ac1 --- /dev/null +++ b/snippets/basop/set-float-reference-branches.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +set -euxo pipefail + +cd "${CI_PROJECT_DIR}" + +# use ivas-float-update as reference for the merge source per default +FLOAT_REF_BRANCH_MERGE_SOURCE=$FLOAT_REF_BRANCH + +# for porting MRs use the name-check script +if [ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME = "main" -a "${CI_MERGE_REQUEST_TITLE,,}" =~ \[PORTING\] ]; then + if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then + bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + # a bit awkward: write to file + standard out first so that the error message is visible in case of failure. Then fill the variable from the file + ci/get_float_ref_branch_name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME | tee tmp_ref_branch.txt + FLOAT_REF_BRANCH_MERGE_SOURCE=$(cat tmp_ref_branch.txt) + fi +fi + +# TODO: remove after merge to main +FLOAT_REF_BRANCH_MERGE_SOURCE=float-pc -- GitLab From 908bdd6d2495aab5c3237f4c00ca01911d7d3a41 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 11:47:00 +0200 Subject: [PATCH 09/30] fix syntax --- snippets/basop/build-binaries.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index e77a71f..4ea0a44 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -38,6 +38,7 @@ merge-source) ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$CUT_COMMIT_FILE" + ;; *) echo "Error: invalid mode: $mode" exit 2 -- GitLab From d5c615c873150ec6a5036c12caa6509be410d443 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 12:03:09 +0200 Subject: [PATCH 10/30] add if in build-binaries.sh --- snippets/basop/build-binaries.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 4ea0a44..c43ace6 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -74,10 +74,13 @@ bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-i make clean make -j "$(nproc)" -mv IVAS_cod "$ivas_cod_filename" -mv IVAS_dec "$ivas_dec_filename" -mv IVAS_rend "$ivas_rend_filename" -if [ -f ISAR_post_rend ]; then mv ISAR_post_rend "$isar_post_rend_filename"; fi +# avoid errors in mv when renaming to same file +if [[ "$mode" != "merge-source" ]]; then + mv IVAS_cod "$ivas_cod_filename" + mv IVAS_dec "$ivas_dec_filename" + mv IVAS_rend "$ivas_rend_filename" + if [ -f ISAR_post_rend ]; then mv ISAR_post_rend "$isar_post_rend_filename"; fi +fi # return to current branch git restore . -- GitLab From 5057e57ded928f37895a800c63e2c919cf7908bd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 12:50:07 +0200 Subject: [PATCH 11/30] fixes for float ref setting and build scripts --- snippets/basop/build-binaries.sh | 2 +- snippets/basop/set-float-reference-branches.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index c43ace6..682fae0 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -32,7 +32,7 @@ merge-target) commit_file="$MERGE_TARGET_COMMIT_FILE" ;; merge-source) - branch_to_check_out="" + branch_to_check_out="$FLOAT_REF_BRANCH_MERGE_SOURCE" ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" diff --git a/snippets/basop/set-float-reference-branches.sh b/snippets/basop/set-float-reference-branches.sh index 8cb3ac1..bd284c3 100644 --- a/snippets/basop/set-float-reference-branches.sh +++ b/snippets/basop/set-float-reference-branches.sh @@ -8,7 +8,7 @@ cd "${CI_PROJECT_DIR}" FLOAT_REF_BRANCH_MERGE_SOURCE=$FLOAT_REF_BRANCH # for porting MRs use the name-check script -if [ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME = "main" -a "${CI_MERGE_REQUEST_TITLE,,}" =~ \[PORTING\] ]; then +if [[ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME = "main" ]] && [[ "${CI_MERGE_REQUEST_TITLE,,}" =~ \[porting\] ]]; then if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh # a bit awkward: write to file + standard out first so that the error message is visible in case of failure. Then fill the variable from the file @@ -19,3 +19,5 @@ fi # TODO: remove after merge to main FLOAT_REF_BRANCH_MERGE_SOURCE=float-pc + +export FLOAT_REF_BRANCH_MERGE_SOURCE -- GitLab From f7d58d145ff06af6b6adb8cd95c9c1d91e6e7631 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 13:08:23 +0200 Subject: [PATCH 12/30] fix branch in build script --- snippets/basop/build-binaries.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 682fae0..8bd798f 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -32,7 +32,7 @@ merge-target) commit_file="$MERGE_TARGET_COMMIT_FILE" ;; merge-source) - branch_to_check_out="$FLOAT_REF_BRANCH_MERGE_SOURCE" + branch_to_check_out="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -45,7 +45,7 @@ merge-source) ;; esac -echo "Building $mode binaries" +echo "Building $mode binaries from branch $branch_to_check_out" # TODO: could probably always be run if [[ "$mode" == "float-ref" ]]; then -- GitLab From b679d76d6aed506e9f4e200e1aa61ccb67d6c4d0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 13:32:54 +0200 Subject: [PATCH 13/30] fix setting of variables in build script --- snippets/basop/build-merge-target-binaries.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/snippets/basop/build-merge-target-binaries.sh b/snippets/basop/build-merge-target-binaries.sh index 6698350..26d4cd8 100644 --- a/snippets/basop/build-merge-target-binaries.sh +++ b/snippets/basop/build-merge-target-binaries.sh @@ -13,7 +13,8 @@ fi # overwrite CI variable temporarily so we can use the other script CI_MERGE_REQUEST_TARGET_BRANCH_NAME_tmp=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -CI_MERGE_REQUEST_TARGET_BRANCH_NAME=$MERGE_REQUEST_TARGET +export CI_MERGE_REQUEST_TARGET_BRANCH_NAME=$MERGE_REQUEST_TARGET -bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target + +export CI_MERGE_REQUEST_TARGET_BRANCH_NAME=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME_tmp -- GitLab From 0ac1f0a193a60785f20016593d939e1cb3883504 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 25 Jul 2025 14:10:33 +0200 Subject: [PATCH 14/30] enable renaming again with new default names --- snippets/basop/build-binaries.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 8bd798f..67c07f9 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -75,12 +75,10 @@ make clean make -j "$(nproc)" # avoid errors in mv when renaming to same file -if [[ "$mode" != "merge-source" ]]; then - mv IVAS_cod "$ivas_cod_filename" - mv IVAS_dec "$ivas_dec_filename" - mv IVAS_rend "$ivas_rend_filename" - if [ -f ISAR_post_rend ]; then mv ISAR_post_rend "$isar_post_rend_filename"; fi -fi +mv IVAS_cod "$ivas_cod_filename" +mv IVAS_dec "$ivas_dec_filename" +mv IVAS_rend "$ivas_rend_filename" +if [ -f ISAR_post_rend ]; then mv ISAR_post_rend "$isar_post_rend_filename"; fi # return to current branch git restore . -- GitLab From 0ec54eb707005336f6fd6c5e571d40fe22550ae7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 28 Jul 2025 12:12:09 +0200 Subject: [PATCH 15/30] add CI file from basop here --- main-basop.yml | 2976 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2976 insertions(+) create mode 100644 main-basop.yml diff --git a/main-basop.yml b/main-basop.yml new file mode 100644 index 0000000..029eb06 --- /dev/null +++ b/main-basop.yml @@ -0,0 +1,2976 @@ +include: + - local: .gitlab-ci/variables.yml + - local: .gitlab-ci/rules-basis.yml + - local: main.yml + +default: + interruptible: true # Make all jobs by default interruptible + +workflow: + name: '$IVAS_PIPELINE_NAME' + rules: + # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + variables: + IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow + # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main + # variables: + # IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing + variables: + IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs' + variables: + IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' + variables: + IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx' + variables: + IVAS_PIPELINE_NAME: 'Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long-fx-fx' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' + variables: + IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' + variables: + IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer' + variables: + IVAS_PIPELINE_NAME: 'Short testvectors sanitizers' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer' + variables: + IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity' + variables: + IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test' + variables: + IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'renderer-framesize-be' + variables: + IVAS_PIPELINE_NAME: 'Renderer framesize BE test on $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' + variables: + IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch + variables: + IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + +# --------------------------------------------------------------- +# Generic script anchors +# --------------------------------------------------------------- + +# These can be used later on to do common tasks + +# Prints useful information for every job and should be used at the beginning of each job +.print-common-info-windows: &print-common-info-windows + - | + echo "Printing common information for build job." + echo "Current job is run on commit $CI_COMMIT_SHA" + echo "Commit time was $CI_COMMIT_TIMESTAMP" + ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression + +.activate-Werror-linux: &activate-Werror-linux + - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile + - sed -i.bak "s/# \(set(CMAKE_C_FLAGS \"\${CMAKE_C_FLAGS} -Werror\")\)/\1/" CMakeLists.txt + +.activate-WX-windows: &activate-WX-windows + - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" + - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } + +# From float CI +.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh + + # need to rename to match the conventions for the renderer tests... + - mv $MERGE_TARGET_ENCODER_PATH $REF_ENCODER_PATH + - mv $MERGE_TARGET_DECODER_PATH $REF_DECODER_PATH + - mv $MERGE_TARGET_RENDERER_PATH $REF_RENDERER_PATH + # NOTE: temporary solution for merge to main + - if [ -f $MERGE_TARGET_POST_RENDERER_PATH ]; then mv $MERGE_TARGET_POST_RENDERER_PATH $REF_POST_RENDERER_PATH; fi + + - make clean + - make -j + +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + +.rules-merge-request: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - when: never + +.rules-merge-request-no-draft: + extends: .rules-basis + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - when: never + +.rules-merge-request-to-main: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-float-ref: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "ivas-float-update" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-main-no-draft: + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-float-ref-no-draft: + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "ivas-float-update" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-pytest-to-ref-short: + rules: + - if: $PYTEST_MLD_SHORT # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-pytest-to-input-short: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-to-input" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + +.rules-pytest-to-ref-enc-short-dmx: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-enc-dmx" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + +.rules-pytest-long: + rules: + - if: $PYTEST_MLD_LONG # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-long" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + +.rules-pytest-long-fx-fx: + rules: + - if: $PYTEST_MLD_LONG_FX_FX # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-long-fx-fx" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + +.rules-coverage: + rules: + - if: $COVERAGE_TEST # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "coverage" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + +.rules-pytest-saturation-smoke-test: + rules: + - if: $PYTEST_SMOKE_TEST # Set by scheduled pipeline + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-saturation-smoke-test" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + +# --------------------------------------------------------------- +# Job templates +# --------------------------------------------------------------- + +# override for centrally defined job to make use of the basop runners +.job-linux: + tags: + - ivas-basop-linux + +# custom variant of this template, we need to update the scripts repo before every build +.test-job-linux: + extends: + - .job-linux + before_script: + - !reference [.job-linux, before_script] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + +.test-job-linux-compares-to-fixed-target: + extends: .test-job-linux + needs: + - job: branch-is-up-to-date-with-target-pre + artifacts: true + - job: build-codec-linux-make + +.print-results-banner: &print-results-banner + - set +x + - echo "" + - echo -e "==================================================================================================================\n================================================== TEST RESULTS ==================================================\n==================================================================================================================\n" + +# template for test jobs on linux that need the TESTV_DIR +.test-job-linux-needs-testv-dir: + extends: .test-job-linux + before_script: + - !reference [ .job-linux, before_script ] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi + - cp -r scripts/testv/* $TESTV_DIR/ + +.ivas-pytest-anchor: &ivas-pytest-anchor + stage: test + needs: ["build-codec-linux-make"] + timeout: "360 minutes" + variables: + # keep "mld" in artifact name for backwards compatibility reasons + CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" + MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" + PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" + SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html" + IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" + script: + - !reference [ .job-linux, before_script ] + + - set -euxo pipefail + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - if [ $USE_LTV -eq 1 ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + - if [ $LEVEL_SCALING != "1.0" ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - fi + + - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then + - BUILD_WITH_DEBUG_MODE_INFO="true" + - fi + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh + + - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") + - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" + - summary_args="MLD DIFF SSNR ODG" + - REPORT_ARG="" + - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi + - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi + - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh + + # DMX comparison only in manual job with no other metrics + - if [ "$COMPARE_DMX" = "true" ]; then + - comp_args="--compare_enc_dmx" + - fi + + - echo "$comp_args" + + ### run pytest + - exit_code=0 + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + ### create histograms + - if [ "$SPLIT_COMPARISON" = "true" ]; then + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT + - else + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME + - fi + + # first for "whole" files comparison + - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + + - if [ "$SPLIT_COMPARISON" = "true" ]; then + - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT + - else + # touch files to suppress warning for missing artifacts + - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT + - fi + + - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts + # This wildcard thingy relies on only one csv file being present per job + - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" + - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME + - else + # create empty file for artifacts to avoid errors + - touch $PAGES_HTML_ARTIFACT_NAME + - touch $MERGED_CSV_ARTIFACT_NAME + - fi + + - if [ $zero_errors != 1 ]; then + - echo "Run errors encountered!" + # TODO: temporary only to not fail MR pipelines on crashes + - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then + - exit $EXIT_CODE_NON_BE + - else + - exit $EXIT_CODE_FAIL + - fi + - fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - $PAGES_HTML_ARTIFACT_NAME + - $CSV_ARTIFACT_NAME + - $CSV_ARTIFACT_SPLIT + - $MERGED_CSV_ARTIFACT_NAME + - $SUMMARY_HTML_ARTIFACT_NAME + - $SUMMARY_HTML_ARTIFACT_SPLIT + - $IMAGES_ARTIFACT_NAME + - $IMAGES_ARTIFACT_SPLIT + expose_as: "pytest compare results" + reports: + junit: + - report-junit.xml + +.check-be-job: + extends: + - .test-job-linux + - .rules-merge-request-to-main-no-draft + - .test-job-linux-compares-to-fixed-target + before_script: + - !reference [ .test-job-linux, before_script ] + - rm -rf tests/dut tests/ref + variables: + USE_LTV: 0 + +.check-be-to-target-job: + extends: + - .check-be-job + stage: check-be + timeout: "300 minutes" + variables: + XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + PYTEST_LOG_TARGET_BRANCH: "pytest-log-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME.txt" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + + - set -euxo pipefail + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - python3 tests/create_short_testvectors.py + + - if [ $USE_LTV -eq 1 ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + - python3 scripts/prepare_combined_format_inputs.py + + - if [ $LEVEL_SCALING != "1.0" ];then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - fi + + # set branches for reference builds + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/set-float-reference-branches.sh + - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + # build merge target branch and correpsonding float reference + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh + - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + # build to-be-merged branch and corresponding float ref branch + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source + + - exit_code_target=0 + - python3 -m pytest $TEST_SUITE -k "not model" -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + + - exit_code=0 + - rm -rf .pytest_cache || true + - python3 -m pytest --tb=no -q $TEST_SUITE -k "not model" -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + + - if [ $exit_code -ne 0 ]; then + - exit_code=$EXIT_CODE_NON_BE + - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true + - num_failures=$(wc -l < failed_testcases_for_printing.txt) + + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true + - num_errors=$(wc -l < errors_testcases_for_printing.txt) + + - *print-results-banner + - echo "Found these $num_failures non-bitexact testcases:" + - cat failed_testcases_for_printing.txt + + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the html report in the job artifacts." + - if [ $num_errors -ne 0 ]; then + - exit_code=1 + - echo "There were errors present in the following testcases:" + - cat errors_testcases_for_printing.txt + - fi + + - exit $exit_code + - else + # create empty files to not have errors at artifact stage + - touch $FAILED_TESTCASES_LIST + - touch $ERRORS_TESTCASES_LIST + - touch $PYTEST_CACHE_ARTIFACT + - *print-results-banner + - echo "All testcases are bitexact." + - fi + - exit $exit_code + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT + - $HTML_REPORT + - $FAILED_TESTCASES_LIST + - $ERRORS_TESTCASES_LIST + - pytest_log.txt + - $PYTEST_CACHE_ARTIFACT + - $FLOAT_REF_COMMIT_FILE + - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE + - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE + - $PYTEST_LOG_TARGET_BRANCH + expose_as: "pytest compare results" + reports: + junit: + - $XML_REPORT + - $XML_REPORT + +.overwrite-pytest-cache-with-artifact: &overwrite-pytest-cache-with-artifact + - if [ -f $PYTEST_CACHE_ARTIFACT ]; then + - rm -rf .pytest_cache || true + - unzip $PYTEST_CACHE_ARTIFACT + - fi + +.check-regressions-pytest-job: + extends: + - .check-be-job + stage: test + timeout: "300 minutes" + variables: + XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + HTML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + CSV_BRANCH: "scores-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + CSV_MAIN: "scores-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv" + IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + + # create empty files for all artifacts to suppress warnings in case of no regressions found or all is BE + - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAX_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAX_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv + - mkdir $IMAGES_ARTIFACT_NAME + + - set -euxo pipefail + + - if [ -s $FAILED_TESTCASES_LIST ]; then + - *overwrite-pytest-cache-with-artifact + - export PYTEST_ADDOPTS=--last-failed + - else + # turn off echoing back of commands for result printout + - *print-results-banner + - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." + - exit 0 + - fi + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + - if [ $USE_LTV -eq 1 ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + - if [ $LEVEL_SCALING != "1.0" ];then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - fi + + # check MR title for flag that allows regressions to be mergable + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) || true + + - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") + - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" + + # set branches for reference builds + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/set-float-reference-branches.sh + # build merge target branch and correpsonding float reference + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target + # build to-be-merged branch and corresponding float ref branch + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source + + # set pytest args for ref creation + - enc_stats_arg="" + - if [ "$ENCODER_TEST" = "true" ]; then + - enc_stats_arg="--enc_stats" + - fi + - enc_dmx_arg="" + - if [ "$COMPARE_DMX" = "true" ]; then + - enc_dmx_arg="--compare_enc_dmx" + - fi + + ### ----- run pytest for branch (dut) first ----- + # create float reference outputs with corresponding branch + - python3 -m pytest $TEST_SUITE -k "not model" -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - *overwrite-pytest-cache-with-artifact + # create dut outputs + - python3 -m pytest --tb=no -q $TEST_SUITE -k "not model" -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH + # Store branch outputs for later comparison + - mv tests/dut tests/dut_branch + # create the summary based on the branch only + - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + - *overwrite-pytest-cache-with-artifact + + ### ----- run pytest for merge target now ----- + # create float reference outputs with corresponding branch + - python3 -m pytest $TEST_SUITE -k "not model" -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - *overwrite-pytest-cache-with-artifact + # create merge-target outputs + - python3 -m pytest --tb=no -q $TEST_SUITE -k "not model" -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN + + ### compare the two csv files for regressions + - regressions_found=0 + - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_BRANCH $CSV_MAIN > regression_log.txt || regressions_found=$? + + - exit_code=0 + - *print-results-banner + - if [ $zero_errors_branch != 1 ]; then + - echo "Run errors encountered!" + - exit_code=$EXIT_CODE_FAIL + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the regressions_crashes.csv files in the job artifacts." + - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then + - cat regression_log.txt + - if [ $allow_regressions_flag == 0 ]; then + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" + - exit_code=$EXIT_CODE_FAIL; + - else + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." + - exit_code=$EXIT_CODE_NON_BE; + - fi + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the regressions_*.csv files in the job artifacts." + - fi + + - exit $exit_code + after_script: + - rm -rf tests/dut tests/ref + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + - $HTML_REPORT_BRANCH + - $HTML_REPORT_MAIN + - $CSV_BRANCH + - $CSV_MAIN + - $SUMMARY_HTML_ARTIFACT_NAME + - $IMAGES_ARTIFACT_NAME + - $FLOAT_REF_COMMIT_FILE + - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE + - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE + - regressions_crashes.csv + - regressions_MLD.csv + - regressions_MAX_ABS_DIFF.csv + - regressions_MIN_SSNR.csv + - regressions_MIN_ODG.csv + - improvements_crashes.csv + - improvements_MLD.csv + - improvements_MAX_ABS_DIFF.csv + - improvements_MIN_SSNR.csv + - improvements_MIN_ODG.csv + expose_as: "pytest compare results" + reports: + junit: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + + +.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor + extends: + - .job-linux + stage: test + needs: ["build-codec-linux-make"] + timeout: "600 minutes" + rules: + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + script: + - !reference [ .job-linux, before_script ] + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + + - set -euxo pipefail + - make_args="CLANG=$CLANG_NUM" + - if [[ $CLANG_NUM == 3 ]]; then + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" + - fi + - make clean + - make -j $make_args >> /dev/null + - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS" + # disable per-testcase timeout for msan to evaluate what is going on that it takes so long + - if [[ $CLANG_NUM = 1 ]]; then + - testcase_timeout_arg="" + - fi + - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: "2 weeks" + paths: + - report-junit.xml + - report.html + reports: + junit: + - report-junit.xml + +.ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor + stage: test + needs: ["build-codec-linux-make"] + timeout: "360 minutes" + variables: + SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" + IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" + script: + - !reference [ .job-linux, before_script ] + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - if [ $USE_LTV -eq 1 ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + - if [ $LEVEL_SCALING != "1.0" ];then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - fi + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh + + - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") + - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" + + ### run pytest + - exit_code=0 + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? + - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true + - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true + - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv + - python3 scripts/parse_xml_report.py report-junit-dut.xml report-dut.csv + - python3 scripts/diff_report.py report-ref.csv report-dut.csv report-diff.csv + + # create summary + - mkdir $IMAGES_ARTIFACT_NAME + - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT + + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - report-junit-ref.xml + - report-ref.html + - report-ref.csv + - report-junit-dut.xml + - report-dut.html + - report-dut.csv + - report-diff.csv + - $IMAGES_ARTIFACT_NAME + - $SUMMARY_HTML_ARTIFACT_NAME + expose_as: "pytest compare to input results" + reports: + junit: + - report-junit-ref.xml + - report-junit-dut.xml + +# --------------------------------------------------------------- +# .pre jobs for setting up things +# --------------------------------------------------------------- + +# See: https://gitlab.com/gitlab-org/gitlab/-/issues/194023 +# Solution to make main branch pipelines uninterruptible while all other +# pipelines can be interrupted by default. This works because all jobs +# after uninterruptible jobs will be uninterruptible. Resource group +# setting avoids rare case where two fast merges could still interrupt +# pipeline. This should be revisited if there are updates to Gitlab. +uninterruptible: + stage: .pre + interruptible: false + resource_group: uninterruptible + script: + - echo "$CI_COMMIT_BRANCH is uninterruptible" + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + tags: + - ivas-basop-linux + +# --------------------------------------------------------------- +# verification jobs +# --------------------------------------------------------------- + +# fail pipeline in the final stage for pipelines on Draft MRs +# this also only runs on Draft MRs, so should always fail +fail-pipeline-if-in-draft: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + stage: postvalidate + tags: + - ivas-linux + script: + - echo "Your MR is still in Draft state, set it to ready to be mergable, then retrigger the pipeline." + - exit 1 + +# this branch runs on merges to main-pc only and will fail if the branch itself does not conform to the naming conventions +check-naming-of-branch-for-basop-update-merges: + extends: + - .rules-merge-request-to-main + - .job-linux + stage: prevalidate + tags: + - ivas-linux + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then + - ci/get_float_ref_branch_name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME + - fi + +branch-is-up-to-date-with-target-pre: + extends: + - .job-linux + - .rules-merge-request + stage: prevalidate + needs: [] + tags: + - ivas-basop-linux + script: + - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" + - TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")" + - echo "TARGET_COMMIT is $TARGET_COMMIT" + - echo "$TARGET_COMMIT" > "$MERGE_TARGET_COMMIT_FILE" + artifacts: + paths: + - $MERGE_TARGET_COMMIT_FILE + +branch-is-up-to-date-with-target-post: + extends: + - .job-linux + - .rules-merge-request-no-draft + stage: postvalidate + tags: + - ivas-basop-linux + script: + - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 123 + fi + allow_failure: + exit_codes: + - 123 + +clang-format-check: + extends: + - .test-job-linux + - .rules-merge-request + variables: + ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" + stage: prevalidate + needs: [] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there + - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch + - > + INSTRUCTIONS_GITLAB="To fix formatting issues:\n + - download the diff patch available as artifact of this job\n + - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n + - run: git apply $PATCH_FILE_NAME\n + - commit new changes" + - > + INSTRUCTIONS_README="To fix formatting issues:\n + - place the patch file in the root directory of your local IVAS repo\n + - run: git apply $PATCH_FILE_NAME\n + - commit new changes" + + - format_problems=0 + - scripts/check-format.sh -afD -p 8 || format_problems=$? + - if [ $format_problems == 0 ] ; then exit 0; fi + + - mkdir tmp-formatting-fix + - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME" + + # Print instructions to job output + - echo -e "$INSTRUCTIONS_GITLAB" + + # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface) + - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt" + + - exit $format_problems + artifacts: + expire_in: 1 day + paths: + - tmp-formatting-fix/ + when: on_failure + name: "$ARTIFACT_BASE_NAME" + expose_as: "formatting patch" + +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- + +# TODO: template rules for the build jobs +# ensure that codec builds on linux +build-codec-linux-make: + rules: + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + timeout: "100 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - *activate-Werror-linux + - make -j + +build-codec-linux-cmake: + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - cmake -B cmake-build -G "Unix Makefiles" + - cmake --build cmake-build -- -j + +# ensure that codec builds on linux with instrumentation active +build-codec-linux-instrumented-make: + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + timeout: "100 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash scripts/prepare_instrumentation.sh -m MEM_ONLY + - make -j -C $INSTR_DIR + +# make sure that the codec builds with msan, asan and usan +build-codec-sanitizers-linux: + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + timeout: "100 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + # TODO: re-enable once all the warnings have been fixed + #- *activate-Werror-linux + - bash ci/build_codec_sanitizers_linux.sh + +build-codec-linux-debugging-make: + rules: + - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + when: never + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + timeout: "100 minutes" + variables: + BUILD_WITH_DEBUG_MODE_INFO: "true" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh + - make -j + +build-codec-windows-msbuild: + rules: + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-windows + timeout: "10 minutes" + tags: + - ivas-windows + script: + - *print-common-info-windows + - *activate-WX-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +### jobs that check for bitexactness of fx encoder and decoder +check-be-to-target-short-enc-0db: + extends: + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # -/-0dB + LEVEL_SCALING: "1.0" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + USE_LTV: 0 + +check-be-to-target-short-enc-+10db: + extends: + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # +10dB + LEVEL_SCALING: "3.162" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + USE_LTV: 0 + +check-be-to-target-short-enc--10db: + extends: + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # -10dB + LEVEL_SCALING: "0.3162" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + USE_LTV: 0 + +check-be-to-target-short-dec-0db: + extends: + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +/-0dB + LEVEL_SCALING: "1.0" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + USE_LTV: 0 + +check-be-to-target-short-dec-+10db: + extends: + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +10dB + LEVEL_SCALING: "3.162" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + USE_LTV: 0 + +check-be-to-target-short-dec--10db: + extends: + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # -10dB + LEVEL_SCALING: "0.3162" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + USE_LTV: 0 + +### jobs that check for regressions on non-BE testcases +check-regressions-short-enc-0db: + extends: + - .check-regressions-pytest-job + needs: + - job: "check-be-to-target-short-enc-0db" + artifacts: true + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # +/-0dB + LEVEL_SCALING: "1.0" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + USE_LTV: 0 + +check-regressions-short-enc-+10db: + extends: + - .check-regressions-pytest-job + needs: + - job: "check-be-to-target-short-enc-+10db" + artifacts: true + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # +10dB + LEVEL_SCALING: "3.162" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + USE_LTV: 0 + +check-regressions-short-enc--10db: + extends: + - .check-regressions-pytest-job + needs: + - job: "check-be-to-target-short-enc--10db" + artifacts: true + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # -10dB + LEVEL_SCALING: "0.3162" + # overwrite decoder with float reference one + DUT_DECODER_PATH: "$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_DECODER_PATH: "$REF_DECODER_PATH" + USE_LTV: 0 + +check-regressions-short-dec-0db: + extends: + - .check-regressions-pytest-job + needs: + - job: "check-be-to-target-short-dec-0db" + artifacts: true + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +/-0dB + LEVEL_SCALING: "1" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + USE_LTV: 0 + +check-regressions-short-dec-+10db: + extends: + - .check-regressions-pytest-job + needs: + - job: "check-be-to-target-short-dec-+10db" + artifacts: true + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +10dB + LEVEL_SCALING: "3.162" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + USE_LTV: 0 + +check-regressions-short-dec--10db: + extends: + - .check-regressions-pytest-job + needs: + - job: "check-be-to-target-short-dec--10db" + artifacts: true + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # -10dB + LEVEL_SCALING: "0.3162" + # overwrite encoder with float reference one + DUT_ENCODER_PATH: "$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" + MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH" + USE_LTV: 0 + +### jobs that compare bitexactness between merge target and source +renderer-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .test-job-linux-compares-to-fixed-target + - .rules-merge-request-no-draft + # TODO: set reasonable timeout, will most likely take less + timeout: "20 minutes" + tags: + - ivas-linux + stage: compare + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + ### Run test using branch scripts and input + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest renderer results" + reports: + junit: + - report-junit.xml + +ivas-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .test-job-linux-compares-to-fixed-target + - .rules-merge-request-no-draft + stage: compare + timeout: "40 minutes" + tags: + - ivas-linux + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - *merge-request-comparison-setup-codec + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + + ### prepare pytest + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -k "not model" + + ### Run test using branch scripts and input + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + ### run pytest + - exit_code=0 + - testcase_timeout=600 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout -k "not model" || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest ivas results" + reports: + junit: + - report-junit.xml + +#TODO: re-enable after merge to main is done +.split-rendering-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .test-job-linux-compares-to-fixed-target + - .rules-merge-request-no-draft + timeout: "60 minutes" + stage: compare + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - testcase_timeout=60 + + # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181 + - mv $DUT_POST_RENDERER_PATH ISAR_post_rend_tmp + - cp $REF_POST_RENDERER_PATH $DUT_POST_RENDERER_PATH + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - mv ISAR_post_rend_tmp $DUT_POST_RENDERER_PATH + + ### Run test using scripts and input from main + - if [ $ref_using_target == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + - comp_args="--mld --ssnr --odg" + - echo "$comp_args" + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest split rendering results" + reports: + junit: + - report-junit.xml + +# test that runs all modes with 1s input signals +codec-smoke-test: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-ref-no-draft + timeout: "20 minutes" + stage: test + needs: ["build-codec-linux-make"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + # LTV update needed as ltv ISM metadata files are used + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - bash ci/smoke_test.sh || true + - tar cJf logs.tar.xz out/logs/ + - ls -al logs.tar.* + ### analyze for failures + - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi + - ret_val=0 + - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi + - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi + - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi + - exit $ret_val + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - smoke_test_output.txt + - smoke_test_output_jbm.txt + - smoke_test_output_hrtf.txt + - logs.tar.xz + expose_as: "Smoke test results" + +# from float +# check for crashes if first received frame on decoder side is an SID +check-first-frame-is-sid: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-ref-no-draft + stage: test + needs: ["build-codec-linux-make"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + # this rm makes check-for-testvectors only check for the signals we actually need in this test + - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json + - python3 -m pytest ci/test_vectors_available.py + - bash ci/run-first-frame-is-sid-test.sh + +# from float +# needs fix to be merged from floating point repo before it can be activated +.check-bitexactness-hrtf-rom-and-file: + extends: + - .test-job-linux + - .rules-merge-request-no-draft + stage: test + needs: ["build-codec-linux-make"] + timeout: "5 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + - make clean + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + # TODO: run full test again once the custom binary files are supported + - python3 -m pytest -k "not diff_from_rom and not test_binary_file" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" + expose_as: "logs-hrtf-loading" + expire_in: "5 days" + + +### From Float +# Check interop IVAS_cod_test -> IVAS_dec_ref +ivas-interop-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-float-ref-no-draft + - .test-job-linux-compares-to-fixed-target + stage: test + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - *merge-request-comparison-setup-codec + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true + + ### prepare pytest + + # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec) + - exit_code=0 + - exit_code2=0 + # set timeout for individual testcase runs to 60 seconds + - testcase_timeout=60 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true + + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "interop test results" + reports: + junit: + - report-junit.xml + +### split rendering tests +split-rendering-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-no-draft + needs: ["build-codec-linux-make"] + stage: test + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + - make -j + - testcase_timeout=10 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "split rendering smoke results" + reports: + junit: + - report-junit.xml + +lc3-wrapper-unit-test: + extends: + - .test-job-linux + - .rules-merge-request-no-draft + needs: ["build-codec-linux-make"] + stage: test + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + + - echo "Assuming here that you either merge to main or to $FLOAT_REF_BRANCH." + - echo "If you have some other merge target, this might fail and manual fiddling is needed." + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "$FLOAT_REF_BRANCH" ]; then + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + - else + - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test + - fi + +# from float +# test renderer executable +renderer-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request-to-float-ref-no-draft + needs: ["build-codec-linux-make"] + stage: test + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - make -j IVAS_rend + - testcase_timeout=60 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer make pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + asan +renderer-asan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-ref-no-draft + needs: ["build-codec-linux-make"] + stage: test + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer asan pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + msan +renderer-msan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-ref-no-draft + needs: ["build-codec-linux-make"] + stage: test + tags: + - ivas-linux + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - testcase_timeout=180 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer msan pytest results" + reports: + junit: + - report-junit.xml + +# from float +# test renderer executable with cmake + usan +renderer-usan: + extends: + - .test-job-linux + - .rules-merge-request-to-float-ref-no-draft + needs: ["build-codec-linux-make"] + stage: test + tags: + - ivas-linux + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - testcase_timeout=180 + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - grep_exit_code=0 + - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest + - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi + + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "renderer usan pytest results" + reports: + junit: + - report-junit.xml + +.sanitizer-selftest-on-mr: + stage: test + extends: + - .rules-merge-request-to-float-ref-no-draft + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + - report-20ms.html + - report-10ms.html + - report-5ms.html + expose_as: "Sanitizer selftest results" + reports: + junit: + - report-junit-20ms.xml + - report-junit-10ms.xml + - report-junit-5ms.xml + +# From float repo +# to be reused in MR and LTV-scheduled sanitizer test jobs +# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section +.sanitizer-selftest-anchor: &sanitizer-selftest-anchor + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - make clean + - make -j CLANG=$CLANG_NUM + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 + + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + +# from float repo +# NOTE: (per-testcase) timeout was adjusted to work for float only +# code selftest testvectors with memory-sanitizer binaries +.codec-msan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=1 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# code selftest testvectors with address-sanitizer binaries +.codec-asan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=2 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# code selftest testvectors with undefined-behaviour-sanitizer binaries +.codec-usan: + extends: + - .sanitizer-selftest-on-mr + tags: + - ivas-linux + before_script: + - CLANG_NUM=3 + - SELFTEST_SANITY_TIMEOUT=180 + - SELF_TEST_PRM_FILE="scripts/config/self_test.prm" + - USE_LTV="" + <<: *sanitizer-selftest-anchor + +# --------------------------------------------------------------- +# Short test jobs for running from web interface or schedule +# --------------------------------------------------------------- + +### jobs that test fx encoder -> flt decoder +ivas-pytest-compare_to_ref-short-enc: + extends: + - .rules-pytest-to-ref-short + - .test-job-linux + before_script: + - USE_LTV=0 + - ENCODER_TEST="true" + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-anchor + +ivas-pytest-compare_to_ref-short-enc-lev-10: + extends: + - .rules-pytest-to-ref-short + - .test-job-linux + before_script: + - USE_LTV=0 + - ENCODER_TEST="true" + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-anchor + +ivas-pytest-compare_to_ref-short-enc-lev+10: + extends: + - .rules-pytest-to-ref-short + - .test-job-linux + before_script: + - USE_LTV=0 + - ENCODER_TEST="true" + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-anchor + +# encoder dmx comparison jobs +ivas-pytest-compare_to_ref-dmx-short-enc: + extends: + - .rules-pytest-to-ref-enc-short-dmx + - .test-job-linux + before_script: + - USE_LTV=0 + - ENCODER_TEST="true" + - COMPARE_DMX="true" + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-anchor + +ivas-pytest-compare_to_ref-dmx-short-enc-lev-10: + extends: + - .rules-pytest-to-ref-enc-short-dmx + - .test-job-linux + before_script: + - USE_LTV=0 + - ENCODER_TEST="true" + - COMPARE_DMX="true" + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-anchor + +ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: + extends: + - .rules-pytest-to-ref-enc-short-dmx + - .test-job-linux + before_script: + - USE_LTV=0 + - ENCODER_TEST="true" + - COMPARE_DMX="true" + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-anchor + +ivas-pytest-enc-msan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=1 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE=$LONG_TEST_SUITE_ENCODER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-enc-asan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=2 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE=$LONG_TEST_SUITE_ENCODER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-enc-usan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=3 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE=$LONG_TEST_SUITE_ENCODER + <<: *ivas-pytest-sanitizers-anchor + +### jobs that test flt encoder -> fx decoder +ivas-pytest-compare_to_ref-short-dec: + extends: + - .rules-pytest-to-ref-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-anchor + +ivas-pytest-compare_to_ref-short-dec-lev-10: + extends: + - .rules-pytest-to-ref-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-anchor + +ivas-pytest-compare_to_ref-short-dec-lev+10: + extends: + - .rules-pytest-to-ref-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-anchor + +### jobs that compare the output synthesis to the input files directly +ivas-pytest-compare-to-input-short-dec: + extends: + - .rules-pytest-to-input-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-compare-to-input-anchor + +ivas-pytest-compare-to-input-short-enc: + extends: + - .rules-pytest-to-input-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-compare-to-input-anchor + +ivas-pytest-dec-msan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=1 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-dec-asan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=2 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +ivas-pytest-dec-usan: + extends: + - .test-job-linux + tags: + - ivas-basop-linux-fast + before_script: + - CLANG_NUM=3 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + <<: *ivas-pytest-sanitizers-anchor + +# --------------------------------------------------------------- +# Long test job +# --------------------------------------------------------------- + +ivas-pytest-compare_ref-long-enc: + extends: + - .rules-pytest-long + - .test-job-linux + before_script: + - USE_LTV=1 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-dec: + extends: + - .rules-pytest-long + - .test-job-linux + tags: + - ivas-basop-linux + - high-memory-capacity + before_script: + - USE_LTV=1 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-enc-lev-10: + extends: + - .rules-pytest-long + - .test-job-linux + before_script: + - USE_LTV=1 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" + - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-dec-lev-10: + extends: + - .rules-pytest-long + - .test-job-linux + tags: + - ivas-basop-linux + - high-memory-capacity + before_script: + - USE_LTV=1 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-enc-lev+10: + extends: + - .rules-pytest-long + - .test-job-linux + before_script: + - USE_LTV=1 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" + - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-dec-lev+10: + extends: + - .rules-pytest-long + - .test-job-linux + tags: + - ivas-basop-linux + - high-memory-capacity + before_script: + - USE_LTV=1 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE" + - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-fx-fx: + extends: + - .rules-pytest-long-fx-fx + - .test-job-linux + tags: + - ivas-linux + before_script: + - USE_LTV=1 + - REF_ENCODER_PATH=./$DUT_ENCODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE_NO_RENDERER" + - LEVEL_SCALING=1.0 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-fx-fx-lev-10: + extends: + - .rules-pytest-long-fx-fx + - .test-job-linux + tags: + - ivas-linux + before_script: + - USE_LTV=1 + - REF_ENCODER_PATH=./$DUT_ENCODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE_NO_RENDERER" + - LEVEL_SCALING=0.3162 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + +ivas-pytest-compare_ref-long-fx-fx-lev+10: + extends: + - .rules-pytest-long-fx-fx + - .test-job-linux + tags: + - ivas-linux + before_script: + - USE_LTV=1 + - REF_ENCODER_PATH=./$DUT_ENCODER_PATH + - TEST_SUITE="$LONG_TEST_SUITE_NO_RENDERER" + - LEVEL_SCALING=3.162 + - SPLIT_COMPARISON="true" + <<: *ivas-pytest-anchor + + +ivas-smoke-test-saturation: + extends: + - .rules-pytest-saturation-smoke-test + - .test-job-linux-needs-testv-dir + script: + - USE_LTV=1 + - LEVEL_SCALING=32768 + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - if [ $USE_LTV -eq 1 ]; then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - fi + - if [ $LEVEL_SCALING != "1.0" ];then + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - fi + - cp -r scripts/testv/* $TESTV_DIR/ + + # skip prepare_mem_dryrun.py script in smoke_test.sh + - sed -i 's/python3 .\/scripts\/prepare_mem_dryrun.py/#python3 .\/scripts\/prepare_mem_dryrun.py/g' ci/smoke_test.sh + + - bash ci/smoke_test.sh + ### analyze for failures + - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_plc.txt ] || ! [ -s smoke_test_output_jbm_noEXT.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi + - ret_val=0 + - if cat smoke_test_output.txt | grep -c "failed" ; then echo "Smoke test without PLC failed"; ret_val=1; fi + - if cat smoke_test_output_plc.txt | grep -c "failed"; then echo "Smoke test with PLC failed"; ret_val=1; fi + - if cat smoke_test_output_jbm_noEXT.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi + - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi + - exit $ret_val + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - smoke_test_output.txt + - smoke_test_output_plc.txt + - smoke_test_output_jbm_noEXT.txt + - smoke_test_output_hrtf.txt + expose_as: "saturation smoke test results" + + +# GCOV/LCOV coverage analysis of self_test suite +coverage-test-on-main-scheduled: + extends: + - .test-job-linux + - .rules-coverage + stage: test + timeout: 3 hours + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + # Build DuT binaries with GCOV + - make clean >> /dev/null + - make GCOV=1 -j + - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in the first pytest call + + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - exit_code_dec=0 + - exit_code_enc=0 + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH || exit_code_dec=$? + - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer + + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH || exit_code_enc=$? + - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer + + # remove apps and lib_util files from coverage + - lcov -r coverage_dec_rend.info "*apps*" -o coverage_dec_rend.info + - lcov -r coverage_dec_rend.info "*lib_util*" -o coverage_dec_rend.info + - lcov -r coverage_enc_dec_rend.info "*apps*" -o coverage_enc_dec_rend.info + - lcov -r coverage_enc_dec_rend.info "*lib_util*" -o coverage_enc_dec_rend.info + + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main enc/dec/rend @ $commit_sha" + - genhtml coverage_dec_rend.info -o coverage_dec_rend -t "Coverage on main -- dec/rend @ $commit_sha" + artifacts: + name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA" + when: always + expire_in: 1 week + paths: + - coverage_enc_dec_rend.info + - coverage_dec_rend.info + - coverage_enc_dec_rend + - coverage_dec_rend + - report-dec.html + - report-enc.html + expose_as: "Coverage result" + reports: + junit: + - report-junit-dec.xml + - report-junit-enc.xml + +# --------------------------------------------------------------- +# EVS 26.444 test job +# --------------------------------------------------------------- + +# check bitexactness to EVS +be-2-evs-26444: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + tags: + - be-2-evs-basop + stage: test + timeout: "120 minutes" # To be revisited + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h + - make -j >> /dev/null + + # copy over to never change the testvector dir + - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test + - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams + + - exit_code=0 + - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi + - exit 0 + + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "EVS 26444 result" + reports: + junit: + - report-junit.xml + +ivas-pytest-renderer: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-renderer" + before_script: + - USE_LTV=0 + - TEST_SUITE="tests/renderer" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-anchor + +peaq-enc-passthrough: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "peaq-enc-passthrough" + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./$REF_DECODER_PATH + - TEST_SUITE="tests/test_enc_passthrough.py" + - DELTA_ODG="true" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-anchor + +# --------------------------------------------------------------- +# Various other tests +# --------------------------------------------------------------- + +# TODO: actually run on MR once main problems are fixed +voip-be-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "voip-be-test" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "ivas-float-update" + # - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + stage: test + needs: ["build-codec-linux-make"] + timeout: "10 minutes" + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - make clean + - make -j > build_log.txt + - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + artifacts: + when: always + expire_in: "5 days" + paths: + - build_log.txt + +check-be-between-renderer-framesizes: + extends: + - .test-job-linux + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "renderer-framesize-be" + stage: test + needs: ["build-codec-linux-make"] + script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh + - make clean + - make -j + - echo "$SHORT_TEST_SUITE" + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh + - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set + + - exit_code=0 + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? + + - exit_code5=0 + - exit_code10=0 + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + + - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true + - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true + - zero_errors=1 + + - *print-results-banner + - echo "!! Encoder command lines are in the 20ms log files only !!!" + - if [ $exit_code -ne 0 ]; then echo "20 ms framesize run already failed with errors."; zero_errors=0; fi + - if [ $zero_errors5 != 1 ]; then echo "run error with 5ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors10 != 1 ]; then echo "run error with 10ms rendering encountered"; zero_errors=0 ; fi + - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi + - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi + - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi + - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + expose_as: "pytest 5ms and 10ms vs 20ms results" + paths: + - report-junit-5ms.xml + - report-5ms.html + - report-junit-10ms.xml + - report-10ms.html + - report-junit-20ms.xml + - report-20ms.html + reports: + junit: + - report-junit-5ms.xml + - report-junit-10ms.xml + - report-junit-20ms.xml + +# To inject backup copy in manual trigger: +# - Store logs backup in accessible folder, e.g. /usr/local/backup +# - Set MANUAL_PIPELINE_TRIGGER to long-term-logs +# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5 +# - Set LOGS_BACKUP_SOURCE_DIR to source folder, e.g. /usr/local/backup +ivas-long-term-job-logs: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs" + - if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs" + tags: + - $RUNNER_TAG + stage: maintenance + timeout: "25 minutes" + script: + - !reference [ .job-linux, before_script ] + - set -euxo pipefail + - echo "Running on RUNNER_TAG = $RUNNER_TAG" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + # Clean up logs + - if [ -d "logs" ]; then rm -rf "logs"; fi + # Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set + - if [ "$LOGS_BACKUP_SOURCE_DIR" != "" ]; then + - cp_ret=0 + - cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ || cp_ret=$? + - if [ "$cp_ret" != 0 ]; then + - echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!" + - exit 1 + - fi + - fi + + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID) + - echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous" + + - if [ "$id_previous" != "-1" ]; then + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + # If there is overlap between injected log and new log, use the new logs. + - unzip -o artifacts.zip + - else + # create logs dir if it doesn't exist (should only happen in first run) + - echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!" + - mkdir logs + - fi + + # create folder for today's results + - TODAY=$(date +'%Y-%m-%d') + - mkdir -p logs/$TODAY + + # Aggregate job logs + - job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10" + - for job_name in $job_names; do + - echo "Getting job logs for $job_name" + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID) + - echo "Job ID from variables - "$job_name", Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts + - mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY + - rm artifacts.zip + - rm -r previous_artifacts + - done + - ls logs + - exit 0 + + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 4 weeks + when: always + paths: + - logs + expose_as: "ivas long term job logs results" + +# To store backup copy: +# - Prepare accessible folder for backup, e.g. /usr/local/backup +# - Set MANUAL_PIPELINE_TRIGGER to backup-long-term-logs +# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5 +# - Set LOGS_BACKUP_TARGET_DIR to source folder, e.g. /usr/local/backup +backup-long-term-job-logs: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs" + - if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs" + tags: + - $RUNNER_TAG + stage: maintenance + timeout: "25 minutes" + script: + - !reference [ .job-linux, before_script ] + - set -euxo pipefail + - echo "Running on RUNNER_TAG = $RUNNER_TAG" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + # Find ID of last run of ivas-long-term-job-logs + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID) + + - if [ "$id_previous" != "-1" ]; then + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip + - else + # create logs dir if it doesn't exist + - mkdir logs + - fi + + # Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set + - if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then + - cp -r logs $LOGS_BACKUP_TARGET_DIR + - exit 0 + - else + - echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!" + - exit 1 + - fi + + +# --------------------------------------------------------------- +# Complexity measurement jobs +# --------------------------------------------------------------- + +# create necessary environment +.complexity-measurements-setup: &complexity-measurements-setup + ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work + - mkdir -p wmops/logs + + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + # this is a testing/maintenance mechanism to force getting the log history from a specific job id + # see below in the concrete complexity jobs + - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip + - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html + - public_dir="$CI_JOB_NAME-public" + + # if is needed to catch case when no artifact is there (first run), similarly as above + # 1. check for public_dir being there as this might not be the case when artifact download failed + # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts + - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then + - mv $public_dir/* wmops/ + # check here if we have the split-by-levels files present - if not, fake them up with the existing global one + # this is needed for the first run with split graphs on a branch where the global version did run previously + # NOTE: checking only for level_1 file here as this should already be sufficient + # NOTE2: also not chechking for RAM for same reason + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi + - fi + + - rm artifacts.zip + - rm -rf $public_dir + + ### 1.5.part: get the corresponding measurement from ivas-float-update + # only do this on main and not for the complexity meaurements run on ivas-float-update + - if [ "$CI_COMMIT_REF_NAME" == "main" ]; then + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $FLOAT_REF_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip + - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true + # add file to arguments only if the artifact could be retrieved to prevent error later. + - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi + - fi + +# prepare artifacts -> move to public directory +.complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts + - public_dir="$CI_JOB_NAME-public" + - mkdir $public_dir + - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ + # move logfiles for links + - mkdir $public_dir/logs + # first move logs + - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") + - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done + - if [ "$CI_COMMIT_REF_NAME" == "$FLOAT_REF_BRANCH" ]; then mv wmops/logs/latest_WMOPS.csv $public_dir/logs/; fi + # copy index page blueprint + - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html + # patch the format in the title + - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html + # do separately here to avoid overwrite complaints by mv + - mv -f ci/complexity_measurements/style.css ${public_dir}/ + +.complexity-measurements-report-summary: &complexity-measurements-report-summary + - *print-results-banner + - if [ $ret_val -eq 0 ]; then + - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." + - elif [ $ret_val -eq 123 ]; then + - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." + - else + - echo -e "Something went wrong in running the codec. Likely some modes were crashing." + - fi + +.complexity-template: + extends: + - .test-job-linux + stage: test + variables: + ret_val: 0 + GET_WMOPS_ARGS: "mem_only" + timeout: 3 hours 30 minutes + before_script: + - !reference [ .test-job-linux, before_script ] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh + - *complexity-measurements-setup + # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present) + - rm -rf COMPLEXITY/logs + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA" + when: always + expire_in: 2 week + paths: + - $CI_JOB_NAME-public + - COMPLEXITY/logs + +complexity-stereo-in-stereo-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + variables: + JOB_ID_INJECT: "" + script: + - in_format=stereo + - out_format=stereo + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-ism-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 1 hour + variables: + JOB_ID_INJECT: "" + script: + - in_format=ISM + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-ism-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 2 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=ISM + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-ism-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 3 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=ISM + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-sba-hoa3-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 4 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=HOA3 + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-sba-hoa3-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 5 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=HOA3 + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-sba-hoa3-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 6 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=HOA3 + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-mc-in-7_1_4-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 7 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=MC + - out_format=7_1_4 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-mc-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 10 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=MC + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-mc-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 12 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=MC + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-masa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 15 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=MASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-masa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 16 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=MASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-masa-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 17 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=MASA + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +# complexity-omasa-in-ext-out: +# extends: +# - .complexity-template +# rules: +# - if: $MEASURE_COMPLEXITY_LINUX +# when: delayed +# start_in: 13 hours + # variables: + # JOB_ID_INJECT: "" +# script: +# - in_format=OMASA +# - out_format=EXT +# - ret_val=0 +# - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? +# - *complexity-measurements-prepare-artifacts +# - *complexity-measurements-report-summary +# - exit $ret_val + +complexity-omasa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 18 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=OMASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-omasa-in-hoa3-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 20 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=OMASA + - out_format=HOA3 + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-StereoDmxEVS-stereo-in-mono-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 22 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=StereoDmxEVS + - out_format=mono + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +# complexity-osba-in-ext-out: +# extends: +# - .complexity-template +# rules: +# - if: $MEASURE_COMPLEXITY_LINUX +# when: delayed +# start_in: 17 hours +# script: +# - in_format=OSBA +# - out_format=EXT +# - ret_val=0 +# - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? +# - *complexity-measurements-prepare-artifacts +# - *complexity-measurements-report-summary +# - exit $ret_val + +complexity-osba-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 22 hours 30 minutes + variables: + JOB_ID_INJECT: "" + script: + - in_format=OSBA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +complexity-osba-in-binaural_room_ir-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 25 hours + variables: + JOB_ID_INJECT: "" + script: + - in_format=OSBA + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - *complexity-measurements-prepare-artifacts + - *complexity-measurements-report-summary + - exit $ret_val + +# job that sets up gitlab pages website +pages: + stage: deploy + tags: + - ivas-basop-linux + rules: + - if: $UPDATE_PAGES + script: + - !reference [ .job-linux, before_script ] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - python3 ci/setup_pages.py + - ls + - ls -lh public + artifacts: + paths: + - public + expire_in: 1 day -- GitLab From 74346a0ab3fa3247db854e743c853559989756fb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 28 Jul 2025 12:18:15 +0200 Subject: [PATCH 16/30] fix includes --- main-basop.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 029eb06..b71be63 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1,6 +1,4 @@ include: - - local: .gitlab-ci/variables.yml - - local: .gitlab-ci/rules-basis.yml - local: main.yml default: -- GitLab From 73f41f7628a42be6a07120f8795dda9aee2998f1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 28 Jul 2025 15:22:51 +0200 Subject: [PATCH 17/30] revise rules for build jobs --- main-basop.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index b71be63..b8bba07 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -947,7 +947,7 @@ build-codec-linux-make: when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' when: never @@ -965,10 +965,8 @@ build-codec-linux-cmake: when: never - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES when: never - - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs - - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: @@ -987,10 +985,9 @@ build-codec-linux-instrumented-make: when: never - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES when: never - - if: $CI_PIPELINE_SOURCE == 'web' + - if: $MEASURE_COMPLEXITY_LINUX - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: @@ -1009,10 +1006,9 @@ build-codec-sanitizers-linux: when: never - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES when: never - - if: $CI_PIPELINE_SOURCE == 'web' + - if: $MANUAL_PIPELINE_TYPE == 'sanitizer' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: @@ -1031,10 +1027,9 @@ build-codec-linux-debugging-make: when: never - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES when: never - - if: $CI_PIPELINE_SOURCE == 'web' + - if: $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: @@ -1053,7 +1048,7 @@ build-codec-windows-msbuild: when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' when: never -- GitLab From 77aab356dfa6dd55cdb2311f6274959bcd9ea078 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 28 Jul 2025 15:38:16 +0200 Subject: [PATCH 18/30] add csv reports to the comparison jobs --- main-basop.yml | 55 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index b8bba07..f28ea6e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1241,6 +1241,10 @@ renderer-pytest-on-merge-request: tags: - ivas-linux stage: compare + variables: + REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml" + REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html" + REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -1262,17 +1266,19 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - *merge-request-comparison-check + - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + allow_failure: exit_codes: - 123 @@ -1281,12 +1287,13 @@ renderer-pytest-on-merge-request: expire_in: 1 week when: always paths: - - report-junit.xml - - report.html + - $REPORT_XML + - $REPORT_HTML + - $REPORT_CSV expose_as: "pytest renderer results" reports: junit: - - report-junit.xml + - $REPORT_XML ivas-pytest-on-merge-request: extends: @@ -1295,6 +1302,10 @@ ivas-pytest-on-merge-request: - .rules-merge-request-no-draft stage: compare timeout: "40 minutes" + variables: + REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml" + REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html" + REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" tags: - ivas-linux script: @@ -1327,11 +1338,13 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=600 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout -k "not model" || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout -k "not model" || exit_code=$? + - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - *merge-request-comparison-check + - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + allow_failure: exit_codes: - 123 @@ -1340,12 +1353,13 @@ ivas-pytest-on-merge-request: expire_in: 1 week when: always paths: - - report-junit.xml - - report.html + - $REPORT_XML + - $REPORT_HTML + - $REPORT_CSV expose_as: "pytest ivas results" reports: junit: - - report-junit.xml + - $REPORT_XML #TODO: re-enable after merge to main is done .split-rendering-pytest-on-merge-request: @@ -1355,6 +1369,10 @@ ivas-pytest-on-merge-request: - .rules-merge-request-no-draft timeout: "60 minutes" stage: compare + variables: + REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml" + REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html" + REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -1380,7 +1398,7 @@ ivas-pytest-on-merge-request: # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181 - mv $DUT_POST_RENDERER_PATH ISAR_post_rend_tmp - cp $REF_POST_RENDERER_PATH $DUT_POST_RENDERER_PATH - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - mv ISAR_post_rend_tmp $DUT_POST_RENDERER_PATH ### Run test using scripts and input from main @@ -1391,11 +1409,13 @@ ivas-pytest-on-merge-request: - echo "$comp_args" # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - *merge-request-comparison-check + - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + allow_failure: exit_codes: - 123 @@ -1404,12 +1424,13 @@ ivas-pytest-on-merge-request: expire_in: 2 week when: always paths: - - report-junit.xml - - report.html + - $REPORT_XML + - $REPORT_HTML + - $REPORT_CSV expose_as: "pytest split rendering results" reports: junit: - - report-junit.xml + - $REPORT_XML # test that runs all modes with 1s input signals codec-smoke-test: -- GitLab From 40ab4b20825e1fe49991ef4e574b51caaf2d1742 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 29 Jul 2025 10:19:37 +0200 Subject: [PATCH 19/30] move parse_xml.py call before anchor with exit --- main-basop.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index f28ea6e..fc992e7 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1275,10 +1275,10 @@ renderer-pytest-on-merge-request: - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - - *merge-request-comparison-check - - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + - *merge-request-comparison-check + allow_failure: exit_codes: - 123 @@ -1341,10 +1341,10 @@ ivas-pytest-on-merge-request: - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout -k "not model" || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - - *merge-request-comparison-check - - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + - *merge-request-comparison-check + allow_failure: exit_codes: - 123 @@ -1412,10 +1412,10 @@ ivas-pytest-on-merge-request: - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - - *merge-request-comparison-check - - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV + - *merge-request-comparison-check + allow_failure: exit_codes: - 123 -- GitLab From 5022c214129b16d225cbb6fb539cdbe683d67f4b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 08:37:30 +0200 Subject: [PATCH 20/30] tags ivas-linux -> ivas-basop-linux --- main-basop.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index fc992e7..c94c534 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -828,7 +828,7 @@ fail-pipeline-if-in-draft: when: never stage: postvalidate tags: - - ivas-linux + - ivas-basop-linux script: - echo "Your MR is still in Draft state, set it to ready to be mergable, then retrigger the pipeline." - exit 1 @@ -840,7 +840,7 @@ check-naming-of-branch-for-basop-update-merges: - .job-linux stage: prevalidate tags: - - ivas-linux + - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then @@ -1056,7 +1056,7 @@ build-codec-windows-msbuild: - .build-job-windows timeout: "10 minutes" tags: - - ivas-windows + - ivas-basop-windows script: - *print-common-info-windows - *activate-WX-windows @@ -1239,7 +1239,7 @@ renderer-pytest-on-merge-request: # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" tags: - - ivas-linux + - ivas-basop-linux stage: compare variables: REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml" @@ -1307,7 +1307,7 @@ ivas-pytest-on-merge-request: REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html" REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" tags: - - ivas-linux + - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -1670,7 +1670,7 @@ renderer-msan: needs: ["build-codec-linux-make"] stage: test tags: - - ivas-linux + - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -1699,7 +1699,7 @@ renderer-usan: needs: ["build-codec-linux-make"] stage: test tags: - - ivas-linux + - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -1783,7 +1783,7 @@ renderer-usan: extends: - .sanitizer-selftest-on-mr tags: - - ivas-linux + - ivas-basop-linux before_script: - CLANG_NUM=1 - SELFTEST_SANITY_TIMEOUT=180 @@ -1796,7 +1796,7 @@ renderer-usan: extends: - .sanitizer-selftest-on-mr tags: - - ivas-linux + - ivas-basop--linux before_script: - CLANG_NUM=2 - SELFTEST_SANITY_TIMEOUT=180 @@ -1809,7 +1809,7 @@ renderer-usan: extends: - .sanitizer-selftest-on-mr tags: - - ivas-linux + - ivas-basop--linux before_script: - CLANG_NUM=3 - SELFTEST_SANITY_TIMEOUT=180 @@ -2111,7 +2111,7 @@ ivas-pytest-compare_ref-long-fx-fx: - .rules-pytest-long-fx-fx - .test-job-linux tags: - - ivas-linux + - ivas-basop-linux before_script: - USE_LTV=1 - REF_ENCODER_PATH=./$DUT_ENCODER_PATH @@ -2125,7 +2125,7 @@ ivas-pytest-compare_ref-long-fx-fx-lev-10: - .rules-pytest-long-fx-fx - .test-job-linux tags: - - ivas-linux + - ivas-basop-linux before_script: - USE_LTV=1 - REF_ENCODER_PATH=./$DUT_ENCODER_PATH @@ -2139,7 +2139,7 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - .rules-pytest-long-fx-fx - .test-job-linux tags: - - ivas-linux + - ivas-basop-linux before_script: - USE_LTV=1 - REF_ENCODER_PATH=./$DUT_ENCODER_PATH -- GitLab From a60d77e2bdb1040ca6d3c0faf2ec0e59ed12f220 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 08:39:29 +0200 Subject: [PATCH 21/30] remove redundant part (already present in main.yml) --- main-basop.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index c94c534..4cb3a4e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1,8 +1,6 @@ include: - local: main.yml -default: - interruptible: true # Make all jobs by default interruptible workflow: name: '$IVAS_PIPELINE_NAME' -- GitLab From fbc1c2a644ab2db49602a07b0e27a74e79099e24 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 12:45:29 +0200 Subject: [PATCH 22/30] convert print-results-banner anchor into script --- main-basop.yml | 17 ++++++----------- snippets/basop/print-results-banner.sh | 5 +++++ 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 snippets/basop/print-results-banner.sh diff --git a/main-basop.yml b/main-basop.yml index 4cb3a4e..24a3a7d 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -239,11 +239,6 @@ workflow: artifacts: true - job: build-codec-linux-make -.print-results-banner: &print-results-banner - - set +x - - echo "" - - echo -e "==================================================================================================================\n================================================== TEST RESULTS ==================================================\n==================================================================================================================\n" - # template for test jobs on linux that need the TESTV_DIR .test-job-linux-needs-testv-dir: extends: .test-job-linux @@ -454,7 +449,7 @@ workflow: - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true - num_errors=$(wc -l < errors_testcases_for_printing.txt) - - *print-results-banner + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - echo "Found these $num_failures non-bitexact testcases:" - cat failed_testcases_for_printing.txt @@ -474,7 +469,7 @@ workflow: - touch $FAILED_TESTCASES_LIST - touch $ERRORS_TESTCASES_LIST - touch $PYTEST_CACHE_ARTIFACT - - *print-results-banner + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - echo "All testcases are bitexact." - fi - exit $exit_code @@ -538,7 +533,7 @@ workflow: - export PYTEST_ADDOPTS=--last-failed - else # turn off echoing back of commands for result printout - - *print-results-banner + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." - exit 0 - fi @@ -612,7 +607,7 @@ workflow: - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_BRANCH $CSV_MAIN > regression_log.txt || regressions_found=$? - exit_code=0 - - *print-results-banner + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - if [ $zero_errors_branch != 1 ]; then - echo "Run errors encountered!" - exit_code=$EXIT_CODE_FAIL @@ -2363,7 +2358,7 @@ check-be-between-renderer-framesizes: - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true - zero_errors=1 - - *print-results-banner + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - echo "!! Encoder command lines are in the 20ms log files only !!!" - if [ $exit_code -ne 0 ]; then echo "20 ms framesize run already failed with errors."; zero_errors=0; fi - if [ $zero_errors5 != 1 ]; then echo "run error with 5ms rendering encountered"; zero_errors=0 ; fi @@ -2576,7 +2571,7 @@ backup-long-term-job-logs: - mv -f ci/complexity_measurements/style.css ${public_dir}/ .complexity-measurements-report-summary: &complexity-measurements-report-summary - - *print-results-banner + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - if [ $ret_val -eq 0 ]; then - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." - elif [ $ret_val -eq 123 ]; then diff --git a/snippets/basop/print-results-banner.sh b/snippets/basop/print-results-banner.sh new file mode 100644 index 0000000..5906ab1 --- /dev/null +++ b/snippets/basop/print-results-banner.sh @@ -0,0 +1,5 @@ +#! /bin/bash + +set -euo pipefail +echo "" +echo -e "==================================================================================================================\n================================================== TEST RESULTS ==================================================\n==================================================================================================================\n" -- GitLab From 2699aba93f5c55a6fc8cef81d337e30e14e0f644 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 12:48:02 +0200 Subject: [PATCH 23/30] use script instead of anchor for Werror activation --- main-basop.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 24a3a7d..1d6d695 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -81,10 +81,6 @@ workflow: echo "Commit time was $CI_COMMIT_TIMESTAMP" ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression -.activate-Werror-linux: &activate-Werror-linux - - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile - - sed -i.bak "s/# \(set(CMAKE_C_FLAGS \"\${CMAKE_C_FLAGS} -Werror\")\)/\1/" CMakeLists.txt - .activate-WX-windows: &activate-WX-windows - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } @@ -949,7 +945,7 @@ build-codec-linux-make: timeout: "100 minutes" script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - *activate-Werror-linux + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh - make -j build-codec-linux-cmake: @@ -968,6 +964,7 @@ build-codec-linux-cmake: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh - cmake -B cmake-build -G "Unix Makefiles" - cmake --build cmake-build -- -j @@ -1010,8 +1007,6 @@ build-codec-sanitizers-linux: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - # TODO: re-enable once all the warnings have been fixed - #- *activate-Werror-linux - bash ci/build_codec_sanitizers_linux.sh build-codec-linux-debugging-make: -- GitLab From 40f88a088f9b43d3eff9322266b4d74bddba156a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 12:54:13 +0200 Subject: [PATCH 24/30] convert merge-request-comparison-setup-codec into script --- main-basop.yml | 21 ++++--------------- .../merge-request-comparison-setup-codec.sh | 11 ++++++++++ 2 files changed, 15 insertions(+), 17 deletions(-) create mode 100644 snippets/merge-request-comparison-setup-codec.sh diff --git a/main-basop.yml b/main-basop.yml index 1d6d695..24347ad 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -86,19 +86,6 @@ workflow: - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } # From float CI -.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec - - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - - # need to rename to match the conventions for the renderer tests... - - mv $MERGE_TARGET_ENCODER_PATH $REF_ENCODER_PATH - - mv $MERGE_TARGET_DECODER_PATH $REF_DECODER_PATH - - mv $MERGE_TARGET_RENDERER_PATH $REF_RENDERER_PATH - # NOTE: temporary solution for merge to main - - if [ -f $MERGE_TARGET_POST_RENDERER_PATH ]; then mv $MERGE_TARGET_POST_RENDERER_PATH $REF_POST_RENDERER_PATH; fi - - - make clean - - make -j - .merge-request-comparison-check: &merge-request-comparison-check - echo "--------------- Running merge-request-comparison-check anchor ---------------" - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1236,7 +1223,7 @@ renderer-pytest-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - *merge-request-comparison-setup-codec + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -1299,7 +1286,7 @@ ivas-pytest-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - *merge-request-comparison-setup-codec + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -1364,7 +1351,7 @@ ivas-pytest-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - *merge-request-comparison-setup-codec + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo @@ -1513,7 +1500,7 @@ ivas-interop-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh - - *merge-request-comparison-setup-codec + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set diff --git a/snippets/merge-request-comparison-setup-codec.sh b/snippets/merge-request-comparison-setup-codec.sh new file mode 100644 index 0000000..f381d4b --- /dev/null +++ b/snippets/merge-request-comparison-setup-codec.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +# need to rename to match the conventions for the renderer tests... +mv $MERGE_TARGET_ENCODER_PATH $REF_ENCODER_PATH +mv $MERGE_TARGET_DECODER_PATH $REF_DECODER_PATH +mv $MERGE_TARGET_RENDERER_PATH $REF_RENDERER_PATH +# NOTE: temporary solution for merge to main +if [ -f $MERGE_TARGET_POST_RENDERER_PATH ]; then mv $MERGE_TARGET_POST_RENDERER_PATH $REF_POST_RENDERER_PATH; fi + +make clean +make -j -- GitLab From 2f0f4976ac204cc4962242d7fc2e3c359b3a6903 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 14:34:37 +0200 Subject: [PATCH 25/30] convert overwrite-pytest-cache-with-artifact into script --- main-basop.yml | 14 ++++---------- .../basop/overwrite-pytest-cache-with-artifact.sh | 10 ++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 snippets/basop/overwrite-pytest-cache-with-artifact.sh diff --git a/main-basop.yml b/main-basop.yml index 24347ad..c72b976 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -482,12 +482,6 @@ workflow: - $XML_REPORT - $XML_REPORT -.overwrite-pytest-cache-with-artifact: &overwrite-pytest-cache-with-artifact - - if [ -f $PYTEST_CACHE_ARTIFACT ]; then - - rm -rf .pytest_cache || true - - unzip $PYTEST_CACHE_ARTIFACT - - fi - .check-regressions-pytest-job: extends: - .check-be-job @@ -512,7 +506,7 @@ workflow: - set -euxo pipefail - if [ -s $FAILED_TESTCASES_LIST ]; then - - *overwrite-pytest-cache-with-artifact + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh - export PYTEST_ADDOPTS=--last-failed - else # turn off echoing back of commands for result printout @@ -565,7 +559,7 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - python3 -m pytest $TEST_SUITE -k "not model" -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? - - *overwrite-pytest-cache-with-artifact + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - python3 -m pytest --tb=no -q $TEST_SUITE -k "not model" -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true @@ -575,12 +569,12 @@ workflow: # create the summary based on the branch only - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - - *overwrite-pytest-cache-with-artifact + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - python3 -m pytest $TEST_SUITE -k "not model" -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? - - *overwrite-pytest-cache-with-artifact + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - python3 -m pytest --tb=no -q $TEST_SUITE -k "not model" -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN diff --git a/snippets/basop/overwrite-pytest-cache-with-artifact.sh b/snippets/basop/overwrite-pytest-cache-with-artifact.sh new file mode 100644 index 0000000..73e798c --- /dev/null +++ b/snippets/basop/overwrite-pytest-cache-with-artifact.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +set -euxo pipefail + +cd "${CI_PROJECT_DIR}" + +if [ -f $PYTEST_CACHE_ARTIFACT ]; then + rm -rf .pytest_cache || true + unzip $PYTEST_CACHE_ARTIFACT +fi -- GitLab From 4a65e671872d6140527937b6f69dedf829630167 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 14:48:50 +0200 Subject: [PATCH 26/30] move merge-request-comparison-check into script --- main-basop.yml | 14 +++---------- .../float/merge-request-comparison-check.sh | 0 snippets/merge-request-comparison-check.sh | 21 +++++++++++++++++++ 3 files changed, 24 insertions(+), 11 deletions(-) delete mode 100644 snippets/float/merge-request-comparison-check.sh create mode 100644 snippets/merge-request-comparison-check.sh diff --git a/main-basop.yml b/main-basop.yml index c72b976..5945e08 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -85,14 +85,6 @@ workflow: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } -# From float CI -.merge-request-comparison-check: &merge-request-comparison-check - - echo "--------------- Running merge-request-comparison-check anchor ---------------" - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 - .rules-merge-request: rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' @@ -1246,7 +1238,7 @@ renderer-pytest-on-merge-request: - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV - - *merge-request-comparison-check + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-check.sh allow_failure: exit_codes: @@ -1312,7 +1304,7 @@ ivas-pytest-on-merge-request: - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV - - *merge-request-comparison-check + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-check.sh allow_failure: exit_codes: @@ -1383,7 +1375,7 @@ ivas-pytest-on-merge-request: - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV - - *merge-request-comparison-check + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-check.sh allow_failure: exit_codes: diff --git a/snippets/float/merge-request-comparison-check.sh b/snippets/float/merge-request-comparison-check.sh deleted file mode 100644 index e69de29..0000000 diff --git a/snippets/merge-request-comparison-check.sh b/snippets/merge-request-comparison-check.sh new file mode 100644 index 0000000..f9cee46 --- /dev/null +++ b/snippets/merge-request-comparison-check.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +# This needs to "source"'d as this reuses vars from the job + +set +x +echo "--------------- Running merge-request-comparison-check anchor ---------------" +if [ $zero_errors != 1 ]; then + echo "Run errors encountered!" + exit $EXIT_CODE_FAIL +fi +if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then + echo "Non-bitexact cases without non-BE tag encountered!" + exit $EXIT_CODE_FAIL +fi +if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then + echo "Non-bitexact cases with non-BE tag encountered" + exit $EXIT_CODE_NON_BE +fi + +echo "All is bit-exact" +exit 0 -- GitLab From 16ce953040e0ce76852b14ddf7b79c289ba2a3b7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 14:55:58 +0200 Subject: [PATCH 27/30] add note about snippets to the readme --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 899e52d..81c6e96 100644 --- a/README.md +++ b/README.md @@ -1 +1,14 @@ -please refer to the [Wiki](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/home) +# IVAS codec CI + +Home of CI configs and scripts for: + +- [IVAS](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec) +- [IVAS BASOP](https://forge.3gpp.org/rep/sa4/audio/ivas-basop) + +Refer to the [IVAS Wiki](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/home) for more info on the CI setup. +This repository is mainly meant to share scripts and configs between the two repositories and (for BASOP) between differently configured branches. + +## Notes about usage of the snippets + +- in general prefer adding scripts here over using yaml anchors in a CI config file +- if your script shall modify some environment variable, you need to run it with `source` instead of calling it. -- GitLab From 82759ceb6c7a5ff800e4625141333b96e0890a9a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 15:34:19 +0200 Subject: [PATCH 28/30] fix typos in build commands of renderer *san tests --- main-basop.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 5945e08..a8bba35 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1635,7 +1635,7 @@ renderer-msan: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout @@ -1664,7 +1664,7 @@ renderer-usan: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout -- GitLab From 55305a103169fc72f5061895913e5edb9beb9772 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 30 Jul 2025 15:57:23 +0200 Subject: [PATCH 29/30] change merge-request-comparison-setup-codec script --- main-basop.yml | 4 ++++ snippets/merge-request-comparison-setup-codec.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/main-basop.yml b/main-basop.yml index a8bba35..8bcf9b7 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1209,6 +1209,7 @@ renderer-pytest-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -1272,6 +1273,7 @@ ivas-pytest-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set @@ -1337,6 +1339,7 @@ ivas-pytest-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -1486,6 +1489,7 @@ ivas-interop-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES --use-main-pc-set diff --git a/snippets/merge-request-comparison-setup-codec.sh b/snippets/merge-request-comparison-setup-codec.sh index f381d4b..622a4f5 100644 --- a/snippets/merge-request-comparison-setup-codec.sh +++ b/snippets/merge-request-comparison-setup-codec.sh @@ -1,5 +1,7 @@ #! /bin/bash +# source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh + # need to rename to match the conventions for the renderer tests... mv $MERGE_TARGET_ENCODER_PATH $REF_ENCODER_PATH mv $MERGE_TARGET_DECODER_PATH $REF_DECODER_PATH -- GitLab From fb3e3656cf5efae3aecddb82b2a2da21f4fefe06 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 31 Jul 2025 09:33:46 +0200 Subject: [PATCH 30/30] change mechanism for getting the float ref branch for a BASOP port branch --- main-basop.yml | 4 +- snippets/basop/get-float-ref-branch-name.sh | 66 +++++++++++++++++++ .../basop/set-float-reference-branches.sh | 13 ++-- 3 files changed, 73 insertions(+), 10 deletions(-) create mode 100755 snippets/basop/get-float-ref-branch-name.sh diff --git a/main-basop.yml b/main-basop.yml index 8bcf9b7..f818c0c 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -796,7 +796,7 @@ fail-pipeline-if-in-draft: - exit 1 # this branch runs on merges to main-pc only and will fail if the branch itself does not conform to the naming conventions -check-naming-of-branch-for-basop-update-merges: +.check-naming-of-branch-for-basop-update-merges: extends: - .rules-merge-request-to-main - .job-linux @@ -806,7 +806,7 @@ check-naming-of-branch-for-basop-update-merges: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then - - ci/get_float_ref_branch_name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME - fi branch-is-up-to-date-with-target-pre: diff --git a/snippets/basop/get-float-ref-branch-name.sh b/snippets/basop/get-float-ref-branch-name.sh new file mode 100755 index 0000000..6a0c93a --- /dev/null +++ b/snippets/basop/get-float-ref-branch-name.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. +# +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. +# +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. +# +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. +# +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +BASOP_PATTERN="^[0-9]+[_-]basop[_-].*" +MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '_basop_', e.g. '123_basop_fix_this_one_bug-2'. +If this branch is porting a float MR with a corresponding float part, then +ticking to this branch is needed for the testing system to match this branch with its float-reference counterpart. +Please rename your branch. You can easily do this by creating a new branch from this branch: + - git checkout -b +You then also need to create a new merge request and update the links in your issue. +See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec + +If this is not a porting branch, ignore above message. +" + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +branchname_in="$1" +branchname_out="float-pc" + +if ! [[ "$branchname_in" =~ $BASOP_PATTERN ]]; then + printf "$MSG_DOES_NOT_MATCH" "$branchname_in" +fi + +# The float ref branch is just the same name, but with basop -> ref replacement +# Replace only the first occurrence, as "basop" may be present in the later description +# If the format is correct, then before "_basop", only numbers can occur +float_ref_branchname="${branchname_in/basop/ref}" +git_result=$(git branch -a) + +# If the branch does not exist, default to "float-pc" +if [[ "$git_result" =~ "$float_ref_branchname" ]]; then + branchname_out="${float_ref_branchname}" +fi + +echo "$branchname_out" diff --git a/snippets/basop/set-float-reference-branches.sh b/snippets/basop/set-float-reference-branches.sh index bd284c3..c02e565 100644 --- a/snippets/basop/set-float-reference-branches.sh +++ b/snippets/basop/set-float-reference-branches.sh @@ -7,14 +7,11 @@ cd "${CI_PROJECT_DIR}" # use ivas-float-update as reference for the merge source per default FLOAT_REF_BRANCH_MERGE_SOURCE=$FLOAT_REF_BRANCH -# for porting MRs use the name-check script -if [[ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME = "main" ]] && [[ "${CI_MERGE_REQUEST_TITLE,,}" =~ \[porting\] ]]; then - if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - # a bit awkward: write to file + standard out first so that the error message is visible in case of failure. Then fill the variable from the file - ci/get_float_ref_branch_name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME | tee tmp_ref_branch.txt - FLOAT_REF_BRANCH_MERGE_SOURCE=$(cat tmp_ref_branch.txt) - fi +if [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[skip[[:space:]_-]name[[:space:]_-]check\] ]] && [[ ! "$CI_MERGE_REQUEST_TITLE" =~ \[CI\] ]]; then + bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + # a bit awkward: write to file + standard out first so that the error message is visible in case of failure. Then fill the variable from the file + bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME | tee tmp_ref_branch.txt + FLOAT_REF_BRANCH_MERGE_SOURCE=$(cat tmp_ref_branch.txt) fi # TODO: remove after merge to main -- GitLab