diff --git a/includes/default-variables.yml b/includes/default-variables.yml index 2774c578ba79b4182014d9a7071649b110dae633..6329664102945f6734e9e5a8179e46df00fd30f2 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -9,7 +9,6 @@ variables: GIT_CLEAN_FLAGS: -ffdxq TESTCASE_TIMEOUT_STV_SANITIZERS: 240 TESTCASE_TIMEOUT_LTV_SANITIZERS: 2400 - BASOP_REFERENCE_BRANCH: "ivas-float-update" SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" PYTEST_ADDOPTS: "" diff --git a/main-basop.yml b/main-basop.yml index 003f2d50d0ca1d1a5489f277659ddef9569e2015..25eda0833e642a2b2c50120a6d129c23c6b46d22 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -428,11 +428,9 @@ workflow: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" # build merge target branch and correpsonding float reference - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh - - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" # build to-be-merged branch and corresponding float ref branch - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source @@ -851,6 +849,7 @@ branch-is-up-to-date-with-target-pre: tags: - ivas-basop-linux script: + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-is-up-to-date-with-target-pre.sh artifacts: paths: diff --git a/snippets/basop/activate-debug-mode-info-if-set.sh b/snippets/basop/activate-debug-mode-info-if-set.sh index 589d8e413ae33da40686ffd64d69b20fe6ec9103..261dbc07802497041ce360801d2d15dd035c8746 100755 --- a/snippets/basop/activate-debug-mode-info-if-set.sh +++ b/snippets/basop/activate-debug-mode-info-if-set.sh @@ -2,8 +2,6 @@ set -euxo pipefail -cd "${CI_PROJECT_DIR}" - if [ "$BUILD_WITH_DEBUG_MODE_INFO" = "true" ]; then sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUG_MODE_INFO\ *\)\*\//\1/g" lib_com/options.h diff --git a/snippets/basop/branch-is-up-to-date-with-target-pre.sh b/snippets/basop/branch-is-up-to-date-with-target-pre.sh index 08a0b8b2890a4be53ea2f4a75782305c3d3b164d..8ec1d8cb98d4d8f9b544143371725f788d380b96 100644 --- a/snippets/basop/branch-is-up-to-date-with-target-pre.sh +++ b/snippets/basop/branch-is-up-to-date-with-target-pre.sh @@ -30,42 +30,50 @@ set -euxo pipefail -float_ref_branchname="ivas-float-update" +float_ref_branchname_in_ivas_codec="main" exit_code_follows_naming_conventions=0 # if branch does not follow the convention, there will be a printout which is irrelevant here, so redirect it to null bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-follows-porting-naming-convention.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME >>/dev/null || exit_code_follows_naming_conventions=$? # if this branch follows the naming convetions, we can try to get the corresponding float MR branch # if this branch does NOT follow the porting naming convention, the float ref is just ivas-float-update, so skip everything if [[ "$exit_code_follows_naming_conventions" == "0" ]]; then - float_ref_branchname=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME) + float_ref_branchname_in_ivas_codec=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME) fi -# fetch all needed branches +# fetch merge target and check if we are up-to-date git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" -git fetch origin ivas-float-update -git fetch origin "$float_ref_branchname" -# then check if we are behind the merge target commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" if [ $commits_behind_count -ne 0 ]; then echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." exit 1 fi +# if we are up-to-date, store the commit SHA for all branches + +MERGE_TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")" +# to get the commit SHAs for the float reference branches, we need to get the main branch from float repo +# below here we will be in the float repo! +pushd "${SCRIPTS_DIR}" +git remote set-branches --add origin "$float_ref_branchname_in_ivas_codec" +git fetch origin "$float_ref_branchname_in_ivas_codec" + +MERGE_SOURCE_FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname_in_ivas_codec")" +FLOAT_REF_COMMIT="$(git rev-parse "origin/main")" +popd + # stop echoing from here on, this is anyway only dbg printout set +x -# if we are up-to-date, store the commit SHA's for all branches -MERGE_TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")" -echo "MERGE_TARGET_COMMIT is $MERGE_TARGET_COMMIT" +echo "CI_COMMIT_SHA (on this branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME) is $CI_COMMIT_SHA" + +echo "MERGE_TARGET_COMMIT (on branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME) is $MERGE_TARGET_COMMIT" echo "$MERGE_TARGET_COMMIT" >"$MERGE_TARGET_COMMIT_FILE" -MERGE_SOURCE_FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")" -echo "MERGE_SOURCE_FLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT" +echo "MERGE_SOURCE_FLOAT_REF_COMMIT (on branch $float_ref_branchname_in_ivas_codec in ivas-codec repo) is $MERGE_SOURCE_FLOAT_REF_COMMIT" echo "$MERGE_SOURCE_FLOAT_REF_COMMIT" >"$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" -FLOAT_REF_COMMIT="$(git rev-parse "origin/ivas-float-update")" -echo "FLOAT_REF_COMMIT is $FLOAT_REF_COMMIT" +echo "FLOAT_REF_COMMIT (on branch main in ivas-codec repo) is $FLOAT_REF_COMMIT" echo "$FLOAT_REF_COMMIT" >"$FLOAT_REF_COMMIT_FILE" # create commits.env file for passing the variables to other jobs diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index d1eb687d5a5b3a8c09e6fc96f829eb1db7d8fed8..7a0f23ba81f55a1eebafe18311e06fd9ae73aeda 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -2,13 +2,12 @@ set -euxo pipefail -cd "${CI_PROJECT_DIR}" - mode="${1:-}" case "$mode" in float-ref) ref_to_check_out="$FLOAT_REF_COMMIT" + working_dir="$SCRIPTS_DIR" ivas_cod_filename="./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -17,6 +16,7 @@ float-ref) ;; float-ref-merge-source) ref_to_check_out="$MERGE_SOURCE_FLOAT_REF_COMMIT" + working_dir="$SCRIPTS_DIR" ivas_cod_filename="./$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$MERGE_SOURCE_FLOAT_REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -25,6 +25,7 @@ float-ref-merge-source) ;; merge-target) ref_to_check_out="$MERGE_TARGET_COMMIT" + working_dir="$CI_PROJECT_DIR" ivas_cod_filename="./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$MERGE_TARGET_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -33,6 +34,7 @@ merge-target) ;; dut) ref_to_check_out="$CI_COMMIT_SHA" + working_dir="$CI_PROJECT_DIR" ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY" ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" @@ -47,7 +49,9 @@ esac echo "Building $mode binaries from branch $ref_to_check_out" -current_commit_sha="$(git rev-parse HEAD)" +# for float ref stuff, we need the ivas-codec repo which is persistently stored on the runners +# working_dir is set as per input argument +pushd "$working_dir" restore_stash() { if [[ "$(git stash list)" != "" ]]; then @@ -56,10 +60,14 @@ restore_stash() { } trap restore_stash EXIT +# the git SHA and stash restoring mechanisms is not really necessary for float-ref branches, but also should not harm, so no if here +current_commit_sha="$(git rev-parse HEAD)" + # store potential changes # should never fail, even if there is nothing to be stored git stash +git fetch origin "$ref_to_check_out" git checkout "$ref_to_check_out" bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh @@ -68,12 +76,18 @@ make clean make -j "$(nproc)" # avoid errors in mv when renaming to same file -mv IVAS_cod "$ivas_cod_filename" -mv IVAS_dec "$ivas_dec_filename" -mv IVAS_rend "$ivas_rend_filename" -if [ -f ISAR_post_rend ]; then mv ISAR_post_rend "$isar_post_rend_filename"; fi +mv IVAS_cod "${CI_PROJECT_DIR}/${ivas_cod_filename}" +mv IVAS_dec "${CI_PROJECT_DIR}/${ivas_dec_filename}" +mv IVAS_rend "${CI_PROJECT_DIR}/${ivas_rend_filename}" +mv ISAR_post_rend "${CI_PROJECT_DIR}/${isar_post_rend_filename}" # return to current branch git restore . git rev-parse HEAD >"$commit_file" git checkout "$current_commit_sha" + +# this used to be a TRAP for EXIT before, but now with the popd at the end, we might not be in the corresponding git repo anymore at EXIT +# we keep the trap to clean up incase of some error and additionally this is called manually here now +restore_stash + +popd diff --git a/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh b/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh index 9ed3ae846c3851819d5b5483593c51c50284bb6d..2af00333d23e026102dfd69ade63cdb4dd75f474 100644 --- a/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh +++ b/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh @@ -28,13 +28,13 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -set -euo pipefail - MSG_NO_FLOAT_REF_BRANCH_FOUND="Your branch name %s looks like it is a porting MR, but there is no corresponding float merge request (%s). If this is intended (nothing to port to ivas-float-update), simply ignore this warning. If there should be a float ref branch for this MR, please check your spelling on your other branch and make sure that you pushed it.\n " +set -euxo pipefail + exit_code_is_porting_mr=0 bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-follows-porting-naming-convention.sh "$CI_COMMIT_REF_NAME" || exit_code_is_porting_mr=$? @@ -43,10 +43,11 @@ if [ "$exit_code_is_porting_mr" == "1" ]; then exit 0 fi -# if this does indeed look like a porting branch, get the float companion branch - if that is ivas-float-update, then warn about possible typo/missing branch +# if this does indeed look like a porting branch, get the float companion branch - if that is the default, then warn about possible typo/missing branch float_ref_branchname=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_COMMIT_REF_NAME) -if [ "$float_ref_branchname" == "ivas-float-update" ]; then +if [ "$float_ref_branchname" == "main" ]; then expected_float_ref_branch="${CI_COMMIT_REF_NAME/basop/ref}" + set +x printf "$MSG_NO_FLOAT_REF_BRANCH_FOUND" "$CI_COMMIT_REF_NAME" "$expected_float_ref_branch" exit 123 fi diff --git a/snippets/basop/get-float-ref-branch-name.sh b/snippets/basop/get-float-ref-branch-name.sh index b1c20988bde549a9dcfde9d348acaa77a8677d6b..6ba19f93cf6d20df08ef0dcb8d7b86745d9a7345 100755 --- a/snippets/basop/get-float-ref-branch-name.sh +++ b/snippets/basop/get-float-ref-branch-name.sh @@ -28,29 +28,38 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. -# NOTE: this does not check e.g. for float_ref_branchname actually existing and will silently return ivas-float-update if not -# Use branch-follows-porting-naming-convention.sh for that - set -euo pipefail +# in this snippet, always print stuff to stderr and ONLY the final echo with of branchname_out should go to stdout +exec 3>&1 # save stdout on file desc 3 +exec 1>&2 # send all stdout to stderr + if [ $# -ne 1 ]; then echo "Usage: $0 " exit 1 fi branchname_in="$1" -branchname_out="ivas-float-update" +# Note: this is ivas-codec ("float") main +branchname_out="main" # The float ref branch is just the same name, but with basop -> ref replacement # Replace only the first occurrence, as "basop" may be present in the later description # If the format is correct, then before "_basop", only numbers can occur float_ref_branchname="${branchname_in/basop/ref}" -git_result=$(git branch -a) -# If the branch does not exist, default to "ivas-float-update" -if [[ "$git_result" =~ "$float_ref_branchname" ]]; then +# now check in ivas-codec repository if a branch with this name exists +exit_code_ls_remote=0 +pushd "${SCRIPTS_DIR}" +git fetch origin +git ls-remote --exit-code --heads origin refs/heads/$float_ref_branchname || exit_code_ls_remote=$? +popd + +# If the branch exists, we use it as ref name instead of main +# NOTE: "git ls-remote --exit-code" returns 2 if the given ref was not found +if [ "$exit_code_ls_remote" -eq "0" ]; then branchname_out="${float_ref_branchname}" fi -echo "$branchname_out" +printf '%s\n' "$branchname_out" >&3 exit 0 diff --git a/snippets/basop/update-scripts-repo.sh b/snippets/basop/update-scripts-repo.sh index 1a4a21d923427964ec9eb9d3c08ac7a262c285dd..47a1cb88143f8eff87effee899a5a6c72eaaf35c 100755 --- a/snippets/basop/update-scripts-repo.sh +++ b/snippets/basop/update-scripts-repo.sh @@ -6,11 +6,13 @@ cd "${CI_PROJECT_DIR}" pushd "${SCRIPTS_DIR}" - # Remove all fetch lines to clean out dead links +# Remove all fetch lines to clean out dead links sed -i '/fetch/d' .git/config -# Add currently used branch -git remote set-branches --add origin "$BASOP_CI_BRANCH_PC_REPO" +# Add currently used branches: +# - BASOP_CI_BRANCH_PC_REPO for scripts/tests +# - main for getting the float ref codec +git remote set-branches --add origin "$BASOP_CI_BRANCH_PC_REPO" main git fetch