Commit dfd266fe authored by Tapani Pihlajakuja's avatar Tapani Pihlajakuja
Browse files

Merge remote-tracking branch 'origin/main' into tjp-adjust-sanitizers

parents 2b2bf597 8e16d9be
Loading
Loading
Loading
Loading
+4 −0
Original line number Diff line number Diff line
variables:
  FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"
  MERGE_SOURCE_FLOAT_REF_COMMIT_FILE: "merge-source-float-ref-git-sha.txt"
  MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"
+33 −18
Original line number Diff line number Diff line
include:
  - local: main.yml
  - local: includes/basop-default-variables.yml

stages:
  - .pre
@@ -380,6 +381,7 @@ workflow:
    - if [ "$DISABLE_HRTF" = "true" ]; then
    -   export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'"
    - fi
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh
  variables:
    USE_LTV: 0

@@ -414,15 +416,13 @@ workflow:
    -   bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh
    - fi

    # set branches for reference builds
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/set-float-reference-branches.sh
    - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
    # build merge target branch and correpsonding float reference
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
    - echo "$FLOAT_REF_BRANCH - $FLOAT_REF_BRANCH_MERGE_SOURCE - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
    # build to-be-merged branch and corresponding float ref branch
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES
@@ -550,13 +550,11 @@ workflow:
    - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")
    - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"

    # set branches for reference builds
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/set-float-reference-branches.sh
    # build merge target branch and correpsonding float reference
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target
    # build to-be-merged branch and corresponding float ref branch
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source

    # set pytest args for ref creation
@@ -827,6 +825,11 @@ fail-pipeline-if-in-draft:
    -   bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
    - fi

# This job runs in MR pipelines and checks if your branch is up-to-date with your target
# If it is not, it fails the whole pipeline early (this should be the first job to run)
# It also stores the target branch's current commit as well as the float_ref branch's current commit
# to ensure that later jobs that need access to these branches all run on the same commit - which is
# important in case the branch is updated during the runtime of the pipeline
branch-is-up-to-date-with-target-pre:
  extends:
    - .job-linux
@@ -836,19 +839,14 @@ branch-is-up-to-date-with-target-pre:
  tags:
    - ivas-basop-linux
  script:
    - |
      commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)"
      if [ $commits_behind_count -ne 0 ]; then
        echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update."
        exit 1
      fi
    - git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
    - TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")"
    - echo "TARGET_COMMIT is $TARGET_COMMIT"
    - echo "$TARGET_COMMIT" > "$MERGE_TARGET_COMMIT_FILE"
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/branch-is-up-to-date-with-target-pre.sh
  artifacts:
    paths:
      - $MERGE_TARGET_COMMIT_FILE
      - $FLOAT_REF_COMMIT_FILE
      - $MERGE_SOURCE_FLOAT_REF_COMMIT_FILE
    reports:
      dotenv: commits.env

branch-is-up-to-date-with-target-post:
  extends:
@@ -1125,6 +1123,7 @@ check-regressions-short-enc-0db:
  needs:
    - job: "check-be-to-target-short-enc-0db"
      artifacts: true
    - !reference [.test-job-linux-compares-to-fixed-target, needs]
  variables:
    TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER"
    # +/-0dB
@@ -1140,6 +1139,7 @@ check-regressions-short-enc-+10db:
  needs:
    - job: "check-be-to-target-short-enc-+10db"
      artifacts: true
    - !reference [.test-job-linux-compares-to-fixed-target, needs]
  variables:
    TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER"
    # +10dB
@@ -1155,6 +1155,7 @@ check-regressions-short-enc--10db:
  needs:
    - job: "check-be-to-target-short-enc--10db"
      artifacts: true
    - !reference [.test-job-linux-compares-to-fixed-target, needs]
  variables:
    TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER"
    # -10dB
@@ -1170,6 +1171,7 @@ check-regressions-short-dec-0db:
  needs:
    - job: "check-be-to-target-short-dec-0db"
      artifacts: true
    - !reference [.test-job-linux-compares-to-fixed-target, needs]
  variables:
    TEST_SUITE: "$SHORT_TEST_SUITE"
    # +/-0dB
@@ -1185,6 +1187,7 @@ check-regressions-short-dec-+10db:
  needs:
    - job: "check-be-to-target-short-dec-+10db"
      artifacts: true
    - !reference [.test-job-linux-compares-to-fixed-target, needs]
  variables:
    TEST_SUITE: "$SHORT_TEST_SUITE"
    # +10dB
@@ -1200,6 +1203,7 @@ check-regressions-short-dec--10db:
  needs:
    - job: "check-be-to-target-short-dec--10db"
      artifacts: true
    - !reference [.test-job-linux-compares-to-fixed-target, needs]
  variables:
    TEST_SUITE: "$SHORT_TEST_SUITE"
    # -10dB
@@ -1228,6 +1232,7 @@ renderer-pytest-on-merge-request:
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh

    # some helper variables - "|| true" to prevent failures from grep not finding anything
    # write to temporary file as workaround for failures observed with piping echo
@@ -1300,6 +1305,7 @@ ivas-pytest-on-merge-request:
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh
    - python3 ci/remove_unsupported_testcases.py $PRM_FILES
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh

    # some helper variables - "|| true" to prevent failures from grep not finding anything
    # write to temporary file as workaround for failures observed with piping echo
@@ -1368,6 +1374,7 @@ split-rendering-pytest-on-merge-request:
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh

    # some helper variables - "|| true" to prevent failures from grep not finding anything
    # write to temporary file as workaround for failures observed with piping echo
@@ -1437,7 +1444,7 @@ codec-smoke-test:
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
    - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline
    - bash ci/smoke_test.sh || true
    - bash ci/smoke_test.sh || true
    - tar cJf logs.tar.xz out/logs/
    - ls -al logs.tar.*
    ### analyze for failures
@@ -1445,7 +1452,14 @@ codec-smoke-test:
    - ret_val=0
    - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi
    - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi
    - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi
    - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then
    -   if [ "$DISABLE_HRTF" = "false" ]; then
    -     echo "Smoke test with external hrtf files failed"
    -     ret_val=1
    -   else
    -     echo "Smoke test with external hrtf files failed, but DISABLE_HRTF is set. Ignoring failure."
    -   fi
    - fi
    - exit $ret_val
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
@@ -1518,6 +1532,7 @@ ivas-interop-on-merge-request:
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/check-commits-behind-count-in-compare-jobs.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
    - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

+51 −0
Original line number Diff line number Diff line
#!/bin/bash

# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
# contributors to this repository. All Rights Reserved.
#
# This software is protected by copyright law and by international treaties.
# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
# contributors to this repository retain full ownership rights in their respective contributions in
# the software. This notice grants no license of any kind, including but not limited to patent
# license, nor is any license granted by implication, estoppel or otherwise.
#
# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
# contributions.
#
# This software is provided "AS IS", without any express or implied warranties. The software is in the
# development stage. It is intended exclusively for experts who have experience with such software and
# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
# and fitness for a particular purpose are hereby disclaimed and excluded.
#
# Any dispute, controversy or claim arising under or in relation to providing this software shall be
# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
# the United Nations Convention on Contracts on the International Sales of Goods.

BASOP_PATTERN="^[0-9]+[_-]basop[_-].*"
MSG_DOES_NOT_MATCH="Your branch name %s does not match the template '<issue_number>_basop_<description>', e.g. '123_basop_fix_this_one_bug-2'.
If this branch is porting a float MR with a corresponding float part, then
ticking to this branch is needed for the testing system to match this branch with its float-reference counterpart.
Please rename your branch. You can easily do this by creating a new branch from this branch:
  - git checkout -b <new branch name here>
You then also need to create a new merge request and update the links in your issue.
See here for details on the porting work process: https://forge.3gpp.org/rep/sa4/audio/ivas-basop/-/wikis/Porting-MRs-from-floating-point-codec#workflow-for-porting-a-merge-request-from-floating-point-codec-to-basop-codec

If this is not a porting branch, ignore above message.
"

if [ $# -ne 1 ]; then
  echo "Usage: $0 <branchname>"
  exit 1
fi

if ! [[ "$1" =~ $BASOP_PATTERN ]]; then
  printf "$MSG_DOES_NOT_MATCH" "$1"
  exit 1
fi
+60 −0
Original line number Diff line number Diff line
#!/bin/bash

# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
# contributors to this repository. All Rights Reserved.
#
# This software is protected by copyright law and by international treaties.
# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
# contributors to this repository retain full ownership rights in their respective contributions in
# the software. This notice grants no license of any kind, including but not limited to patent
# license, nor is any license granted by implication, estoppel or otherwise.
#
# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
# contributions.
#
# This software is provided "AS IS", without any express or implied warranties. The software is in the
# development stage. It is intended exclusively for experts who have experience with such software and
# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
# and fitness for a particular purpose are hereby disclaimed and excluded.
#
# Any dispute, controversy or claim arising under or in relation to providing this software shall be
# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
# the United Nations Convention on Contracts on the International Sales of Goods.

# fetch all needed branches
git fetch origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
git fetch origin ivas-float-update
float_ref_branchname=$(${CI_PROJECT_DIR}/ivas-codec-ci/snippets/basop/get-float-ref-branch-name.sh $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME)
git fetch origin "$float_ref_branchname"

# then check if we are behind the merge target
commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)"
if [ $commits_behind_count -ne 0 ]; then
  echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update."
  exit 1
fi

# if we are up-to-date, store the commit SHA's for all branches
MERGE_TARGET_COMMIT="$(git rev-parse "origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME")"
echo "MERGE_TARGET_COMMIT is $MERGE_TARGET_COMMIT"
echo "$MERGE_TARGET_COMMIT" >"$MERGE_TARGET_COMMIT_FILE"

MERGE_SOURCE_FLOAT_REF_COMMIT="$(git rev-parse "origin/$float_ref_branchname")"
echo "MERGE_SOURCE_FLOAT_REF_COMMIT is $MERGE_SOURCE_FLOAT_REF_COMMIT"
echo "$MERGE_SOURCE_FLOAT_REF_COMMIT" >"$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE"

FLOAT_REF_COMMIT="$(git rev-parse "origin/ivas-float-update")"
echo "FLOAT_REF_COMMIT is $FLOAT_REF_COMMIT"
echo "$FLOAT_REF_COMMIT" >"$FLOAT_REF_COMMIT_FILE"

# create commits.env file for passing the variables to other jobs
echo "MERGE_TARGET_COMMIT=$MERGE_TARGET_COMMIT" >>commits.env
echo "MERGE_SOURCE_FLOAT_REF_COMMIT=$MERGE_SOURCE_FLOAT_REF_COMMIT" >>commits.env
echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" >>commits.env
+6 −16
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ mode="${1:-}"

case "$mode" in
float-ref)
  branch_to_check_out="$FLOAT_REF_BRANCH"
  ref_to_check_out="$FLOAT_REF_COMMIT"
  ivas_cod_filename="./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_dec_filename="./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_rend_filename="./$REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
@@ -16,7 +16,7 @@ float-ref)
  commit_file="$FLOAT_REF_COMMIT_FILE"
  ;;
float-ref-merge-source)
  branch_to_check_out="$FLOAT_REF_BRANCH_MERGE_SOURCE"
  ref_to_check_out="$MERGE_SOURCE_FLOAT_REF_COMMIT"
  ivas_cod_filename="./$MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_dec_filename="./$MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_rend_filename="./$MERGE_SOURCE_FLOAT_REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
@@ -24,25 +24,15 @@ float-ref-merge-source)
  commit_file="$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE"
  ;;
merge-target)
  branch_to_check_out="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
  ref_to_check_out="$MERGE_TARGET_COMMIT"
  ivas_cod_filename="./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_dec_filename="./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_rend_filename="./$MERGE_TARGET_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  isar_post_rend_filename="./$MERGE_TARGET_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  commit_file="$MERGE_TARGET_COMMIT_FILE"
  ;;
merge-source)
  branch_to_check_out="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
  ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  commit_file="$CUT_COMMIT_FILE"
  ;;
dut)
  # This produces the same binary names as merge-source, but sets branch_to_check_out differently
  # so that this is usable in non-merge-request pipelines
  branch_to_check_out="$CI_COMMIT_REF_NAME"
  ref_to_check_out="$CI_COMMIT_SHA"
  ivas_cod_filename="./$DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_dec_filename="./$DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY"
  ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY"
@@ -55,7 +45,7 @@ dut)
  ;;
esac

echo "Building $mode binaries from branch $branch_to_check_out"
echo "Building $mode binaries from branch $ref_to_check_out"

current_commit_sha="$(git rev-parse HEAD)"

@@ -70,7 +60,7 @@ trap restore_stash EXIT
# should never fail, even if there is nothing to be stored
git stash

git checkout "$branch_to_check_out"
git checkout "$ref_to_check_out"

bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh

Loading