From 9d3cd06e17d657aff8576eb3a65c9ecc49357a9b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Nov 2025 16:47:56 +0100 Subject: [PATCH 1/9] use test_renderer_short.py --- main-basop.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 30b1c4e..00937a2 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1296,13 +1296,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer_short.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer_short.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1679,7 +1679,7 @@ renderer-smoke-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - make -j IVAS_rend - testcase_timeout=60 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1705,7 +1705,7 @@ renderer-asan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1734,7 +1734,7 @@ renderer-msan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1763,11 +1763,11 @@ renderer-usan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? - - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer_short.py with Clang undefined-behavior-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" -- GitLab From 73de3ef4e2548ed3d08ee77f5e1fff4bee981deb Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Nov 2025 16:55:38 +0100 Subject: [PATCH 2/9] Revert "use test_renderer_short.py" This reverts commit 9d3cd06e17d657aff8576eb3a65c9ecc49357a9b. --- main-basop.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 00937a2..30b1c4e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1296,13 +1296,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer_short.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer_short.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1679,7 +1679,7 @@ renderer-smoke-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - make -j IVAS_rend - testcase_timeout=60 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1705,7 +1705,7 @@ renderer-asan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1734,7 +1734,7 @@ renderer-msan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1763,11 +1763,11 @@ renderer-usan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_short.py --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? - - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer_short.py with Clang undefined-behavior-sanitizer"; exit 1; fi + - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" -- GitLab From a76ddac0e8994b6a28495b032f1ceb52213c896f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Nov 2025 16:56:31 +0100 Subject: [PATCH 3/9] use renderer_short test in correct way --- main-basop.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index 30b1c4e..ac88250 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1296,13 +1296,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1679,7 +1679,7 @@ renderer-smoke-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - make -j IVAS_rend - testcase_timeout=60 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -1705,7 +1705,7 @@ renderer-asan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1734,7 +1734,7 @@ renderer-msan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" @@ -1763,7 +1763,7 @@ renderer-usan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=usan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - testcase_timeout=180 - - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$? -- GitLab From f5b202cb006b7fb78d9081cdaa3b1a27f5d1c3f4 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 20 Nov 2025 11:29:09 +0100 Subject: [PATCH 4/9] [revertme] profile renderer pytest testcase durations --- main-basop.yml | 610 ++++++++++++++++++++++++------------------------- 1 file changed, 301 insertions(+), 309 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index ac88250..ea1aad9 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -15,77 +15,76 @@ stages: - postvalidate - deploy - workflow: - name: '$IVAS_PIPELINE_NAME' + name: "$IVAS_PIPELINE_NAME" rules: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: - IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main # variables: # IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: - IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs' variables: - IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Aggregate long term logs: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' variables: - IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Backup long term logs: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' variables: - IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Run comparison tools against float ref: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx' variables: - IVAS_PIPELINE_NAME: 'Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' variables: - IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long-fx-fx' variables: - IVAS_PIPELINE_NAME: 'Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input' variables: - IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' variables: - IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Run saturation smoke-test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' variables: - IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "EVS 26.444 test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer' variables: - IVAS_PIPELINE_NAME: 'Long testvectors sanitizers' + IVAS_PIPELINE_NAME: "Long testvectors sanitizers" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer' variables: - IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Renderer test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity' variables: - IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Complexity Measurement on $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test' variables: - IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Voip BE test on $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'renderer-framesize-be' variables: - IVAS_PIPELINE_NAME: 'Renderer framesize BE test on $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Renderer framesize BE test on $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' variables: - IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "PEAQ encoder pass-through test: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: - IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' variables: - IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "IVAS Conformance Linux: $CI_COMMIT_BRANCH" - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: - IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH" # --------------------------------------------------------------- # Generic script anchors @@ -249,7 +248,7 @@ workflow: .test-job-linux-needs-testv-dir: extends: .test-job-linux before_script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -270,31 +269,31 @@ workflow: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - FLOAT_REF_COMMIT=$(cat "$FLOAT_REF_COMMIT_FILE") - echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" - if [ $LEVEL_SCALING != "1.0" ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - - BUILD_WITH_DEBUG_MODE_INFO="true" + - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") @@ -309,7 +308,7 @@ workflow: # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then - - comp_args="--compare_enc_dmx" + - comp_args="--compare_enc_dmx" - fi - echo "$comp_args" @@ -324,9 +323,9 @@ workflow: ### create histograms - if [ "$SPLIT_COMPARISON" = "true" ]; then - - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT - else - - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - fi # first for "whole" files comparison @@ -334,35 +333,35 @@ workflow: - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - if [ "$SPLIT_COMPARISON" = "true" ]; then - - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT + - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT - else # touch files to suppress warning for missing artifacts - - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT + - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT - fi - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - - unzip artifacts.zip -d previous_artifacts + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts # This wildcard thingy relies on only one csv file being present per job - - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" - - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME + - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" + - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME - else # create empty file for artifacts to avoid errors - - touch $PAGES_HTML_ARTIFACT_NAME - - touch $MERGED_CSV_ARTIFACT_NAME + - touch $PAGES_HTML_ARTIFACT_NAME + - touch $MERGED_CSV_ARTIFACT_NAME - fi - if [ $zero_errors != 1 ]; then - - echo "Run errors encountered!" + - echo "Run errors encountered!" # TODO: temporary only to not fail MR pipelines on crashes - - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - - exit $EXIT_CODE_NON_BE - - else - - exit $EXIT_CODE_FAIL - - fi + - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then + - exit $EXIT_CODE_NON_BE + - else + - exit $EXIT_CODE_FAIL + - fi - fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 @@ -396,10 +395,10 @@ workflow: - .rules-merge-request-to-main-no-draft - .test-job-linux-compares-to-fixed-target before_script: - - !reference [ .test-job-linux, before_script ] + - !reference [.test-job-linux, before_script] - rm -rf tests/dut tests/ref - if [ "$DISABLE_HRTF" = "true" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh variables: @@ -423,17 +422,17 @@ workflow: - python3 tests/create_short_testvectors.py - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - python3 scripts/prepare_combined_format_inputs.py - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi # TODO: remove once ref branch is switched @@ -456,39 +455,39 @@ workflow: - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - - exit_code=$EXIT_CODE_NON_BE - - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache - - - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true - - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true - - num_failures=$(wc -l < failed_testcases_for_printing.txt) - - - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true - - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true - - num_errors=$(wc -l < errors_testcases_for_printing.txt) - - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - - echo "Found these $num_failures non-bitexact testcases:" - - cat failed_testcases_for_printing.txt - - - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - - echo "The individual command lines can be found in the html report in the job artifacts." - - if [ $num_errors -ne 0 ]; then - - exit_code=1 - - echo "There were errors present in the following testcases:" - - cat errors_testcases_for_printing.txt - - fi - - - exit $exit_code + - exit_code=$EXIT_CODE_NON_BE + - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true + - num_failures=$(wc -l < failed_testcases_for_printing.txt) + + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true + - num_errors=$(wc -l < errors_testcases_for_printing.txt) + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - echo "Found these $num_failures non-bitexact testcases:" + - cat failed_testcases_for_printing.txt + + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the html report in the job artifacts." + - if [ $num_errors -ne 0 ]; then + - exit_code=1 + - echo "There were errors present in the following testcases:" + - cat errors_testcases_for_printing.txt + - fi + + - exit $exit_code - else # create empty files to not have errors at artifact stage - - touch $FAILED_TESTCASES_LIST - - touch $ERRORS_TESTCASES_LIST - - touch $PYTEST_CACHE_ARTIFACT - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - - echo "All testcases are bitexact." + - touch $FAILED_TESTCASES_LIST + - touch $ERRORS_TESTCASES_LIST + - touch $PYTEST_CACHE_ARTIFACT + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - echo "All testcases are bitexact." - fi - exit $exit_code @@ -541,27 +540,27 @@ workflow: - set -euxo pipefail - if [ -s $FAILED_TESTCASES_LIST ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh - - export PYTEST_ADDOPTS=--last-failed + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh + - export PYTEST_ADDOPTS=--last-failed - else # turn off echoing back of commands for result printout - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." - - exit 0 + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." + - exit 0 - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi # TODO: remove once ref branch is switched @@ -584,11 +583,11 @@ workflow: # set pytest args for ref creation - enc_stats_arg="" - if [ "$ENCODER_TEST" = "true" ]; then - - enc_stats_arg="--enc_stats" + - enc_stats_arg="--enc_stats" - fi - enc_dmx_arg="" - if [ "$COMPARE_DMX" = "true" ]; then - - enc_dmx_arg="--compare_enc_dmx" + - enc_dmx_arg="--compare_enc_dmx" - fi ### ----- run pytest for branch (dut) first ----- @@ -621,25 +620,25 @@ workflow: - exit_code=0 - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - if [ $zero_errors_branch != 1 ]; then - - echo "Run errors encountered!" - - exit_code=$EXIT_CODE_FAIL - - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - - echo "The individual command lines can be found in the regressions_crashes.csv files in the job artifacts." + - echo "Run errors encountered!" + - exit_code=$EXIT_CODE_FAIL + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the regressions_crashes.csv files in the job artifacts." - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then - - cat regression_log.txt - - if [ $allow_regressions_flag == 0 ]; then - - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" - - exit_code=$EXIT_CODE_FAIL; - - else - - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." - - exit_code=$EXIT_CODE_NON_BE; - - fi - - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - - echo "The individual command lines can be found in the regressions_*.csv files in the job artifacts." + - cat regression_log.txt + - if [ $allow_regressions_flag == 0 ]; then + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" + - exit_code=$EXIT_CODE_FAIL; + - else + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." + - exit_code=$EXIT_CODE_NON_BE; + - fi + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the regressions_*.csv files in the job artifacts." - fi - exit $exit_code @@ -681,14 +680,13 @@ workflow: - $XML_REPORT_BRANCH - $XML_REPORT_MAIN - .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor extends: - .job-linux stage: test needs: ["build-codec-linux-make"] script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -697,20 +695,20 @@ workflow: - set -euxo pipefail - make_args="CLANG=$CLANG_NUM" - if [[ $CLANG_NUM == 3 ]]; then - - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make_args="$make_args IGNORELIST=1" + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" - fi - make clean - make -j $make_args >> /dev/null - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS" # disable per-testcase timeout for msan to evaluate what is going on that it takes so long - if [[ $CLANG_NUM = 1 ]]; then - - testcase_timeout_arg="" + - testcase_timeout_arg="" - fi - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) @@ -734,20 +732,20 @@ workflow: SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut @@ -843,8 +841,8 @@ check-float-ref-companion-branch-exists-for-porting-merge-requests: tags: - ivas-basop-linux script: - # we need to source the script here so that the correct exit code is returned from the global job shell and the "allow_failure" recognizes it - - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh + # we need to source the script here so that the correct exit code is returned from the global job shell and the "allow_failure" recognizes it + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh allow_failure: exit_codes: - 123 @@ -937,7 +935,6 @@ clang-format-check: name: "$ARTIFACT_BASE_NAME" expose_as: "formatting patch" - # This job runs in manual jobs and scheduled jobs that compares to the float reference. # It fetches the latest commit of ivas-float-update and stores in FLOAT_COMMIT_REF pytest-to-ref-pre: @@ -1017,9 +1014,9 @@ build-codec-linux-instrumented-make: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ] || [ "$CI_COMMIT_REF_NAME" == "ivas-float-update" ]; then - - bash scripts/prepare_instrumentation.sh + - bash scripts/prepare_instrumentation.sh - else - - bash scripts/prepare_instrumentation.sh -m MEM_ONLY + - bash scripts/prepare_instrumentation.sh -m MEM_ONLY - fi - make -j -C $INSTR_DIR @@ -1296,13 +1293,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout --durations=0 || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout --durations=0 || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1367,11 +1364,11 @@ ivas-pytest-on-merge-request: - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi - if [ "$DISABLE_HRTF" = "true" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi ### prepare pytest @@ -1501,12 +1498,12 @@ codec-smoke-test: - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then - - if [ "$DISABLE_HRTF" = "false" ]; then - - echo "Smoke test with external hrtf files failed" - - ret_val=1 - - else - - echo "Smoke test with external hrtf files failed, but DISABLE_HRTF is set. Ignoring failure." - - fi + - if [ "$DISABLE_HRTF" = "false" ]; then + - echo "Smoke test with external hrtf files failed" + - ret_val=1 + - else + - echo "Smoke test with external hrtf files failed, but DISABLE_HRTF is set. Ignoring failure." + - fi - fi - exit $ret_val artifacts: @@ -1564,7 +1561,6 @@ check-first-frame-is-sid: expose_as: "logs-hrtf-loading" expire_in: "5 days" - ### From Float # Check interop IVAS_cod_test -> IVAS_dec_ref ivas-interop-on-merge-request: @@ -1583,7 +1579,7 @@ ivas-interop-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -1661,9 +1657,9 @@ lc3-wrapper-unit-test: - echo "Assuming here that you either merge to main or to $FLOAT_REF_BRANCH." - echo "If you have some other merge target, this might fail and manual fiddling is needed." - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "$FLOAT_REF_BRANCH" ]; then - - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test - else - - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test + - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test - fi # from float @@ -1726,7 +1722,7 @@ renderer-msan: - .rules-merge-request-to-float-ref-no-draft needs: ["build-codec-linux-cmake"] stage: test - tags: + tags: - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -1755,7 +1751,7 @@ renderer-usan: - .rules-merge-request-to-float-ref-no-draft needs: ["build-codec-linux-cmake"] stage: test - tags: + tags: - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -1808,56 +1804,56 @@ renderer-usan: # to be reused in MR and LTV-scheduled sanitizer test jobs # set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section .sanitizer-selftest-anchor: &sanitizer-selftest-anchor - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 + + - if [ "$DISABLE_HRTF" = "true" ]; then + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - fi - - if [ "$DISABLE_HRTF" = "true" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - - fi + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - fi - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - - fi + - make_args="CLANG=$CLANG_NUM" + # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) + # shorten files to 1s for that case + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then + - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all + + # also need different setup for ubsan + - if [[ $CLANG_NUM == 3 ]]; then + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" + - fi + - fi - - make_args="CLANG=$CLANG_NUM" - # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) - # shorten files to 1s for that case - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all - - # also need different setup for ubsan - - if [[ $CLANG_NUM == 3 ]]; then - - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make_args="$make_args IGNORELIST=1" - - fi - - fi + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 - - exit_code20=0 - - exit_code10=0 - - exit_code5=0 + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + - make clean + - make -j $make_args - - make clean - - make -j $make_args - - # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? - - # check different renderer fremsizes only for float - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then - # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option - # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active - # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - - fi + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + + # check different renderer fremsizes only for float + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - fi - - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # from float repo # NOTE: (per-testcase) timeout was adjusted to work for float only @@ -1985,29 +1981,29 @@ ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: ivas-pytest-compare_to_ref-short-dec: extends: - .rules-pytest-to-ref-short - - .test-job-linux + - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor - + ivas-pytest-compare_to_ref-short-dec-lev-10: extends: - .rules-pytest-to-ref-short - - .test-job-linux + - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor - + ivas-pytest-compare_to_ref-short-dec-lev+10: extends: - .rules-pytest-to-ref-short - - .test-job-linux + - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH @@ -2053,7 +2049,7 @@ ivas-pytest-compare_ref-long-enc: - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-dec: extends: - .rules-pytest-long @@ -2062,13 +2058,13 @@ ivas-pytest-compare_ref-long-dec: - ivas-basop-linux - high-memory-capacity before_script: - - USE_LTV=1 + - USE_LTV=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-enc-lev-10: extends: - .rules-pytest-long @@ -2080,7 +2076,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: - LEVEL_SCALING=0.3162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-dec-lev-10: extends: - .rules-pytest-long @@ -2107,7 +2103,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: - LEVEL_SCALING=3.162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-dec-lev+10: extends: - .rules-pytest-long @@ -2116,7 +2112,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: - ivas-basop-linux - high-memory-capacity before_script: - - USE_LTV=1 + - USE_LTV=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 @@ -2174,12 +2170,12 @@ ivas-pytest-msan: - ivas-basop-linux-fast resource_group: basop-long-sanitizers rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" timeout: "24 hours" before_script: - CLANG_NUM=1 @@ -2193,12 +2189,12 @@ ivas-pytest-asan: - ivas-basop-linux-fast resource_group: basop-long-sanitizers rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" timeout: "6 hours" before_script: - CLANG_NUM=2 @@ -2212,12 +2208,12 @@ ivas-pytest-usan: - ivas-basop-linux-fast resource_group: basop-long-sanitizers rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" timeout: "6 hours" before_script: - CLANG_NUM=3 @@ -2237,7 +2233,7 @@ test-long-self-test: - 123 parallel: matrix: - - LEVEL: + - LEVEL: - 0dB - minus10dB - plus10dB @@ -2249,20 +2245,20 @@ test-long-self-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - - case $LEVEL in - "0dB") - LEVEL_SCALING=1.0 - ;; - "minus10dB") - LEVEL_SCALING=0.3162 - ;; - "plus10dB") - LEVEL_SCALING=3.162 - ;; + - case $LEVEL in + "0dB") + LEVEL_SCALING=1.0 + ;; + "minus10dB") + LEVEL_SCALING=0.3162 + ;; + "plus10dB") + LEVEL_SCALING=3.162 + ;; esac - if [ $LEVEL_SCALING != "1.0" ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") @@ -2290,7 +2286,6 @@ test-long-self-test: - make clean - make -j - ### prepare pytest # create references @@ -2306,7 +2301,6 @@ test-long-self-test: - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 - artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -2330,11 +2324,11 @@ ivas-smoke-test-saturation: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - fi - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -2361,7 +2355,6 @@ ivas-smoke-test-saturation: - smoke_test_output_hrtf.txt expose_as: "saturation smoke test results" - # GCOV/LCOV coverage analysis of self_test suite coverage-test-on-main-scheduled: extends: @@ -2431,7 +2424,7 @@ be-2-evs-26444: - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi - exit 0 - + artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" expire_in: 1 week @@ -2563,7 +2556,7 @@ ivas-long-term-job-logs: stage: maintenance timeout: "25 minutes" script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - set -euxo pipefail - echo "Running on RUNNER_TAG = $RUNNER_TAG" - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -2573,26 +2566,26 @@ ivas-long-term-job-logs: - if [ -d "logs" ]; then rm -rf "logs"; fi # Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set - if [ "$LOGS_BACKUP_SOURCE_DIR" != "" ]; then - - cp_ret=0 - - cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ || cp_ret=$? - - if [ "$cp_ret" != 0 ]; then - - echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!" - - exit 1 - - fi + - cp_ret=0 + - cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ || cp_ret=$? + - if [ "$cp_ret" != 0 ]; then + - echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!" + - exit 1 + - fi - fi - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID) - echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous" - if [ "$id_previous" != "-1" ]; then - # Unzip artifacts to recover past logs dir - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - # If there is overlap between injected log and new log, use the new logs. - - unzip -o artifacts.zip + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + # If there is overlap between injected log and new log, use the new logs. + - unzip -o artifacts.zip - else - # create logs dir if it doesn't exist (should only happen in first run) - - echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!" - - mkdir logs + # create logs dir if it doesn't exist (should only happen in first run) + - echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!" + - mkdir logs - fi # create folder for today's results @@ -2602,14 +2595,14 @@ ivas-long-term-job-logs: # Aggregate job logs - job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10" - for job_name in $job_names; do - - echo "Getting job logs for $job_name" - - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID) - - echo "Job ID from variables - "$job_name", Job ID from script - $id_previous" - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - - unzip artifacts.zip -d previous_artifacts - - mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY - - rm artifacts.zip - - rm -r previous_artifacts + - echo "Getting job logs for $job_name" + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID) + - echo "Job ID from variables - "$job_name", Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts + - mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY + - rm artifacts.zip + - rm -r previous_artifacts - done - ls logs - exit 0 @@ -2633,7 +2626,7 @@ ivas-long-term-job-logs-overview: stage: maintenance timeout: "25 minutes" script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -2656,20 +2649,19 @@ ivas-long-term-regressions: stage: maintenance timeout: "25 minutes" script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh # Find regressions - for MEASURE in MLD MAX_ABS_DIFF; do - - python3 scripts/find_regressions_from_logs.py logs regressions_"$MEASURE".csv --measure "$MEASURE" --days 7 --ratio_thr 1.1 - - mkdir regressions_"$MEASURE" - - python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv - - mv regression*bash regressions_"$MEASURE" + - python3 scripts/find_regressions_from_logs.py logs regressions_"$MEASURE".csv --measure "$MEASURE" --days 7 --ratio_thr 1.1 + - mkdir regressions_"$MEASURE" + - python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv + - mv regression*bash regressions_"$MEASURE" - done - artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" expire_in: 4 weeks @@ -2695,7 +2687,7 @@ backup-long-term-job-logs: stage: maintenance timeout: "25 minutes" script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - set -euxo pipefail - echo "Running on RUNNER_TAG = $RUNNER_TAG" - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -2705,31 +2697,30 @@ backup-long-term-job-logs: - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID) - if [ "$id_previous" != "-1" ]; then - # Unzip artifacts to recover past logs dir - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - - unzip artifacts.zip + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip - else - # create logs dir if it doesn't exist - - mkdir logs + # create logs dir if it doesn't exist + - mkdir logs - fi # Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set - if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then - - cp -r logs $LOGS_BACKUP_TARGET_DIR - - exit 0 + - cp -r logs $LOGS_BACKUP_TARGET_DIR + - exit 0 - else - - echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!" - - exit 1 + - echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!" + - exit 1 - fi - # --------------------------------------------------------------- # Complexity measurement jobs # --------------------------------------------------------------- # create necessary environment -.complexity-measurements-setup: &complexity-measurements-setup - ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work +.complexity-measurements-setup: + &complexity-measurements-setup ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work - mkdir -p wmops/logs - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) @@ -2745,20 +2736,20 @@ backup-long-term-job-logs: # 1. check for public_dir being there as this might not be the case when artifact download failed # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then - - mv $public_dir/* wmops/ + - mv $public_dir/* wmops/ # check here if we have the split-by-levels files present - if not, fake them up with the existing global one # this is needed for the first run with split graphs on a branch where the global version did run previously # NOTE: checking only for level_1 file here as this should already be sufficient # NOTE2: also not chechking for RAM for same reason - - wmops_all_global="wmops/log_wmops_all.txt" - - ram_all_global="wmops/log_ram_all.txt" - - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then - - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") - - for suffix in "${suffixes[@]}"; do - - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt - - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt - - done - - fi + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi - fi - rm artifacts.zip @@ -2767,16 +2758,17 @@ backup-long-term-job-logs: ### 1.5.part: get the corresponding measurement from ivas-float-update # only do this on main and not for the complexity meaurements run on ivas-float-update - if [ "$CI_COMMIT_REF_NAME" == "main" ]; then - - job_id=$(python3 ci/get_id_of_last_job_occurence.py $FLOAT_REF_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - - echo $job_id - - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $FLOAT_REF_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip + - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi + - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi - fi # prepare artifacts -> move to public directory -.complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts +.complexity-measurements-prepare-artifacts: + &complexity-measurements-prepare-artifacts - public_dir="$CI_JOB_NAME-public" - mkdir $public_dir - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ @@ -2796,11 +2788,11 @@ backup-long-term-job-logs: .complexity-measurements-report-summary: &complexity-measurements-report-summary - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - if [ $ret_val -eq 0 ]; then - - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." + - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." - elif [ $ret_val -eq 123 ]; then - - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." + - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." - else - - echo -e "Something went wrong in running the codec. Likely some modes were crashing." + - echo -e "Something went wrong in running the codec. Likely some modes were crashing." - fi .complexity-template: @@ -2812,7 +2804,7 @@ backup-long-term-job-logs: GET_WMOPS_ARGS: "mem_only" timeout: 3 hours 30 minutes before_script: - - !reference [ .test-job-linux, before_script ] + - !reference [.test-job-linux, before_script] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh @@ -3069,8 +3061,8 @@ complexity-masa-in-hoa3-out: # - if: $MEASURE_COMPLEXITY_LINUX # when: delayed # start_in: 13 hours - # variables: - # JOB_ID_INJECT: "" +# variables: +# JOB_ID_INJECT: "" # script: # - in_format=OMASA # - out_format=EXT @@ -3236,7 +3228,7 @@ pages: rules: - if: $UPDATE_PAGES script: - - !reference [ .job-linux, before_script ] + - !reference [.job-linux, before_script] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - python3 ci/setup_pages.py -- GitLab From 238a9a43dfae6724b13d23380b54990230f07b6c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Nov 2025 13:43:24 +0100 Subject: [PATCH 5/9] Revert "[revertme] profile renderer pytest testcase durations" This reverts commit f5b202cb006b7fb78d9081cdaa3b1a27f5d1c3f4. --- main-basop.yml | 610 +++++++++++++++++++++++++------------------------ 1 file changed, 309 insertions(+), 301 deletions(-) diff --git a/main-basop.yml b/main-basop.yml index ea1aad9..ac88250 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -15,76 +15,77 @@ stages: - postvalidate - deploy + workflow: - name: "$IVAS_PIPELINE_NAME" + name: '$IVAS_PIPELINE_NAME' rules: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: - IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" + IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main # variables: # IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: - IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs' variables: - IVAS_PIPELINE_NAME: "Aggregate long term logs: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' variables: - IVAS_PIPELINE_NAME: "Backup long term logs: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' variables: - IVAS_PIPELINE_NAME: "Run comparison tools against float ref: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx' variables: - IVAS_PIPELINE_NAME: "Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' variables: - IVAS_PIPELINE_NAME: "Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long-fx-fx' variables: - IVAS_PIPELINE_NAME: "Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input' variables: - IVAS_PIPELINE_NAME: "Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' variables: - IVAS_PIPELINE_NAME: "Run saturation smoke-test: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' variables: - IVAS_PIPELINE_NAME: "EVS 26.444 test: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer' variables: - IVAS_PIPELINE_NAME: "Long testvectors sanitizers" + IVAS_PIPELINE_NAME: 'Long testvectors sanitizers' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer' variables: - IVAS_PIPELINE_NAME: "Renderer test: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity' variables: - IVAS_PIPELINE_NAME: "Complexity Measurement on $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test' variables: - IVAS_PIPELINE_NAME: "Voip BE test on $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'renderer-framesize-be' variables: - IVAS_PIPELINE_NAME: "Renderer framesize BE test on $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Renderer framesize BE test on $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' variables: - IVAS_PIPELINE_NAME: "PEAQ encoder pass-through test: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: - IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' variables: - IVAS_PIPELINE_NAME: "IVAS Conformance Linux: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch variables: - IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH" + IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' # --------------------------------------------------------------- # Generic script anchors @@ -248,7 +249,7 @@ workflow: .test-job-linux-needs-testv-dir: extends: .test-job-linux before_script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -269,31 +270,31 @@ workflow: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - FLOAT_REF_COMMIT=$(cat "$FLOAT_REF_COMMIT_FILE") - echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT" - if [ $LEVEL_SCALING != "1.0" ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - - BUILD_WITH_DEBUG_MODE_INFO="true" + - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") @@ -308,7 +309,7 @@ workflow: # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then - - comp_args="--compare_enc_dmx" + - comp_args="--compare_enc_dmx" - fi - echo "$comp_args" @@ -323,9 +324,9 @@ workflow: ### create histograms - if [ "$SPLIT_COMPARISON" = "true" ]; then - - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT - else - - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME + - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - fi # first for "whole" files comparison @@ -333,35 +334,35 @@ workflow: - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - if [ "$SPLIT_COMPARISON" = "true" ]; then - - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms - - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT + - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms + - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT - else # touch files to suppress warning for missing artifacts - - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT + - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT - fi - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - - unzip artifacts.zip -d previous_artifacts + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts # This wildcard thingy relies on only one csv file being present per job - - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" - - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME + - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" + - python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME - else # create empty file for artifacts to avoid errors - - touch $PAGES_HTML_ARTIFACT_NAME - - touch $MERGED_CSV_ARTIFACT_NAME + - touch $PAGES_HTML_ARTIFACT_NAME + - touch $MERGED_CSV_ARTIFACT_NAME - fi - if [ $zero_errors != 1 ]; then - - echo "Run errors encountered!" + - echo "Run errors encountered!" # TODO: temporary only to not fail MR pipelines on crashes - - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - - exit $EXIT_CODE_NON_BE - - else - - exit $EXIT_CODE_FAIL - - fi + - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then + - exit $EXIT_CODE_NON_BE + - else + - exit $EXIT_CODE_FAIL + - fi - fi - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 @@ -395,10 +396,10 @@ workflow: - .rules-merge-request-to-main-no-draft - .test-job-linux-compares-to-fixed-target before_script: - - !reference [.test-job-linux, before_script] + - !reference [ .test-job-linux, before_script ] - rm -rf tests/dut tests/ref - if [ "$DISABLE_HRTF" = "true" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh variables: @@ -422,17 +423,17 @@ workflow: - python3 tests/create_short_testvectors.py - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - python3 scripts/prepare_combined_format_inputs.py - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi # TODO: remove once ref branch is switched @@ -455,39 +456,39 @@ workflow: - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - - exit_code=$EXIT_CODE_NON_BE - - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache - - - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true - - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true - - num_failures=$(wc -l < failed_testcases_for_printing.txt) - - - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true - - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true - - num_errors=$(wc -l < errors_testcases_for_printing.txt) - - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - - echo "Found these $num_failures non-bitexact testcases:" - - cat failed_testcases_for_printing.txt - - - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - - echo "The individual command lines can be found in the html report in the job artifacts." - - if [ $num_errors -ne 0 ]; then - - exit_code=1 - - echo "There were errors present in the following testcases:" - - cat errors_testcases_for_printing.txt - - fi - - - exit $exit_code + - exit_code=$EXIT_CODE_NON_BE + - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true + - num_failures=$(wc -l < failed_testcases_for_printing.txt) + + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true + - num_errors=$(wc -l < errors_testcases_for_printing.txt) + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - echo "Found these $num_failures non-bitexact testcases:" + - cat failed_testcases_for_printing.txt + + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the html report in the job artifacts." + - if [ $num_errors -ne 0 ]; then + - exit_code=1 + - echo "There were errors present in the following testcases:" + - cat errors_testcases_for_printing.txt + - fi + + - exit $exit_code - else # create empty files to not have errors at artifact stage - - touch $FAILED_TESTCASES_LIST - - touch $ERRORS_TESTCASES_LIST - - touch $PYTEST_CACHE_ARTIFACT - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - - echo "All testcases are bitexact." + - touch $FAILED_TESTCASES_LIST + - touch $ERRORS_TESTCASES_LIST + - touch $PYTEST_CACHE_ARTIFACT + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - echo "All testcases are bitexact." - fi - exit $exit_code @@ -540,27 +541,27 @@ workflow: - set -euxo pipefail - if [ -s $FAILED_TESTCASES_LIST ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh - - export PYTEST_ADDOPTS=--last-failed + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh + - export PYTEST_ADDOPTS=--last-failed - else # turn off echoing back of commands for result printout - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." - - exit 0 + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." + - exit 0 - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi # TODO: remove once ref branch is switched @@ -583,11 +584,11 @@ workflow: # set pytest args for ref creation - enc_stats_arg="" - if [ "$ENCODER_TEST" = "true" ]; then - - enc_stats_arg="--enc_stats" + - enc_stats_arg="--enc_stats" - fi - enc_dmx_arg="" - if [ "$COMPARE_DMX" = "true" ]; then - - enc_dmx_arg="--compare_enc_dmx" + - enc_dmx_arg="--compare_enc_dmx" - fi ### ----- run pytest for branch (dut) first ----- @@ -620,25 +621,25 @@ workflow: - exit_code=0 - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - if [ $zero_errors_branch != 1 ]; then - - echo "Run errors encountered!" - - exit_code=$EXIT_CODE_FAIL - - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - - echo "The individual command lines can be found in the regressions_crashes.csv files in the job artifacts." + - echo "Run errors encountered!" + - exit_code=$EXIT_CODE_FAIL + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the regressions_crashes.csv files in the job artifacts." - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then - - cat regression_log.txt - - if [ $allow_regressions_flag == 0 ]; then - - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" - - exit_code=$EXIT_CODE_FAIL; - - else - - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." - - exit_code=$EXIT_CODE_NON_BE; - - fi - - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - - echo "The individual command lines can be found in the regressions_*.csv files in the job artifacts." + - cat regression_log.txt + - if [ $allow_regressions_flag == 0 ]; then + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" + - exit_code=$EXIT_CODE_FAIL; + - else + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." + - exit_code=$EXIT_CODE_NON_BE; + - fi + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the regressions_*.csv files in the job artifacts." - fi - exit $exit_code @@ -680,13 +681,14 @@ workflow: - $XML_REPORT_BRANCH - $XML_REPORT_MAIN + .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor extends: - .job-linux stage: test needs: ["build-codec-linux-make"] script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -695,20 +697,20 @@ workflow: - set -euxo pipefail - make_args="CLANG=$CLANG_NUM" - if [[ $CLANG_NUM == 3 ]]; then - - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make_args="$make_args IGNORELIST=1" + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" - fi - make clean - make -j $make_args >> /dev/null - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS" # disable per-testcase timeout for msan to evaluate what is going on that it takes so long - if [[ $CLANG_NUM = 1 ]]; then - - testcase_timeout_arg="" + - testcase_timeout_arg="" - fi - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) @@ -732,20 +734,20 @@ workflow: SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - - testcase_timeout=$TESTCASE_TIMEOUT_STV + - testcase_timeout=$TESTCASE_TIMEOUT_STV - fi - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut @@ -841,8 +843,8 @@ check-float-ref-companion-branch-exists-for-porting-merge-requests: tags: - ivas-basop-linux script: - # we need to source the script here so that the correct exit code is returned from the global job shell and the "allow_failure" recognizes it - - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh + # we need to source the script here so that the correct exit code is returned from the global job shell and the "allow_failure" recognizes it + - source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/check-float-ref-companion-branch-exists-for-porting-merge-requests.sh allow_failure: exit_codes: - 123 @@ -935,6 +937,7 @@ clang-format-check: name: "$ARTIFACT_BASE_NAME" expose_as: "formatting patch" + # This job runs in manual jobs and scheduled jobs that compares to the float reference. # It fetches the latest commit of ivas-float-update and stores in FLOAT_COMMIT_REF pytest-to-ref-pre: @@ -1014,9 +1017,9 @@ build-codec-linux-instrumented-make: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ] || [ "$CI_COMMIT_REF_NAME" == "ivas-float-update" ]; then - - bash scripts/prepare_instrumentation.sh + - bash scripts/prepare_instrumentation.sh - else - - bash scripts/prepare_instrumentation.sh -m MEM_ONLY + - bash scripts/prepare_instrumentation.sh -m MEM_ONLY - fi - make -j -C $INSTR_DIR @@ -1293,13 +1296,13 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 - - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout --durations=0 || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout --durations=0 || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1364,11 +1367,11 @@ ivas-pytest-on-merge-request: - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi - if [ "$DISABLE_HRTF" = "true" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - fi - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi ### prepare pytest @@ -1498,12 +1501,12 @@ codec-smoke-test: - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then - - if [ "$DISABLE_HRTF" = "false" ]; then - - echo "Smoke test with external hrtf files failed" - - ret_val=1 - - else - - echo "Smoke test with external hrtf files failed, but DISABLE_HRTF is set. Ignoring failure." - - fi + - if [ "$DISABLE_HRTF" = "false" ]; then + - echo "Smoke test with external hrtf files failed" + - ret_val=1 + - else + - echo "Smoke test with external hrtf files failed, but DISABLE_HRTF is set. Ignoring failure." + - fi - fi - exit $ret_val artifacts: @@ -1561,6 +1564,7 @@ check-first-frame-is-sid: expose_as: "logs-hrtf-loading" expire_in: "5 days" + ### From Float # Check interop IVAS_cod_test -> IVAS_dec_ref ivas-interop-on-merge-request: @@ -1579,7 +1583,7 @@ ivas-interop-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - fi # some helper variables - "|| true" to prevent failures from grep not finding anything @@ -1657,9 +1661,9 @@ lc3-wrapper-unit-test: - echo "Assuming here that you either merge to main or to $FLOAT_REF_BRANCH." - echo "If you have some other merge target, this might fail and manual fiddling is needed." - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "$FLOAT_REF_BRANCH" ]; then - - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test + - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test - else - - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test + - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test - fi # from float @@ -1722,7 +1726,7 @@ renderer-msan: - .rules-merge-request-to-float-ref-no-draft needs: ["build-codec-linux-cmake"] stage: test - tags: + tags: - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -1751,7 +1755,7 @@ renderer-usan: - .rules-merge-request-to-float-ref-no-draft needs: ["build-codec-linux-cmake"] stage: test - tags: + tags: - ivas-basop-linux script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -1804,56 +1808,56 @@ renderer-usan: # to be reused in MR and LTV-scheduled sanitizer test jobs # set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section .sanitizer-selftest-anchor: &sanitizer-selftest-anchor - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 - - - if [ "$DISABLE_HRTF" = "true" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" - - fi - - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then - - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" - - fi + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - testcase_timeout=$SELFTEST_SANITY_TIMEOUT + - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 - - make_args="CLANG=$CLANG_NUM" - # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) - # shorten files to 1s for that case - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then - - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all - - # also need different setup for ubsan - - if [[ $CLANG_NUM == 3 ]]; then - - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" - - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make_args="$make_args IGNORELIST=1" - - fi - - fi + - if [ "$DISABLE_HRTF" = "true" ]; then + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'" + - fi - - exit_code20=0 - - exit_code10=0 - - exit_code5=0 + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then + - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'" + - fi - - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi + - make_args="CLANG=$CLANG_NUM" + # for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs) + # shorten files to 1s for that case + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "main" ]; then + - python3 tests/create_short_testvectors.py --cut_len 1.0 --inplace --all + + # also need different setup for ubsan + - if [[ $CLANG_NUM == 3 ]]; then + - export UBSAN_OPTIONS="suppressions=scripts/ubsan_basop.supp,report_error_type=1,print_stacktrace=1" + - python3 scripts/basop_create_ignorelist_for_ubsan.py + - make_args="$make_args IGNORELIST=1" + - fi + - fi - - make clean - - make -j $make_args + - exit_code20=0 + - exit_code10=0 + - exit_code5=0 - # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - # check different renderer fremsizes only for float - - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then - # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option - # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active - # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - - fi + - make clean + - make -j $make_args + + # run encoder and decoder with 20ms renderer framesize first, use reference creation mode + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + + # check different renderer fremsizes only for float + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" != "main" ]; then + # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option + # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active + # we are only interested in runtime errors from the sanitizers and ignore the diffs + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - fi - - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi + - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi # from float repo # NOTE: (per-testcase) timeout was adjusted to work for float only @@ -1981,29 +1985,29 @@ ivas-pytest-compare_to_ref-dmx-short-enc-lev+10: ivas-pytest-compare_to_ref-short-dec: extends: - .rules-pytest-to-ref-short - - .test-job-linux + - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor - + ivas-pytest-compare_to_ref-short-dec-lev-10: extends: - .rules-pytest-to-ref-short - - .test-job-linux + - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 <<: *ivas-pytest-anchor - + ivas-pytest-compare_to_ref-short-dec-lev+10: extends: - .rules-pytest-to-ref-short - - .test-job-linux + - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH @@ -2049,7 +2053,7 @@ ivas-pytest-compare_ref-long-enc: - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-dec: extends: - .rules-pytest-long @@ -2058,13 +2062,13 @@ ivas-pytest-compare_ref-long-dec: - ivas-basop-linux - high-memory-capacity before_script: - - USE_LTV=1 + - USE_LTV=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-enc-lev-10: extends: - .rules-pytest-long @@ -2076,7 +2080,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: - LEVEL_SCALING=0.3162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-dec-lev-10: extends: - .rules-pytest-long @@ -2103,7 +2107,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: - LEVEL_SCALING=3.162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor - + ivas-pytest-compare_ref-long-dec-lev+10: extends: - .rules-pytest-long @@ -2112,7 +2116,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: - ivas-basop-linux - high-memory-capacity before_script: - - USE_LTV=1 + - USE_LTV=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 @@ -2170,12 +2174,12 @@ ivas-pytest-msan: - ivas-basop-linux-fast resource_group: basop-long-sanitizers rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" timeout: "24 hours" before_script: - CLANG_NUM=1 @@ -2189,12 +2193,12 @@ ivas-pytest-asan: - ivas-basop-linux-fast resource_group: basop-long-sanitizers rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" timeout: "6 hours" before_script: - CLANG_NUM=2 @@ -2208,12 +2212,12 @@ ivas-pytest-usan: - ivas-basop-linux-fast resource_group: basop-long-sanitizers rules: - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_ASAN_USAN + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" timeout: "6 hours" before_script: - CLANG_NUM=3 @@ -2233,7 +2237,7 @@ test-long-self-test: - 123 parallel: matrix: - - LEVEL: + - LEVEL: - 0dB - minus10dB - plus10dB @@ -2245,20 +2249,20 @@ test-long-self-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - - case $LEVEL in - "0dB") - LEVEL_SCALING=1.0 - ;; - "minus10dB") - LEVEL_SCALING=0.3162 - ;; - "plus10dB") - LEVEL_SCALING=3.162 - ;; + - case $LEVEL in + "0dB") + LEVEL_SCALING=1.0 + ;; + "minus10dB") + LEVEL_SCALING=0.3162 + ;; + "plus10dB") + LEVEL_SCALING=3.162 + ;; esac - if [ $LEVEL_SCALING != "1.0" ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") @@ -2286,6 +2290,7 @@ test-long-self-test: - make clean - make -j + ### prepare pytest # create references @@ -2301,6 +2306,7 @@ test-long-self-test: - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi - exit 0 + artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" when: always @@ -2324,11 +2330,11 @@ ivas-smoke-test-saturation: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - fi - if [ $LEVEL_SCALING != "1.0" ];then - - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -2355,6 +2361,7 @@ ivas-smoke-test-saturation: - smoke_test_output_hrtf.txt expose_as: "saturation smoke test results" + # GCOV/LCOV coverage analysis of self_test suite coverage-test-on-main-scheduled: extends: @@ -2424,7 +2431,7 @@ be-2-evs-26444: - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi - exit 0 - + artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" expire_in: 1 week @@ -2556,7 +2563,7 @@ ivas-long-term-job-logs: stage: maintenance timeout: "25 minutes" script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - set -euxo pipefail - echo "Running on RUNNER_TAG = $RUNNER_TAG" - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -2566,26 +2573,26 @@ ivas-long-term-job-logs: - if [ -d "logs" ]; then rm -rf "logs"; fi # Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set - if [ "$LOGS_BACKUP_SOURCE_DIR" != "" ]; then - - cp_ret=0 - - cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ || cp_ret=$? - - if [ "$cp_ret" != 0 ]; then - - echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!" - - exit 1 - - fi + - cp_ret=0 + - cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ || cp_ret=$? + - if [ "$cp_ret" != 0 ]; then + - echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!" + - exit 1 + - fi - fi - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID) - echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous" - if [ "$id_previous" != "-1" ]; then - # Unzip artifacts to recover past logs dir - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - # If there is overlap between injected log and new log, use the new logs. - - unzip -o artifacts.zip + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + # If there is overlap between injected log and new log, use the new logs. + - unzip -o artifacts.zip - else - # create logs dir if it doesn't exist (should only happen in first run) - - echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!" - - mkdir logs + # create logs dir if it doesn't exist (should only happen in first run) + - echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!" + - mkdir logs - fi # create folder for today's results @@ -2595,14 +2602,14 @@ ivas-long-term-job-logs: # Aggregate job logs - job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10" - for job_name in $job_names; do - - echo "Getting job logs for $job_name" - - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID) - - echo "Job ID from variables - "$job_name", Job ID from script - $id_previous" - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - - unzip artifacts.zip -d previous_artifacts - - mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY - - rm artifacts.zip - - rm -r previous_artifacts + - echo "Getting job logs for $job_name" + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID) + - echo "Job ID from variables - "$job_name", Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts + - mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY + - rm artifacts.zip + - rm -r previous_artifacts - done - ls logs - exit 0 @@ -2626,7 +2633,7 @@ ivas-long-term-job-logs-overview: stage: maintenance timeout: "25 minutes" script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh @@ -2649,19 +2656,20 @@ ivas-long-term-regressions: stage: maintenance timeout: "25 minutes" script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - set -euxo pipefail - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh # Find regressions - for MEASURE in MLD MAX_ABS_DIFF; do - - python3 scripts/find_regressions_from_logs.py logs regressions_"$MEASURE".csv --measure "$MEASURE" --days 7 --ratio_thr 1.1 - - mkdir regressions_"$MEASURE" - - python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv - - mv regression*bash regressions_"$MEASURE" + - python3 scripts/find_regressions_from_logs.py logs regressions_"$MEASURE".csv --measure "$MEASURE" --days 7 --ratio_thr 1.1 + - mkdir regressions_"$MEASURE" + - python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv + - mv regression*bash regressions_"$MEASURE" - done + artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" expire_in: 4 weeks @@ -2687,7 +2695,7 @@ backup-long-term-job-logs: stage: maintenance timeout: "25 minutes" script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - set -euxo pipefail - echo "Running on RUNNER_TAG = $RUNNER_TAG" - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh @@ -2697,30 +2705,31 @@ backup-long-term-job-logs: - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID) - if [ "$id_previous" != "-1" ]; then - # Unzip artifacts to recover past logs dir - - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - - unzip artifacts.zip + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip - else - # create logs dir if it doesn't exist - - mkdir logs + # create logs dir if it doesn't exist + - mkdir logs - fi # Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set - if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then - - cp -r logs $LOGS_BACKUP_TARGET_DIR - - exit 0 + - cp -r logs $LOGS_BACKUP_TARGET_DIR + - exit 0 - else - - echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!" - - exit 1 + - echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!" + - exit 1 - fi + # --------------------------------------------------------------- # Complexity measurement jobs # --------------------------------------------------------------- # create necessary environment -.complexity-measurements-setup: - &complexity-measurements-setup ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work +.complexity-measurements-setup: &complexity-measurements-setup + ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work - mkdir -p wmops/logs - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) @@ -2736,20 +2745,20 @@ backup-long-term-job-logs: # 1. check for public_dir being there as this might not be the case when artifact download failed # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then - - mv $public_dir/* wmops/ + - mv $public_dir/* wmops/ # check here if we have the split-by-levels files present - if not, fake them up with the existing global one # this is needed for the first run with split graphs on a branch where the global version did run previously # NOTE: checking only for level_1 file here as this should already be sufficient # NOTE2: also not chechking for RAM for same reason - - wmops_all_global="wmops/log_wmops_all.txt" - - ram_all_global="wmops/log_ram_all.txt" - - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then - - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") - - for suffix in "${suffixes[@]}"; do - - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt - - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt - - done - - fi + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi - fi - rm artifacts.zip @@ -2758,17 +2767,16 @@ backup-long-term-job-logs: ### 1.5.part: get the corresponding measurement from ivas-float-update # only do this on main and not for the complexity meaurements run on ivas-float-update - if [ "$CI_COMMIT_REF_NAME" == "main" ]; then - - job_id=$(python3 ci/get_id_of_last_job_occurence.py $FLOAT_REF_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - - echo $job_id - - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $FLOAT_REF_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip + - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi + - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi - fi # prepare artifacts -> move to public directory -.complexity-measurements-prepare-artifacts: - &complexity-measurements-prepare-artifacts +.complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts - public_dir="$CI_JOB_NAME-public" - mkdir $public_dir - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ @@ -2788,11 +2796,11 @@ backup-long-term-job-logs: .complexity-measurements-report-summary: &complexity-measurements-report-summary - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh - if [ $ret_val -eq 0 ]; then - - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." + - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." - elif [ $ret_val -eq 123 ]; then - - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." + - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." - else - - echo -e "Something went wrong in running the codec. Likely some modes were crashing." + - echo -e "Something went wrong in running the codec. Likely some modes were crashing." - fi .complexity-template: @@ -2804,7 +2812,7 @@ backup-long-term-job-logs: GET_WMOPS_ARGS: "mem_only" timeout: 3 hours 30 minutes before_script: - - !reference [.test-job-linux, before_script] + - !reference [ .test-job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh @@ -3061,8 +3069,8 @@ complexity-masa-in-hoa3-out: # - if: $MEASURE_COMPLEXITY_LINUX # when: delayed # start_in: 13 hours -# variables: -# JOB_ID_INJECT: "" + # variables: + # JOB_ID_INJECT: "" # script: # - in_format=OMASA # - out_format=EXT @@ -3228,7 +3236,7 @@ pages: rules: - if: $UPDATE_PAGES script: - - !reference [.job-linux, before_script] + - !reference [ .job-linux, before_script ] - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - python3 ci/setup_pages.py -- GitLab From 7049eab434adfc6ae466fa424c3886ed6394e42a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Nov 2025 13:45:52 +0100 Subject: [PATCH 6/9] add cutting of input files to 5s for renderer test --- main-basop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-basop.yml b/main-basop.yml index ac88250..109364e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1296,6 +1296,7 @@ renderer-pytest-on-merge-request: - exit_code=0 - testcase_timeout=60 + - python3 tests/create_short_testvectors.py --cut_len 5.0 --all --inplace - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? ### Run test using branch scripts and input -- GitLab From f77e71a3ceb2e5f6f5160793a55a4c5482f8bae7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Nov 2025 13:46:33 +0100 Subject: [PATCH 7/9] cut to 5s signal length for split-rendering test, too --- main-basop.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/main-basop.yml b/main-basop.yml index 109364e..8806c20 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1446,6 +1446,7 @@ split-rendering-pytest-on-merge-request: # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181 - mv ISAR_post_rend ISAR_post_rend_tmp - cp $REF_POST_RENDERER_PATH ISAR_post_rend + - python3 tests/create_short_testvectors.py --cut_len 5.0 --all --inplace - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - mv ISAR_post_rend_tmp ISAR_post_rend -- GitLab From 3a4d4e323c4b588e375759996c86df426655e41f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Nov 2025 14:27:54 +0100 Subject: [PATCH 8/9] Revert "cut to 5s signal length for split-rendering test, too" This reverts commit f77e71a3ceb2e5f6f5160793a55a4c5482f8bae7. --- main-basop.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/main-basop.yml b/main-basop.yml index 8806c20..109364e 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1446,7 +1446,6 @@ split-rendering-pytest-on-merge-request: # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181 - mv ISAR_post_rend ISAR_post_rend_tmp - cp $REF_POST_RENDERER_PATH ISAR_post_rend - - python3 tests/create_short_testvectors.py --cut_len 5.0 --all --inplace - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? - mv ISAR_post_rend_tmp ISAR_post_rend -- GitLab From f6f0941d625668548e18ecd1d0460991e4775ab1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Nov 2025 15:18:55 +0100 Subject: [PATCH 9/9] add -fast tag for split rendering MR job --- main-basop.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main-basop.yml b/main-basop.yml index 109364e..594ccfa 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1413,6 +1413,8 @@ split-rendering-pytest-on-merge-request: - .test-job-linux-needs-testv-dir - .test-job-linux-compares-to-fixed-target - .rules-merge-request-no-draft + tags: + - ivas-basop-linux-fast timeout: "60 minutes" stage: compare variables: -- GitLab