From a5c4bb901bcb5f7a047746c31e910f571371d14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Wed, 29 Jan 2025 12:12:01 +0100 Subject: [PATCH 1/2] Add script to filter out smoke test items broken in basop --- scripts/smoketest-basop-filter.py | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 scripts/smoketest-basop-filter.py diff --git a/scripts/smoketest-basop-filter.py b/scripts/smoketest-basop-filter.py new file mode 100644 index 0000000000..faa170f7e0 --- /dev/null +++ b/scripts/smoketest-basop-filter.py @@ -0,0 +1,53 @@ +#! /usr/bin/env python3 + +import argparse +import json +import logging +import sys + + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logger = logging.getLogger() + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("input", help="input JSON", type=argparse.FileType('r')) + parser.add_argument("output", help="output JSON", type=argparse.FileType('w'), nargs="?") + parser.add_argument("--inline", "-i", help="patch input inline", action="store_true") + + args = parser.parse_args() + + if not args.inline and not args.output: + logging.error("You must specify an output file or use --inline") + return 1 + + modes = json.load(args.input) + + for mode in ("MC", "stereo", "OMASA"): + for encoder_config_name in modes[mode].keys(): + try: + del modes[mode][encoder_config_name]["dec"]["EXT"] + except KeyError: + logger.warning(f"No EXT dec in {mode} -> {encoder_config_name}, skipping") + continue + else: + logger.info(f"Found EXT dec in {mode} -> {encoder_config_name}, removing") + + for encoder_config_name in tuple(modes["OSBA"].keys()): + if not "FOA" in encoder_config_name: + continue + + logger.info(f"Removing OSBA FOA encoder config {encoder_config_name}") + del modes["OSBA"][encoder_config_name] + + if args.inline: + with open(args.input.name, "w") as f: + json.dump(modes, f, indent=4) + else: + json.dump(modes, args.output, indent=4) + + +if __name__ == "__main__": + sys.exit(main()) -- GitLab From c05306e832004a095fae908910a3ed9ae0b77385 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 31 Jan 2025 15:48:24 +0100 Subject: [PATCH 2/2] run smoke test with filtered config in MRs to basop-ci-branch --- .gitlab-ci.yml | 101 ++++++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 416b86f3fc..6d0d8bf458 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,6 +132,12 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h +.get-basop-float-reference: &get-basop-float-reference + - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch + - make -j -C ivas-basop + - cp ivas-basop/IVAS_cod ./ + - cp ivas-basop/IVAS_dec ./ + .merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons - *disable-debugging-macro @@ -279,10 +285,24 @@ stages: when: never - when: on_success -.rules-merge-request: +.rules-merge-request-to-main: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-basop-ci-branch-or-main: extends: .rules-basis rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main") - if: $CI_PIPELINE_SOURCE == 'push' when: never @@ -327,7 +347,7 @@ stages: stage: test extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week @@ -396,7 +416,7 @@ uninterruptible: branch-is-up-to-date-with-main-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -408,7 +428,7 @@ branch-is-up-to-date-with-main-pre: check-self-test-names-pre: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -418,7 +438,7 @@ check-self-test-names-pre: check-no-duplicates-in-self-tests: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: prevalidate needs: [] tags: @@ -428,7 +448,7 @@ check-no-duplicates-in-self-tests: branch-is-up-to-date-with-main-post: extends: - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate tags: - ivas-linux @@ -440,24 +460,19 @@ branch-is-up-to-date-with-main-post: .basop-ci-branch-compat-template: extends: - .test-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" - changes: - - tests/**/* - - scripts/**/* - - if: $CI_PIPELINE_SOURCE == 'push' - when: never + - .rules-merge-request-to-basop-ci-branch tags: - ivas-linux-fast before_script: - - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch - - make -j -C ivas-basop - python3 ci/remove_unsupported_testcases.py $PARAM_FILE - python3 tests/create_short_testvectors.py + - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true + - *get-basop-float-reference + + - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi @@ -579,7 +594,7 @@ build-codec-windows-msbuild: codec-smoke-test: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-basop-ci-branch-or-main timeout: "20 minutes" tags: - ivas-linux-fast @@ -589,7 +604,17 @@ codec-smoke-test: - *print-common-info # LTV update needed as ltv ISM metadata files are used - *update-ltv-repo - - bash ci/smoke_test.sh + + # for MRs to basop-ci-branch, we want to test with the BASOP float reference build + # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros + - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then + - python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline + - *get-basop-float-reference + - bash ci/smoke_test.sh coverage + - else + - bash ci/smoke_test.sh + - fi + ### analyze for failures - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi - ret_val=0 @@ -648,7 +673,7 @@ codec-usan: pytest-compare-20ms-and-5ms-rendering: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] script: @@ -695,7 +720,7 @@ pytest-compare-20ms-and-5ms-rendering: renderer-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -717,7 +742,7 @@ renderer-smoke-test: renderer-asan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -741,7 +766,7 @@ renderer-asan: renderer-msan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -765,7 +790,7 @@ renderer-msan: renderer-usan: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -793,7 +818,7 @@ renderer-usan: renderer-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" @@ -845,7 +870,7 @@ renderer-pytest-on-merge-request: split-rendering-smoke-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] stage: test script: @@ -866,7 +891,7 @@ split-rendering-smoke-test: lc3-wrapper-unit-test: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-cmake"] stage: test script: @@ -878,7 +903,7 @@ lc3-wrapper-unit-test: split-rendering-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main needs: ["build-codec-linux-make"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" @@ -950,7 +975,7 @@ split-rendering-pytest-on-merge-request: ivas-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "14 minutes" @@ -1004,7 +1029,7 @@ ivas-pytest-on-merge-request: ivas-interop-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "10 minutes" @@ -1053,7 +1078,7 @@ ivas-interop-on-merge-request: evs-pytest-on-merge-request: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: compare needs: ["build-codec-linux-cmake", "codec-smoke-test"] timeout: "10 minutes" @@ -1105,7 +1130,7 @@ evs-pytest-on-merge-request: voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-make"] timeout: "10 minutes" @@ -1118,7 +1143,7 @@ voip-be-on-merge-request: clang-format-check: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: prevalidate @@ -1165,7 +1190,7 @@ clang-format-check: check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-linux stage: test @@ -1192,7 +1217,7 @@ check-first-frame-is-sid: .lc3plus-ensure-no-code-changes: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: postvalidate needs: [] timeout: "5 minutes" @@ -1207,7 +1232,7 @@ check-first-frame-is-sid: check-bitexactness-hrtf-rom-and-file: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1229,7 +1254,7 @@ check-bitexactness-hrtf-rom-and-file: check-bitexactness-ext-and-transport-format: extends: - .test-job-linux - - .rules-merge-request + - .rules-merge-request-to-main stage: test needs: ["build-codec-linux-cmake"] timeout: "5 minutes" @@ -1256,7 +1281,7 @@ check-bitexactness-ext-and-transport-format: # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - - .rules-merge-request + - .rules-merge-request-to-main tags: - ivas-windows stage: test -- GitLab