Commit 93fb29c2 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'basop-ci-to-main' into 'main' 

[CI][BASOP-CI] Move basop-CI-related changes to main

See merge request !2421
parents 741802fb 370b8299

.gitlab-ci.yml

+46 −43
Original line number Diff line number Diff line 
variables:

  # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this

  # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information

  IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main

  IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF move-basop-ci-to-main



include:

  - local: .gitlab-ci/variables.yml
@@ -65,15 +65,6 @@ workflow: 
    echo "Commit time was $CI_COMMIT_TIMESTAMP"

    ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression



.get-basop-float-reference: &get-basop-float-reference

  - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch --depth 1 ivas-basop

  - cd ivas-basop

  - git status

  - make -j

  - cd -

  - cp ivas-basop/IVAS_cod ./

  - cp ivas-basop/IVAS_dec ./



.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec 

  ### build test binaries, initial clean for paranoia reasons

  - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh
@@ -236,24 +227,42 @@ branch-is-up-to-date-with-main-post: 
        exit 1

      fi



### BASOP compatibility test jobs

### these jobs run the tests with the BASOP executables to check if added testcases work fine both with float and BASOP codec

### run on manual trigger only to not prolong MR pipelines too much

.basop-ci-branch-compat-template:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-basop-ci-branch

  stage: basop-compat

  # allows for pipelines to pass when these were not started

  allow_failure: true

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"

      when: manual

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

  tags:

    - ivas-linux

    - ivas-linux-fast

  before_script:

    - !reference [.job-linux, before_script]

    - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm scripts/config/self_test_ltv.prm

    - python3 tests/create_short_testvectors.py

    - python3 scripts/prepare_combined_format_inputs.py



    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/build-binaries-from-basop-repo.sh main



    - *get-basop-float-reference

    # long timeout for BASOP executables

    - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS



    # hack for renderer tests to match the expected naming

    - mv IVAS_rend IVAS_rend_ref



    - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT

    # BASOP renderer test uses cut input signals

    - |

      if [ "$TEST_SUITE" == "tests/renderer_short/test_renderer.py" ]; then

        python3 tests/create_short_testvectors --inplace --all --cut_len 5.0

      fi

    - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout



  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
@@ -263,62 +272,56 @@ branch-is-up-to-date-with-main-post: 
      - report-junit.xml

      - report.html



    expose_as: "check-float-reference results"

    expose_as: "basop-compat results"

    reports:

      junit:

        - report-junit.xml



basop-ref-compat-stv:

### The stv jobs are run for both main and ivas-float-update

basop-compat-stv:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    TEST_SUITE: "$SHORT_TEST_SUITE tests/renderer"

    TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS

  script:

    - exit 0

  parallel:

    matrix:

      - TEST_SUITE:

        - tests/codec_be_on_mr_nonselection/test_param_file.py

        - tests/renderer_short/test_renderer.py



basop-ref-compat-ltv:

  extends:

    - .basop-ci-branch-compat-template

  tags:

    - ivas-linux-fast

  variables:

    TEST_SUITE: $LONG_TEST_SUITE

    TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS

  script:

    - exit 0



basop-ref-compat-encoder-stv:

basop-compat-encoder-dmx-comp:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    TEST_SUITE: $SHORT_TEST_SUITE_ENCODER

    # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option

    PYTEST_ARGS: "--compare_enc_dmx"

    TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS

  script:

    - exit 0



basop-ref-compat-encoder-ltv:

basop-compat-ltv:

  extends:

    - .basop-ci-branch-compat-template

  tags:

    - ivas-linux-fast

  variables:

    TEST_SUITE: $LONG_TEST_SUITE_ENCODER

    TEST_SUITE: $LONG_TEST_SUITE

    TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS

  script:

    - exit 0



basop-ref-compat-encoder-dmx-comp:

basop-compat-encoder-ltv:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    TEST_SUITE: $SHORT_TEST_SUITE_ENCODER

    # USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option

    PYTEST_ARGS: "--compare_enc_dmx"

    TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS

    TEST_SUITE: $LONG_TEST_SUITE_ENCODER

    TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS

  script:

    - exit 0





# ---------------------------------------------------------------

# Build jobs

# ---------------------------------------------------------------
@@ -398,7 +401,7 @@ build-codec-windows-msbuild: 
codec-smoke-test:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request-to-basop-ci-branch-or-main

    - .rules-merge-request-to-main

  timeout: "20 minutes"

  tags:

    - ivas-linux-fast
@@ -415,11 +418,11 @@ codec-smoke-test: 
    # LTV update needed as ltv ISM metadata files are used

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh



    # for MRs to basop-ci-branch, we want to test with the BASOP float reference build

    # for MRs to basop-ci-branch-pc, we want to test with the BASOP float reference build

    # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros

    # TODO: make manual for basop-CI merges

    - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then

    -   python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline

    -   *get-basop-float-reference

    -   bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/float/build-binaries-from-basop-repo.sh ivas-float-update

    -   bash ci/smoke_test.sh coverage

    - else

    -   bash ci/smoke_test.sh

ci/basop-pages/create_report_pages.py

+2 −2
Original line number Diff line number Diff line
@@ -102,7 +102,7 @@ COLUMNS = [ 
    "testcase",

    "Format",

    "Category",

    "Result",

    "result",

    "MLD",

    "MAX_ABS_DIFF",

    "MIN_SSNR",
@@ -187,7 +187,7 @@ def tr_from_row(row, id_current, id_previous): 
        prev = row[f"{c}-{id_previous}"]

        curr = row[f"{c}-{id_current}"]



        if c == "Result":

        if c == "result":

            # print errors in bold red font

            td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL

            tr.append(td_tmpl.format(prev))

ci/remove_unsupported_testcases.py

deleted100644 → 0
+0 −147
Original line number Diff line number Diff line 
__copyright__ = """

(C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,

Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

contributors to this repository. All Rights Reserved.



This software is protected by copyright law and by international treaties.

The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,

Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,

Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,

Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other

contributors to this repository retain full ownership rights in their respective contributions in

the software. This notice grants no license of any kind, including but not limited to patent

license, nor is any license granted by implication, estoppel or otherwise.



Contributors are required to enter into the IVAS codec Public Collaboration agreement before making

contributions.



This software is provided "AS IS", without any express or implied warranties. The software is in the

development stage. It is intended exclusively for experts who have experience with such software and

solely for the purpose of inspection. All implied warranties of non-infringement, merchantability

and fitness for a particular purpose are hereby disclaimed and excluded.



Any dispute, controversy or claim arising under or in relation to providing this software shall be

submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in

accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

the United Nations Convention on Contracts on the International Sales of Goods.

"""



from pathlib import Path

import argparse



# Enter tag of testcases to remove here WITHOUT the leading //

TESTCASES_MAIN = [

    "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",

    "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out",

    "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id",

    "OSBA planar FOA 1ISM at 256 kbps, 48 kHz in, 48 kHz out, BINAURAL out",

    "Multi-channel 5_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, BINAURAL out, FER at 10%, bandwidth switching",

    "OSBA FOA 4ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, FER at 5%, bandwidth switching",

    "OSBA FOA 2ISM at 64 kbps, 48kHz in, 48kHz out, HOA3 out, bandwidth switching",

    "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",

    "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out",

    "OSBA planar FOA 1ISM at 256 kbps, 48 kHz in, 48 kHz out, BINAURAL out",

    "SBA 3OA at 128 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB rendconf sel acoustic env",

    "OSBA FOA 4ISM at br sw 13.2 to 512 kbps, 48kHz in, 16kHz out, BINAURAL out (Model from file), FER at 5%, bandwidth switching",

    "stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, EXT out",

    "SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 32kHz in, 32kHz out, DTX on, EXT out",

    "SBA 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out",

    "SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, random FER at 5%, EXT out",

    "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out",

    "Multi-channel 5_1_2 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, EXT out",

    "Multi-channel 5_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 32kHz out, EXT out",

    "Multi-channel 7_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 16kHz out, EXT out",

    "Multi-channel 7_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out",

    "SBA planar 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out",

    "SBA planar FOA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out",

    "SBA 2OA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out",

    "SBA planar 3OA bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, random FER at 5%, EXT out",

    "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)",

    "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)",

    "Multi-channel 5_1 at 32 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (ParamBin)",

    "MASA 1dir 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration",

    "MASA 1dir 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration",

    "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration",

    "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration",

    "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",

    "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5",

    "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5",

    "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out,  JBM Prof 5",

    # object editing starts here

    "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM",

    "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC",

    "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",

    "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",

    # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions

    "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM",

    "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC",

    "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",

    "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",

    "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC",

    "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",

    "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",

    "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM",

    "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC",

    "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM",

    "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC",

    "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing",

    "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5",

]

TESTCASES_MAIN_PC = [

    "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR",

    "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR",

    "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR",

    "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration",

    "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR",

    "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out",

    "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration",

    "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR",

    "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR",

    "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out",

    "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out",

    "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out",

    "OSBA 2OA 2ISM bitrate switching, 32kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, JBM Prof 5",

]

TESTCASES_MAIN_PC_REENABLE = [

    "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id",

]

def remove_testcases(cfg: Path, testcases: list):

    """

    Go through file line by line and copy all testcases except the given ones

    """

    with open(cfg, "r") as f:

        content_in = f.readlines()



    content_out = list()

    copy_flag = True

    for line in content_in:

        if any([tc in line for tc in testcases]):

            copy_flag = False



        if copy_flag:

            content_out.append(line)

        elif line == "\n":

            copy_flag = True



    with open(cfg, "w") as f:

        f.write("".join(content_out))





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("cfg_files", nargs="+", type=Path)

    parser.add_argument("--use-main-pc-set", action="store_true")

    args = parser.parse_args()



    testcases = TESTCASES_MAIN

    if args.use_main_pc_set:

        # Add further unsupported test cases

        testcases.extend(TESTCASES_MAIN_PC)

        # Re-enable test cases that are now supported

        for case in TESTCASES_MAIN_PC_REENABLE:

            testcases.remove(case)



    for f in args.cfg_files:

        remove_testcases(f, testcases)

scripts/basop_create_ignorelist_for_ubsan.py

+6 −0
Original line number Diff line number Diff line
@@ -9,6 +9,12 @@ basop_files = here.glob("lib_*/basop*.[ch]") 


ignorefiles = sorted(set(non_ivas_files) - set(basop_files))



# add files here that should not be ignored

# NOTE: this is meant to be used in BASOP main where everything has the _fx suffix in the filenames

remove_files = ["lib_dec/jbm_pcmdsp_apa_fx.c"]

for file in remove_files:

    ignorefiles.remove(pathlib.Path(file))



ignorefunctions = ["L_add_o", "L_add_co", "L_sub_o", "shl_o", "L_deposit_h"]



with open("ubsan_ignorelist.txt", "w") as f:

scripts/config/self_test_basop_encoder.prm

+65 −65

File changed.

Preview size limit exceeded, changes collapsed.