Merge branch 'main' into 44-clang-error-for-MASA-condition-from-self_test.py

  rules:

    - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined)

      when: never

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions)

      when: never

    - when: on_success

.rules-merge-request:

    - python3 scripts/self_test.py --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    paths:

      - scripts/ref/logs/

      - test_output.txt

    expose_as: 'Msan selftest results'

# code selftest testvectors with address-sanitizer binaries

    - python3 scripts/self_test.py --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    paths:

      - scripts/ref/logs/

      - test_output.txt

    expose_as: 'Asan selftest results'

# compare bit exactness between target and source branch

    - be-2-evs-temp

  stage: test

  needs: [ "build-codec-linux-cmake" ]

  timeout: "20 minutes" # To be revisited

  script:

    - *print-common-info

      junit: report-junit.xml

# parameterizable job for sanitizer tests per format

# how to set up: create a schedule (CI/CD -> schedules) and enter the respective values for the environment variables:

#   - SANITIZER_TEST_IN_FMT: input format

#   - SANITIZER_TEST_OUT_FMTS: list of output formats, blank-separated, e.g.: stereo mono 5_1

#   - SANITIZER_TEST_TESTS: list of checks to do, can be one of CLANG1, CLANG2, CLANG3, VALGRIND

sanitizer-test-on-main-scheduled:

  extends: .test-job-linux-needs-testv-dir

  extends: 

    - .test-job-linux-needs-testv-dir

    # this next one is maybe not really needed, since there is the rule checking for the existence of the env vars below, but use for clarity

    - .rules-main-scheduled

  tags:

    - sanitizer_test_main

  stage: test

  rules:

    # only run in scheduled pipeline that passes this env var

    - if: $SANITIZER_TEST_IN_FMT

    # only run in scheduled pipeline that passes this env vars

    - if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS

  script:

    - *print-common-info

    - echo "Running scheduled sanitizer"

    # - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS

    - echo "Running scheduled sanitizer tests $SANITIZER_TEST_TESTS for input format $SANITIZER_TEST_IN_FMT and output format(s) $SANITIZER_TEST_OUT_FMTS"

    - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS --tests $SANITIZER_TEST_TESTS

  artifacts:

    name: "sanitizer-test-results-and-error_pattern-$SANITIZER_TEST_IN_FMT-in-$SANITIZER_TEST_OUT_FMTS-out"

    when: always

    paths:

      - ep_015.g192

      - "./*/logs"

# ---------------------------------------------------------------

import sys

SEARCH_FOR = "warning:"

SEARCH_FOR = "warning"

RETURN_FOUND = 123

#!/usr/bin/env python3

import argparse

import sys

import subprocess

import pathlib

DURATION = "120"

CFG = "ci_linux.json"

SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3", "VALGRIND"]

EP_FILE = "ep_015.g192"

GENPATT_CMD = f"gen-patt -tailstat -fer -g192 -gamma 0 -rate 0.15 -tol 0.001 -reset -n {int(DURATION) * 50} {EP_FILE}"

EIDXOR_CMD = "eid-xor -vbr -fer {bitstream} {ep_file} {out_file}"

MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]

SCRIPT_DIR = pathlib.Path("./scripts").resolve()

def main(args):

    in_format = args.in_format

    out_formats = args.out_formats

    tests = args.tests

    run_fec = not args.skip_fec

    assert all([t in SUPPORTED_TESTS for t in tests])

    modes = get_modes(in_format)

    returncode = run_check(modes, out_formats, tests, run_fec=run_fec)

    sys.exit(returncode)

def get_modes(in_format: str) -> list:

    cmd = [SCRIPT_DIR.joinpath("runIvasCodec.py"), "-l"]

    list_process = subprocess.run(cmd, capture_output=True)

    output = list_process.stdout.decode("utf8")

    # correction for multichannel modes to avoid selecting some mono modes...

    if in_format in MC_MODES:

        in_format = "MC_" + in_format

    return [m for m in output.splitlines() if in_format in m]

def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True):

    ### always run encoder and decoder with no frameloss

    cmd_no_fec = [

        str(SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py")),

        "-U",

        DURATION,

        "-p",

        CFG,

        "--checks",

        *tests,

        "-m",

        *modes,

        "--oc",

        *out_formats,

    ]

    print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec)))

    proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    for c in iter(lambda: proc.stdout.read(1), b""):

        sys.stdout.buffer.write(c)

    proc.wait()

    if proc.returncode not in [0, 101]:

        raise IvasBuildAndRunFailed("Failed at first run (no PLC)")

    returncode_no_fec = proc.returncode

    if not run_fec:

        return returncode_no_fec

    ### second run: decoder only with disturbed bitstream

    # generate error pattern

    subprocess.call(GENPATT_CMD.split())

    # cleanup to avoid script errors

    # we want "logs" and "dec" subfolders to be empty -> delete and recreate them

    cleanup_folders = ["logs", "dec"]

    for t in tests:

        for fol in cleanup_folders:

            for fi in pathlib.Path(t).joinpath(fol).iterdir():

                fi.unlink()

    cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]

    print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))

    proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    for c in iter(lambda: proc.stdout.read(1), b""):

        sys.stdout.buffer.write(c)

    proc.wait()

    returncode_fec = proc.returncode

    if returncode_fec not in [0, 101]:

        raise IvasBuildAndRunFailed("failed at second run (PLC)")

    return 101 if 101 in [returncode_no_fec, returncode_fec] else 0

class IvasBuildAndRunFailed(Exception):

    pass

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("in_format", type=str)

    parser.add_argument("out_formats", type=str, nargs="+")

    parser.add_argument("--tests", type=str, nargs="+", default=["CLANG1", "CLANG2"])

    parser.add_argument("--skip_fec", action="store_true")

    sys.exit(main(parser.parse_args()))

    int16_t *synth_out                                          /* o  : integer 16 bits synthesis signal        */

);

#ifdef FIX_I98_HANDLES_TO_NULL

void ivas_initialize_handles_enc(

    Encoder_Struct *st_ivas                                     /* i/o: IVAS encoder structure                  */

);

#endif

ivas_error ivas_init_encoder(

    Encoder_Struct *st_ivas,                                    /* i/o: IVAS encoder structure                  */

    Indice ind_list[][MAX_NUM_INDICES],                         /* i  : indices list                            */

);

void ivas_sba_mix_matrix_determiner(

    Decoder_Struct *st_ivas,                                    /* i/o: IVAS decoder struct                     */

    float in_out[][L_FRAME48k],                                 /* i/o: transport/output audio channels         */

    SPAR_DEC_HANDLE hSpar,                                      /* i/o: SPAR decoder handle                     */

    float output[][L_FRAME48k],                                 /* i/o: transport/output audio channels         */

    const int16_t bfi,                                          /* i  : BFI flag                                */

    const int16_t nchan_remapped,                               /* i  : num channels after remapping of TCs     */

    const int16_t output_frame                                  /* i  : output frame length                     */

);

/*#define FIX_IVAS_185_MDCT_ST_PLC_FADEOUT*/            /* IVAS-185 fix bug in TCX-PLC fadeout for MDCT-Stereo and improve fadeout by fading to background noise instead of white noise */

/*#define FIX_I1_113*/                                  /* under review : MCT bit distribution optimization for SBA high bitrates*/

#define FIX_ADAP_STEFI_SHIFT                            /* Issue 89: fix bug in parameter shift of adaptive stereo filling */

#define SPAR_SCALING_HARMONIZATION                      /* issue 80: Changes to harmonize scaling in spar */

#define SPAR_SCALING_HARMONIZATION                      /* Issue 80: Changes to harmonize scaling in spar */

#define SBA_INTERN_CONFIG_FIX_HOA2                      /* Issue 99 : Fix for incorrect internal_config when output format is HOA2 or FOA*/

#define FIX_I98_HANDLES_TO_NULL                         /* Issue 98: do the setting of all handles to NULL in one place */

/* ################## End DEVELOPMENT switches ######################### */