Commit ab5de883 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into... 

Merge branch 'main' into 201-add-test-for-corner-case-of-to-be-decoded-bitstream-starting-with-an-sid
parents a9394ece 44c5bb4f

.gitlab-ci.yml

+3 −2
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main

    - if: $CI_PIPELINE_SOURCE == "web"



stages:

  - maintenance
@@ -749,8 +750,8 @@ codec-comparison-on-main-push: 
    when: always

    paths:

      - ep_015.g192

      # second wildcard is necessary to get encoder and no-PLC run logs

      - "CLANG*/logs*"

      - ./LOGS_PLC

      - ./LOGS_noPLC



### --- sanitizer schedule A ---

ci/collect_artifacts.py

0 → 100755
+103 −0
Original line number Diff line number Diff line 
#! /usr/bin/env python3

import pathlib

import argparse

import re





TEST_TYPES = ["sanitizers"]





def main(args):



    test = args.test

    file = args.console_out_file

    if test == "sanitizers":

        collect_for_sanitizer_test(file)





def find_failed_files_for_sanitizer_test(

    console_log: list, subfolder: str, which="LOGS"

) -> dict():



    assert which in ["LOGS", "FILE_BASENAMES"]



    pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)"

    pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*"



    files_found = dict()

    for line in console_log:

        m_line = re.match(pattern_line, line)



        if m_line is not None:

            _, test, filename = m_line.groups()

            filename = pathlib.Path(filename).name

            m_file = re.match(pattern_file, filename)

            if m_file is None:

                print(f"Unexpected: no match on {filename} with {pattern_file} - skip")

                continue

            filename_start = m_file.groups()[0]



            if which == "LOGS":

                folder = pathlib.Path(f"{test}/{subfolder}/")

                files = [

                    f for f in folder.iterdir() if f.name.startswith(filename_start)

                ]

            elif which == "FILE_BASENAMES":

                files = [filename_start]

            if test in files_found:

                files_found[test].extend(files)

            else:

                files_found[test] = files



    return files_found





def collect_for_sanitizer_test(file):



    with open(file) as f:

        console_log = f.readlines()



    files_to_archive_noPLC = find_failed_files_for_sanitizer_test(

        console_log, "logs_noPLC"

    )

    files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs")



    log_folder = pathlib.Path("./LOGS_PLC")

    log_folder.mkdir()

    for test in files_to_archive.keys():

        log_folder.joinpath(test).mkdir()

    for test, files in files_to_archive.items():

        folder = log_folder.joinpath(test)

        for p in files:

            source = pathlib.Path(p)

            target = folder.joinpath(source.name)

            source.rename(target)



    log_folder_noPLC = pathlib.Path("./LOGS_noPLC")

    log_folder_noPLC.mkdir()

    for test in files_to_archive_noPLC.keys():

        log_folder_noPLC.joinpath(test).mkdir()

    for test, files in files_to_archive_noPLC.items():

        folder = log_folder_noPLC.joinpath(test)

        for p in files:

            source = pathlib.Path(p)

            target = folder.joinpath(source.name)

            source.rename(target)





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument(

        "test",

        type=str,

        choices=TEST_TYPES,

        help="for which test should artifacts be collected?",

    )

    parser.add_argument(

        "console_out_file",

        type=str,

        help="file with stdout from IvasBuildAndRunChecks.py",

    )

    args = parser.parse_args()



    main(args)

ci/run_scheduled_sanitizer_test.py

100644 → 100755
+58 −15
Original line number Diff line number Diff line
@@ -5,6 +5,13 @@ import sys 
import subprocess

import pathlib



CI_SCRIPT_DIR = "./ci"

sys.path.append(CI_SCRIPT_DIR)

from collect_artifacts import (

    find_failed_files_for_sanitizer_test,

    collect_for_sanitizer_test,

)





DURATION = "120"

CFG = "ci_linux_ltv.json"
@@ -16,6 +23,8 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] 


SCRIPT_DIR = pathlib.Path("./scripts").resolve()



CONSOLE_OUT_FILE = "output_san.txt"





def main(args):

    in_format = args.in_format
@@ -28,6 +37,8 @@ def main(args): 
    modes = get_modes(in_format)

    returncode = run_check(modes, out_formats, tests, run_fec=run_fec)



    collect_for_sanitizer_test(CONSOLE_OUT_FILE)



    sys.exit(returncode)
@@ -37,7 +48,7 @@ def get_modes(in_format: str) -> list: 
        SCRIPT_DIR.joinpath("runIvasCodec.py"),

        "-C",

        "MC" if in_format in MC_MODES else in_format,

            "-l"

        "-l",

    ]

    list_process = subprocess.run(cmd, capture_output=True)
@@ -49,6 +60,11 @@ def get_modes(in_format: str) -> list: 
        in_format = "MC_" + in_format + "_b"

        mode_list = [m for m in mode_list if in_format in m]



    # TODO: remove once #185 is fixed

    # temporarily skip 24.4kbps SBA bitrate

    if in_format == "SBA":

        mode_list = [m for m in mode_list if not "b24_4" in m]



    return mode_list
@@ -69,21 +85,43 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
        *out_formats,

    ]



    print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec)))

    print(

        "======== Script command line WITHOUT plc: ========\n{}".format(

            " ".join(cmd_no_fec)

        )

    )



    proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    with open(CONSOLE_OUT_FILE, "a") as f:

        proc = subprocess.Popen(

            cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE

        )

        for c in iter(lambda: proc.stdout.read(1), b""):

            sys.stdout.buffer.write(c)

            f.write(c.decode("utf8"))

        proc.wait()



    if proc.returncode not in [0, 101]:

        raise IvasBuildAndRunFailed("Failed at first run (no PLC)")



    returncode_no_fec = proc.returncode

    print("returncode_no_fec:", returncode_no_fec)

    if returncode_no_fec not in [0, 101]:

        raise IvasBuildAndRunFailed("Failed at first run (no PLC)")



    if not run_fec:

        return returncode_no_fec



    # delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run

    with open(CONSOLE_OUT_FILE) as f:

        console_log = f.readlines()

    failed_files = find_failed_files_for_sanitizer_test(

        console_log, "logs", "FILE_BASENAMES"

    )

    for t in failed_files.keys():

        bs_folder = pathlib.Path(f"{t}/enc")

        file_starts = failed_files[t]

        for f in bs_folder.iterdir():

            for fs in file_starts:

                if f.name.startswith(fs):

                    f.unlink()



    ### second run: decoder only with disturbed bitstream



    # generate error pattern
@@ -106,7 +144,11 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
            path.mkdir()



    cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]

    print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))

    print(

        "======== Script command line WITH plc: ========\n{}".format(

            " ".join(cmd_no_fec)

        )

    )



    proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    for c in iter(lambda: proc.stdout.read(1), b""):
@@ -114,6 +156,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
    proc.wait()



    returncode_fec = proc.returncode

    print("returncode_fec:", returncode_fec)



    if returncode_fec not in [0, 101]:

        raise IvasBuildAndRunFailed("failed at second run (PLC)")

lib_com/options.h

100644 → 100755
+2 −1
Original line number Diff line number Diff line
@@ -153,7 +153,8 @@ 
#define SBA_BR_SWITCHING                                /* Issue 114: Changes for sba bit rate switching*/

#define FIX_AGC_WINFUNC_MEMORY                          /* Issue 62: lower agc_com.winFunc memory consumption */

#define REMOVE_SID_HARM_LEFTOVERS                       /* Issue 192: remove leftovers from the SID bitrate harmonization */



#define FIX_MCT_UNINIT_MEM                              /* Issue 166: Reading of uninitialized memory in TCX range coder */

#define FIX_IGF_NOISE_REPETITION                        /* Issue 182: fix repetition of same noise in IGF */





/* ################## End DEVELOPMENT switches ######################### */

lib_dec/dec_tcx.c

100644 → 100755
+18 −0
Original line number Diff line number Diff line
@@ -1309,8 +1309,25 @@ void decoder_tcx_noisefilling( 
        IGFDecReplicateTCX10State( st->hIGFDec );

    }



#ifdef FIX_IGF_NOISE_REPETITION

    if ( st->element_mode != EVS_MONO )

    {

        if ( bfi )

        {

            nf_seed = st->seed_tcx_plc;

        }

        else if ( nf_seed == 0 )

        {

            nf_seed = *st->hIGFDec->igfData.igfInfo.nfSeed;

        }

    }

#endif



    if ( st->igf )

    {

#ifdef FIX_IGF_NOISE_REPETITION

        *st->hIGFDec->igfData.igfInfo.nfSeed = (int16_t) ( nf_seed * 31821L + 13849L );

#else

        if ( bfi && st->element_mode != EVS_MONO )

        {

            *st->hIGFDec->igfData.igfInfo.nfSeed = (int16_t) ( st->seed_tcx_plc * 31821L + 13849L );
@@ -1319,6 +1336,7 @@ void decoder_tcx_noisefilling( 
        {

            *st->hIGFDec->igfData.igfInfo.nfSeed = (int16_t) ( nf_seed * 31821L + 13849L );

        }

#endif

    }



    return;