Commit 1772896f authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch... 

Merge branch '154-filter-output-of-sanitizer-tests-such-that-only-failing-conditions-are-stored' into 'main'

Resolve "Filter output of sanitizer tests such that only failing conditions are stored"

See merge request !190
parents 36a71ff2 a0815ac7

.gitlab-ci.yml

+3 −2
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main

    - if: $CI_PIPELINE_SOURCE == "web"



stages:

  - maintenance
@@ -722,8 +723,8 @@ codec-comparison-on-main-push: 
    when: always

    paths:

      - ep_015.g192

      # second wildcard is necessary to get encoder and no-PLC run logs

      - "CLANG*/logs*"

      - ./LOGS_PLC

      - ./LOGS_noPLC



### --- sanitizer schedule A ---

ci/collect_artifacts.py

0 → 100755
+103 −0
Original line number Diff line number Diff line 
#! /usr/bin/env python3

import pathlib

import argparse

import re





TEST_TYPES = ["sanitizers"]





def main(args):



    test = args.test

    file = args.console_out_file

    if test == "sanitizers":

        collect_for_sanitizer_test(file)





def find_failed_files_for_sanitizer_test(

    console_log: list, subfolder: str, which="LOGS"

) -> dict():



    assert which in ["LOGS", "FILE_BASENAMES"]



    pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)"

    pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*"



    files_found = dict()

    for line in console_log:

        m_line = re.match(pattern_line, line)



        if m_line is not None:

            _, test, filename = m_line.groups()

            filename = pathlib.Path(filename).name

            m_file = re.match(pattern_file, filename)

            if m_file is None:

                print(f"Unexpected: no match on {filename} with {pattern_file} - skip")

                continue

            filename_start = m_file.groups()[0]



            if which == "LOGS":

                folder = pathlib.Path(f"{test}/{subfolder}/")

                files = [

                    f for f in folder.iterdir() if f.name.startswith(filename_start)

                ]

            elif which == "FILE_BASENAMES":

                files = [filename_start]

            if test in files_found:

                files_found[test].extend(files)

            else:

                files_found[test] = files



    return files_found





def collect_for_sanitizer_test(file):



    with open(file) as f:

        console_log = f.readlines()



    files_to_archive_noPLC = find_failed_files_for_sanitizer_test(

        console_log, "logs_noPLC"

    )

    files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs")



    log_folder = pathlib.Path("./LOGS_PLC")

    log_folder.mkdir()

    for test in files_to_archive.keys():

        log_folder.joinpath(test).mkdir()

    for test, files in files_to_archive.items():

        folder = log_folder.joinpath(test)

        for p in files:

            source = pathlib.Path(p)

            target = folder.joinpath(source.name)

            source.rename(target)



    log_folder_noPLC = pathlib.Path("./LOGS_noPLC")

    log_folder_noPLC.mkdir()

    for test in files_to_archive_noPLC.keys():

        log_folder_noPLC.joinpath(test).mkdir()

    for test, files in files_to_archive_noPLC.items():

        folder = log_folder_noPLC.joinpath(test)

        for p in files:

            source = pathlib.Path(p)

            target = folder.joinpath(source.name)

            source.rename(target)





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument(

        "test",

        type=str,

        choices=TEST_TYPES,

        help="for which test should artifacts be collected?",

    )

    parser.add_argument(

        "console_out_file",

        type=str,

        help="file with stdout from IvasBuildAndRunChecks.py",

    )

    args = parser.parse_args()



    main(args)

ci/run_scheduled_sanitizer_test.py

100644 → 100755
+53 −15
Original line number Diff line number Diff line
@@ -5,6 +5,13 @@ import sys 
import subprocess

import pathlib



CI_SCRIPT_DIR = "./ci"

sys.path.append(CI_SCRIPT_DIR)

from collect_artifacts import (

    find_failed_files_for_sanitizer_test,

    collect_for_sanitizer_test,

)





DURATION = "120"

CFG = "ci_linux_ltv.json"
@@ -16,6 +23,8 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] 


SCRIPT_DIR = pathlib.Path("./scripts").resolve()



CONSOLE_OUT_FILE = "output_san.txt"





def main(args):

    in_format = args.in_format
@@ -28,6 +37,8 @@ def main(args): 
    modes = get_modes(in_format)

    returncode = run_check(modes, out_formats, tests, run_fec=run_fec)



    collect_for_sanitizer_test(CONSOLE_OUT_FILE)



    sys.exit(returncode)
@@ -37,7 +48,7 @@ def get_modes(in_format: str) -> list: 
        SCRIPT_DIR.joinpath("runIvasCodec.py"),

        "-C",

        "MC" if in_format in MC_MODES else in_format,

            "-l"

        "-l",

    ]

    list_process = subprocess.run(cmd, capture_output=True)
@@ -74,21 +85,43 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
        *out_formats,

    ]



    print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec)))

    print(

        "======== Script command line WITHOUT plc: ========\n{}".format(

            " ".join(cmd_no_fec)

        )

    )



    proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    with open(CONSOLE_OUT_FILE, "a") as f:

        proc = subprocess.Popen(

            cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE

        )

        for c in iter(lambda: proc.stdout.read(1), b""):

            sys.stdout.buffer.write(c)

            f.write(c.decode("utf8"))

        proc.wait()



    if proc.returncode not in [0, 101]:

        raise IvasBuildAndRunFailed("Failed at first run (no PLC)")



    returncode_no_fec = proc.returncode

    print("returncode_no_fec:", returncode_no_fec)

    if returncode_no_fec not in [0, 101]:

        raise IvasBuildAndRunFailed("Failed at first run (no PLC)")



    if not run_fec:

        return returncode_no_fec



    # delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run

    with open(CONSOLE_OUT_FILE) as f:

        console_log = f.readlines()

    failed_files = find_failed_files_for_sanitizer_test(

        console_log, "logs", "FILE_BASENAMES"

    )

    for t in failed_files.keys():

        bs_folder = pathlib.Path(f"{t}/enc")

        file_starts = failed_files[t]

        for f in bs_folder.iterdir():

            for fs in file_starts:

                if f.name.startswith(fs):

                    f.unlink()



    ### second run: decoder only with disturbed bitstream



    # generate error pattern
@@ -111,7 +144,11 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
            path.mkdir()



    cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]

    print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))

    print(

        "======== Script command line WITH plc: ========\n{}".format(

            " ".join(cmd_no_fec)

        )

    )



    proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    for c in iter(lambda: proc.stdout.read(1), b""):
@@ -119,6 +156,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
    proc.wait()



    returncode_fec = proc.returncode

    print("returncode_fec:", returncode_fec)



    if returncode_fec not in [0, 101]:

        raise IvasBuildAndRunFailed("failed at second run (PLC)")

scripts/IvasBuildAndRunChecks.py

+3 −3
Original line number Diff line number Diff line
@@ -193,15 +193,15 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript): 
                self.args["create_complexity_tables"]

            )



        returncode = 0

        for check in checks:

            runner = br.build_and_run_dict[check]["runner"]

            failed_encs = runner.failed_modes["enc"]

            failed_decs = runner.failed_modes["dec"]

            if len(failed_encs) > 0 or len(failed_decs) > 0:

                return RET_CODE_FAILURE

            else:

                return 0

                returncode =  RET_CODE_FAILURE



        return returncode



if __name__ == "__main__":

    script = IvasBuildAndRunChecks()