Commit 72888f76 authored by Jan Kiene's avatar Jan Kiene
Browse files

delete bitstreams of failed encs before PLC run

parent 1c25ef13

ci/collect_artifacts.py

+15 −8
Original line number Diff line number Diff line
@@ -15,7 +15,11 @@ def main(args): 
        collect_for_sanitizer_test(file)





def find_failed_files_for_sanitizer_test(console_log: str, subfolder:str) -> dict():

def find_failed_files_for_sanitizer_test(

    console_log: list, subfolder: str, which="LOGS"

) -> dict():



    assert which in ["LOGS", "FILE_BASENAMES"]



    pattern_line = "(Encoding|Decoding) failed .*for \/.*(CLANG.|VALGRIND)\/(.*)"

    pattern_file = "(.*_b[0-9]*_.*_rs|.*_b[0-9]*_.*_cbr).*"
@@ -33,12 +37,13 @@ def find_failed_files_for_sanitizer_test(console_log: str, subfolder:str) -> dic 
                continue

            filename_start = m_file.groups()[0]



            if which == "LOGS":

                folder = pathlib.Path(f"{test}/{subfolder}/")

                files = [

                f

                for f in folder.iterdir()

                if f.name.startswith(filename_start)

                    f for f in folder.iterdir() if f.name.startswith(filename_start)

                ]

            elif which == "FILE_BASENAMES":

                files = [filename_start]

            if test in files_found:

                files_found[test].extend(files)

            else:
@@ -52,7 +57,9 @@ def collect_for_sanitizer_test(file): 
    with open(file) as f:

        console_log = f.readlines()



    files_to_archive_noPLC = find_failed_files_for_sanitizer_test(console_log, "logs_noPLC")

    files_to_archive_noPLC = find_failed_files_for_sanitizer_test(

        console_log, "logs_noPLC"

    )

    files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs")



    log_folder = pathlib.Path("./LOGS_PLC")

ci/run_scheduled_sanitizer_test.py

+21 −1
Original line number Diff line number Diff line
@@ -5,6 +5,10 @@ import sys 
import subprocess

import pathlib



CI_SCRIPT_DIR = "./ci"

sys.path.append(CI_SCRIPT_DIR)

from collect_artifacts import find_failed_files_for_sanitizer_test





DURATION = "30"

CFG = "ci_linux_ltv.json"
@@ -82,7 +86,9 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
    )



    with open(CONSOLE_OUT_FILE, "a") as f:

        proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        proc = subprocess.Popen(

            cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE

        )

        for c in iter(lambda: proc.stdout.read(1), b""):

            sys.stdout.buffer.write(c)

            f.write(c.decode("utf8"))
@@ -96,6 +102,20 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
    if not run_fec:

        return returncode_no_fec



    # delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run

    with open(CONSOLE_OUT_FILE) as f:

        console_log = f.readlines()

    failed_files = find_failed_files_for_sanitizer_test(

        console_log, "logs", "FILE_BASENAMES"

    )

    for t in tests:

        bs_folder = pathlib.Path(f"{t}/enc")

        file_starts = failed_files[t]

        for f in bs_folder.iterdir():

            for fs in file_starts:

                if f.name.startswith(fs):

                    bs_folder.joinpath(f).unlink()



    ### second run: decoder only with disturbed bitstream



    # generate error pattern