Add scripts/merge_logs.py

#!/usr/bin/env python3

import argparse

from pathlib import Path

import pandas as pd

def main(logs_dir, output_filename, measure):

    input_path = Path(logs_dir)

    logs = [f for f in input_path.iterdir() if f.is_dir()]

    # Build dict of scores

    logdict = {}

    for log in logs:

        date = log.name

        logdict[date] = {}

        for logfile in log.glob('*.csv'):

            tmp = logfile.name.split('-')

            job = '-'.join(tmp[3:-4])

            #sha = tmp[-1].split('.')[0]

            #logdict[date]["sha"] = sha # Maybe we want to store the SHA too somehow.

            data = pd.read_csv(logfile, usecols=["testcase", measure])

            logdict[date][job] = {}

            for testcase, value in zip(data["testcase"], data[measure]):

                logdict[date][job][testcase] = value

    # Restructure dict

    csv_rows = []

    for date, jobs in logdict.items():

        for job, testcases in jobs.items():

            for testcase, value in testcases.items():

                csv_rows.append((job, testcase, date, value))

    result = pd.DataFrame(csv_rows, columns=["job","testcase","date","value"])

    result = result.pivot(index=['job', 'testcase'], columns='date', values="value").reset_index()

    result.to_csv(output_filename, sep=';', index=False)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="logs dir")

    parser.add_argument(

        "logs_dir",

        type=str,

        help="Logs dir, e.g. logs",

    )

    parser.add_argument(

        "output_filename",

        type=str,

        help="Filename of the combined csv file. e.g mld.csv",

    )

    parser.add_argument(

        "--measure",

        type=str,

        help="Measure for summary, one of MLD MIN_SSNR MAX_ABS_DIFF MIN_ODG, (default: MLD)",

        default="MLD",

    )

    args = parser.parse_args()

    main(args.logs_dir, args.output_filename, args.measure)

 No newline at end of file