Unverified Commit 8742c1bb authored by norvell's avatar norvell
Browse files

Add filtering options for jobs

parent 94cbb18a

ci/process_long_term_logs.py

+43 −5
Original line number Diff line number Diff line
@@ -26,14 +26,17 @@ def read_csv_files(root_dir): 


def parse_csv_data(csv_data):

    """keep 'testcase', 'format', 'MLD', 'MAX_ABS_DIFF', 'MIN_ODG', 'MIN_SSNR'  and add

    'date' column."""

    'date' and 'job' column."""

    cols_to_keep = ["testcase", "format", "MLD", "MAX_ABS_DIFF", "MIN_ODG", "MIN_SSNR"]

    parsed_data = {}

    for key, df in csv_data.items():

        tmp = key.split("-")

        job = "-".join(tmp[4:-4])

        cols = [col for col in cols_to_keep if col in df.columns]

        date = os.path.basename(os.path.dirname(key))

        new_df = df[cols].copy()

        new_df["date"] = date

        new_df["job"] = job

        parsed_data[key] = new_df



    # concatenate all dataframe in the dictionary
@@ -74,6 +77,22 @@ def plot_data(df, args): 
        pattern = re.compile(args.match, re.IGNORECASE)

        df = df[df["testcase"].str.contains(pattern, na=False)]



    # Filter jobs based on job-include/job-reject/job-match arguments

    if args.job_include:

        mask = pd.Series(False, index=df.index)

        for tag in args.job_include:

            mask |= df["job"].str.contains(tag, case=False, na=False)

        df = df[mask]

    if args.job_reject:

        mask = pd.Series(False, index=df.index)

        for tag in args.job_reject:

            mask |= df["job"].str.contains(tag, case=False, na=False)

        df = df[~mask]

    if args.job_match:

        pattern = re.compile(args.job_match, re.IGNORECASE)

        df = df[df["job"].str.contains(pattern, na=False)]





    # Group by 'format' and 'date' to get rows with max 'MLD' per group

    idx = df.groupby(["format", "date"])[measure].idxmax()

    max = df.loc[idx].reset_index(drop=True)
@@ -110,8 +129,10 @@ def plot_data(df, args): 
                name=f"{maxmin_str} {measure}",

                hovertext=[

                    f"Testcase: {tc}<br>{maxmin_str} {measure}: {value:.4f}"

                    f"<br>Job: {job}"

                    f"<br>Date: {date.date()}"

                    for tc, value, date in zip(

                    for job, tc, value, date in zip(

                        data["job"],

                        data["testcase"],

                        data[measure],

                        data["date"],
@@ -190,18 +211,35 @@ if __name__ == "__main__": 
        "--include",

        nargs="+",

        type=str,

        help="List of tags to include",

        help="List of tags to include in testcases",

    )

    parser.add_argument(

        "--reject",

        nargs="+",

        type=str,

        help="List of tags to reject",

        help="List of tags to reject in testcases",

    )

    parser.add_argument(

        "--match",

        type=str,

        help="Regex pattern for selecting tests",

        help="Regex pattern for selecting testcases",

    )

    parser.add_argument(

        "--job-include",

        nargs="+",

        type=str,

        help="List of tags to include in jobs",

    )

    parser.add_argument(

        "--job-reject",

        nargs="+",

        type=str,

        help="List of tags to reject in jobs",

    )

    parser.add_argument(

        "--job-match",

        type=str,

        help="Regex pattern for selecting jobs",

    )

    args = parser.parse_args()