Unverified Commit b52261bc authored by norvell's avatar norvell
Browse files

Make measure a command line parameter, to allow creating separate html filer... 

Make measure a command line parameter, to allow creating separate html filer for MLD and MAX_ABS_DIFF
parent 9ba5d516

ci/process_long_term_logs.py

+27 −18
Original line number Diff line number Diff line
@@ -39,27 +39,26 @@ def parse_csv_data(csv_data): 
    return concat_df





def plot_data(df, output_filename, days):

    """plot max values for 'MLD' and 'MAX_ABS_DIFF' data and save

def plot_data(df, output_filename, days, measure):

    """plot max values for measure and data and save

    to html file."""

    # Convert 'date' to datetime

    df["date"] = pd.to_datetime(df["date"], errors="coerce")

    df["MLD"] = pd.to_numeric(df["MLD"], errors="coerce")

    df["MAX_ABS_DIFF"] = pd.to_numeric(df["MAX_ABS_DIFF"], errors="coerce")

    df["MLD"] = pd.to_numeric(df[measure], errors="coerce")



    # Filter out rows older than "days"

    cutoff = df["date"].max() - pd.Timedelta(days=days)

    df = df[df["date"] > cutoff].reset_index(drop=True)



    # Drop rows with NaT and NaN

    df = df.dropna(subset=["date", "MLD", "MAX_ABS_DIFF"])

    df = df.dropna(subset=["date", measure])



    # Group by 'format' and 'date' to get rows with max 'MLD' per group

    idx = df.groupby(['format', 'date'])['MLD'].idxmax()

    idx = df.groupby(['format', 'date'])[measure].idxmax()

    max = df.loc[idx].reset_index(drop=True)

    idx = df.groupby(['format', 'date'])['MLD'].idxmin()

    idx = df.groupby(['format', 'date'])[measure].idxmin()

    min = df.loc[idx].reset_index(drop=True)

    mean = df.groupby(['format', 'date'])['MLD'].mean().to_frame('mean').reset_index()

    mean = df.groupby(['format', 'date'])[measure].mean().to_frame('mean').reset_index()



    formats = sorted(df["format"].unique())
@@ -76,25 +75,27 @@ def plot_data(df, output_filename, days): 


        data_mld = max[max["format"] == fmt].sort_values("date")



        # Add max 'MLD' to primary y-axis

        # Add max measure to plots

        fig.add_trace(

            go.Scatter(

                x=data_mld["date"],

                y=data_mld["MLD"],

                y=data_mld[measure],

                mode="lines+markers",

                name=f"Max MLD",

                name=f"Max {measure}",

                hovertext=[

                    f"Testcase: {tc}<br>MLD: {mld:.4f}"

                    f"Testcase: {tc}<br>{measure}: {mld:.4f}"

                    f" {format}<br>Date: {date.date()}"

                    for tc, mld, format, date in zip(

                        data_mld["testcase"],

                        data_mld["MLD"],

                        data_mld[measure],

                        data_mld["format"],

                        data_mld["date"],

                    )



                ],

                hoverinfo="text",

                marker_color="red",

                showlegend=(i==0),

            ),

            row=row,

            col=col,
@@ -108,9 +109,9 @@ def plot_data(df, output_filename, days): 
                x=data_mld["date"],

                y=data_mld["mean"],

                mode="lines+markers",

                name=f"Mean MLD",

                name=f"Mean {measure}",

                hovertext=[

                    f"Mean MLD: {mld:.4f}"

                    f"Mean {measure}: {mld:.4f}"

                    f" {format}<br>Date: {date.date()}"

                    for mld, format, date in zip(

                        data_mld["mean"],
@@ -120,13 +121,15 @@ def plot_data(df, output_filename, days): 


                ],

                hoverinfo="text",

                marker_color="blue",

                showlegend=(i==0),

            ),

            row=row,

            col=col,

        )



    fig.update_layout(

        title_text="History: MLD",        

        title_text=f"History: {measure}",        

        legend=dict(x=1, y=1, orientation="v"),

        hovermode="x unified",

    )
@@ -157,8 +160,14 @@ if __name__ == "__main__": 
        help="Number of days in history. Default: 30",

        default=30,

    )

    parser.add_argument(

        "--measure",

        type=str,

        help="Measure for analysis: MLD, MAX_ABS_DIFF, MIN_ODG, default: MLD",

        default="MLD",

    )        

    args = parser.parse_args()



    csv_data = read_csv_files(args.root_dir)

    data = parse_csv_data(csv_data)

    plot_data(data, args.output_filename, args.days)
 
    plot_data(data, args.output_filename, args.days, args.measure)