From ddbf7020763d8b99049ae4510b407be5266d732c Mon Sep 17 00:00:00 2001 From: malenov Date: Wed, 8 Feb 2023 18:36:33 +0100 Subject: [PATCH 1/7] remove export_mem() and use only update_mem() change the contents of the mem_analysis.csv file (write only malloc and free events along with frame numbers) --- apps/decoder.c | 8 ++--- apps/encoder.c | 4 +-- apps/renderer.c | 4 +-- lib_debug/wmc_auto.c | 86 +++++++++++++++++++++----------------------- lib_debug/wmc_auto.h | 4 --- 5 files changed, 45 insertions(+), 61 deletions(-) diff --git a/apps/decoder.c b/apps/decoder.c index f018814423..76a053c0f6 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -1505,9 +1505,7 @@ static ivas_error decodeG192( } #ifdef WMOPS update_wmops(); -#ifdef MEM_COUNT_DETAILS - export_mem( "mem_analysis.csv" ); -#endif + update_mem(); #endif } @@ -1978,9 +1976,7 @@ static ivas_error decodeVoIP( #ifdef WMOPS update_wmops(); -#ifdef MEM_COUNT_DETAILS - export_mem( "mem_analysis.csv" ); -#endif + update_mem(); #endif } diff --git a/apps/encoder.c b/apps/encoder.c index d789fd5486..b73e3d27f9 100644 --- a/apps/encoder.c +++ b/apps/encoder.c @@ -717,9 +717,7 @@ int main( #ifdef WMOPS update_wmops(); -#ifdef MEM_COUNT_DETAILS - export_mem( "mem_analysis.csv" ); -#endif + update_mem(); #endif } diff --git a/apps/renderer.c b/apps/renderer.c index e5c6aebfd6..19b88a92dd 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -999,9 +999,7 @@ int main( #ifdef WMOPS update_wmops(); -#ifdef MEM_COUNT_DETAILS - export_mem( "mem_analysis.csv" ); -#endif + update_mem(); #endif } diff --git a/lib_debug/wmc_auto.c b/lib_debug/wmc_auto.c index 6c71a6d37d..f92f264c5e 100644 --- a/lib_debug/wmc_auto.c +++ b/lib_debug/wmc_auto.c @@ -341,12 +341,6 @@ void update_wmops( void ) start_cnt = ops_cnt; - if ( heap_allocation_call_tree_size > 0 ) - { - /* update intra-frame heap memory and inter-frame heap memory*/ - update_mem(); - } - /* increment frame counter */ update_cnt++; @@ -554,6 +548,11 @@ void print_wmops( void ) #define ROUND_BLOCK_SIZE( n ) ( ( ( n ) + BLOCK_ROUNDING - 1 ) & ~( BLOCK_ROUNDING - 1 ) ) #define IS_CALLOC( str ) ( str[0] == 'c' ) +#ifdef MEM_COUNT_DETAILS +const char *csv_filename = "mem_analysis.csv"; +static FILE *fid_csv_filename = NULL; +#endif + typedef struct { char function_name[MAX_FUNCTION_NAME_LENGTH + 1]; @@ -688,6 +687,25 @@ void reset_mem( Counting_Size cnt_size ) size_wc_inter_frame_heap = 0; location_wc_inter_frame_heap = -1; +#ifdef MEM_COUNT_DETAILS + /* Check, if the .csv file has already been opened */ + if ( fid_csv_filename == NULL ) + { + fid_csv_filename = fopen( csv_filename, "wb" ); + + if ( fid_csv_filename == NULL ) + { + fprintf( stderr, "\nCannot open %s!\n\n", csv_filename ); + exit( -1 ); + } + } + else + { + /* reset file */ + rewind( fid_csv_filename ); + } +#endif + return; } @@ -960,6 +978,11 @@ void *mem_alloc( ptr_record->block_size = size; ptr_record->total_block_size += size; +#ifdef MEM_COUNT_DETAILS + /* Export heap memory allocation record to the .csv file */ + fprintf( fid_csv_filename, "A,%d,%s,%d,%d\n", update_cnt, ptr_record->name, ptr_record->lineno, ptr_record->block_size ); +#endif + if ( ptr_record->frame_allocated != -1 ) { fprintf( stderr, "Fct=%s, Ln=%i: %s!\n", func_name, func_lineno, "Error: Attempt to Allocate the Same Memory Block with Freeing it First!" ); @@ -1295,6 +1318,11 @@ void mem_free( const char *func_name, int func_lineno, void *ptr ) /* Check, if Out-Of-Bounds Access has been Detected */ ptr_record->OOB_Flag = mem_check_OOB( ptr_record ); +#ifdef MEM_COUNT_DETAILS + /* Export heap memory de-allocation record to the .csv file */ + fprintf( fid_csv_filename, "D,%d,%s,%d,%d\n", update_cnt, ptr_record->name, ptr_record->lineno, ptr_record->block_size ); +#endif + /* De-Allocate Memory Block */ tmp_ptr = (char *) ptr; tmp_ptr -= BLOCK_ROUNDING; @@ -1713,45 +1741,6 @@ static void mem_count_summary( void ) return; } -/*-------------------------------------------------------------------* - * export_mem() - * - * Export detailed (per-item) information about heap memory usage to a .csv file - *--------------------------------------------------------------------*/ - -void export_mem( const char *csv_filename ) -{ - int i; - static FILE *fid = NULL; - allocator_record *record_ptr; - - if ( csv_filename == NULL || strcmp( csv_filename, "" ) == 0 ) - { - return; - } - - /* Check, if the .csv file has already been opened */ - if ( fid == NULL ) - { - fid = fopen( csv_filename, "wb" ); - - if ( fid == NULL ) - { - fprintf( stderr, "\nCannot open %s!\n\n", csv_filename ); - exit( -1 ); - } - } - - /* Export individual heap memory records to a .csv file */ - for ( i = 0; i < Num_Records; i++ ) - { - record_ptr = &( allocation_list[i] ); - fprintf( fid, "%s:%d,%d;", record_ptr->name, record_ptr->lineno, record_ptr->block_size ); - } - fprintf( fid, "\n" ); - - return; -} #endif /*-------------------------------------------------------------------* @@ -1902,6 +1891,13 @@ void print_mem( ROM_Size_Lookup_Table Const_Data_PROM_Table[] ) free( list_wc_inter_frame_heap ); } +#ifdef MEM_COUNT_DETAILS + if ( fid_csv_filename != NULL ) + { + fclose( fid_csv_filename ); + } +#endif + return; } diff --git a/lib_debug/wmc_auto.h b/lib_debug/wmc_auto.h index 578be9b532..9e20a4c7c8 100644 --- a/lib_debug/wmc_auto.h +++ b/lib_debug/wmc_auto.h @@ -1007,9 +1007,6 @@ void mem_free( const char *func_name, int func_lineno, void *ptr ); void reset_mem( Counting_Size cnt_size ); void print_mem( ROM_Size_Lookup_Table Const_Data_PROM_Table[] ); -#ifdef MEM_COUNT_DETAILS -void export_mem( const char *csv_filename ); -#endif int push_stack( const char *filename, const char *fctname ); int pop_stack( const char *filename, const char *fctname ); @@ -1031,7 +1028,6 @@ void reset_stack( void ); #define free_( ptr ) free( ptr ) #define reset_mem( cnt_size ) #define print_mem( Const_Data_PROM_Table ) -#define export_mem( csv_filename ) #define push_stack( file, fct ) #define pop_stack( file, fct ) -- GitLab From a34a480c3f2b68fcb1f20d1a69e4f003af61765b Mon Sep 17 00:00:00 2001 From: malenov Date: Wed, 8 Feb 2023 18:39:49 +0100 Subject: [PATCH 2/7] Python script to plot detailed memory analysis for intra-frame and inter-frame memory blocks (based on pandas package) --- scripts/mem_analysis_malloc.py | 114 +++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 scripts/mem_analysis_malloc.py diff --git a/scripts/mem_analysis_malloc.py b/scripts/mem_analysis_malloc.py new file mode 100644 index 0000000000..b1dd106fee --- /dev/null +++ b/scripts/mem_analysis_malloc.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +import os +import argparse +import csv +import sys +import struct +import numpy as np +import pandas as pd + +import matplotlib +import tkinter +matplotlib.use('TkAgg') # Qt4Agg TkAgg Gtk3Agg +import matplotlib.pyplot as plt + +# set matplotlib in interactive mode (plots figure and return control) +plt.ion() +plt.close('all') + +parser = argparse.ArgumentParser(description='Analyze memory output file') +parser.add_argument('input',type=str,help='Memory analysis file (e.g. mem_analysis.csv)') +args = parser.parse_args() +fileIn = args.input + +# read input .csv file +df = pd.read_csv(fileIn, header=None, names=['action_type', 'frame', 'name', 'line_no', 'mem_size']) +Nframes = max(df['frame']) + +# merge identical memory blocks -> create new column 'count' +df = df.groupby(df.columns.tolist(),as_index=False).size() +df = df.rename(columns={'size': 'count'}) + +# df.loc[df['action_type'] == 'D', 'frame'] += 0.9 + +# remove column 'action_type' +df.drop(columns = ['action_type']) + +# merge records with the same signature but different frame number -> create list of frames (sort in ascending order) +df = df.groupby(['name', 'line_no', 'mem_size', 'count'],as_index=False).agg({'frame': list}) +df['frame'] = df['frame'].apply(np.sort) +df['list_len'] = df['frame'].apply(len) # auxiliary column -> length of the list + +# merge records with the same name and frame but different line number -> sum memory sizes and counts +df = df.groupby(['name', 'list_len'],as_index=False).agg({'line_no': list, 'mem_size': sum, 'count':sum, 'frame':'first'}) + +# sort by memory size (put inter-frame memory blocks to the bottom) +select_inter = (df['list_len'] == 2) & (df.apply(lambda row : row['frame'][-1] - row['frame'][0], axis = 1) >= Nframes) +df_inter = df[select_inter] +df_inter = df_inter.sort_values(by=['mem_size'], ascending=False) +df_intra = df[~select_inter] +df_intra = df_intra.sort_values(by=['list_len'], ascending=True) + +# plot inter-frame memory blocks in horizontal bar graph (use .broken_barh method) +fig, ax = plt.subplots() +colors = iter(plt.cm.rainbow(np.linspace(0, 1, len(df.index)))) +y_ticks = [] +y_tick_labels = [] +y_low = 1 +for index, row in df_inter.iterrows(): + l = f"{row['name']} on lines: {row['line_no']}, total memory size: {row['count']}x{row['mem_size']}" + fms = row['frame'].reshape((-1,2)) # convert frame numbers to two-column format + fms[:,1] -= fms[:,0] # second column should now represent the duration of the segment + fmsT = fms.T + fms_list = list(zip(fmsT[0], fmsT[1])) + c = next(colors) + ax.broken_barh(xranges=fms_list, yrange=(y_low, row['count']), label=l, color=c) + y_ticks.append(y_low + 0.5 * row['count']) + y_tick_labels.append(row['name']) + y_low += row['count'] + 0.5 + print(l + f", {row['count'] * fms.shape[0]/Nframes:.3f} mallocs per frame") + +# plot intra-frame memory blocks in horizontal bar graph (use .broken_barh method) +df_intra = df_intra.groupby(['name'],as_index=False).agg({'list_len': list, 'line_no': list, 'mem_size': list, 'count': list, 'frame': list}) +for index, row in df_intra.iterrows(): + # repeat for all grouped memory records in the row + for j in range(len(row['list_len'])): + l = f"{row['name']} on lines: {row['line_no'][j]}, total memory size: {row['count'][j]}x{row['mem_size'][j]}" + fms = row['frame'][j].astype(float).reshape((-1,2)) # convert frame numbers to two-column format + fms[:,1] -= fms[:,0] # second column should now represent the duration of the segment + idx_zero = fms[:,1] == 0 # find zero-length segments -> these are intra-frame memory blocks + fms[idx_zero, 0] += 0.1 + fms[idx_zero, 1] = 0.8 + fmsT = fms.T + fms_list = list(zip(fmsT[0], fmsT[1])) + c = next(colors) + ax.broken_barh(xranges=fms_list, yrange=(y_low, row['count'][j]), label=l, color=c, alpha=0.6) + print(l + f", {row['count'][j] * fms.shape[0]/Nframes:.3f} mallocs per frame") + y_ticks.append(y_low + 0.5 * max(row['count'])) + y_tick_labels.append(row['name']) + y_low += max(row['count']) + 0.5 + +ax.set_yticks(y_ticks) +ax.set_yticklabels(y_tick_labels) +ax.set_ylabel('Memory size') +ax.set_xlabel('Frame') +ax.set_title('Memory analysis') + +# shrink current axis's height by 20% on the bottom to fit the legend +box = ax.get_position() +ax.set_position([box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8]) + +# insert the legend below the graph +ax.legend(loc="upper left", bbox_to_anchor=(0, -0.1), ncol=2, borderaxespad=0, fontsize='small') + +# magnify to "almost" fullscreen size +mng = plt.get_current_fig_manager() +mng.full_screen_toggle() +# mng.frame.Maximize(True) +# mng.window.showMaximized() +# mng.window.state('zoomed') + +# show and save the figure +plt.show() +# plt.savefig(os.path.splitext(args.input)[0] + '.png', dpi=600) -- GitLab From 03c367f1959e1a480d3633433af2cc788b9c37f1 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 9 Feb 2023 09:10:20 +0100 Subject: [PATCH 3/7] Remove obsolete scripts/mem_analysis.m --- scripts/mem_analysis.m | 47 ------------------------------------------ 1 file changed, 47 deletions(-) delete mode 100644 scripts/mem_analysis.m diff --git a/scripts/mem_analysis.m b/scripts/mem_analysis.m deleted file mode 100644 index 98611c7a30..0000000000 --- a/scripts/mem_analysis.m +++ /dev/null @@ -1,47 +0,0 @@ -%% - -fid = fopen('../mem_analysis.csv','r'); - -% lines = readlines('../mem_analysis.csv','EmptyLineRule','skip'); - -names = {}; -% Nframes = length(lines); - -% mem = zeros(1,Nframes); -mem = []; -i = 1; - -% Load mem_analysis.csv into matrix -line = fgetl(fid); -while(ischar(line)) - entries = split(line,';'); - for j = 1:length(entries) - a = split(entries(j),','); - if(length(a)==2) - name = a{1}; - num = str2double(a{2}); - I = find(strcmp(names,name)); - if isempty(I) - names{end+1} = name; - I = length(names); - end - mem(I,i) = num; - end - end - i = i + 1; - line = fgetl(fid); -end - -fclose(fid); - -% Reorder the matrix such that the fixed allocations are at the bottom: -% --> Find all rows which are constant (sum of abs diff is zero). The >0 is -% to limit the reordering to constant values only. -[~,indx] = sort(sum(abs(diff(mem,[],2)),2)>0); -mem = mem(indx,:); -names = names(indx); - -bar(mem',1,'stacked') -legend(names,'location','eastoutside','interpreter','none') -xlabel('frame') -ylabel('memory size') \ No newline at end of file -- GitLab From ff5d0f28681ff2972d8fe664bb120c74635aa120 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 9 Feb 2023 10:35:48 +0100 Subject: [PATCH 4/7] renaming python script --- scripts/{mem_analysis_malloc.py => mem_analysis.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{mem_analysis_malloc.py => mem_analysis.py} (100%) diff --git a/scripts/mem_analysis_malloc.py b/scripts/mem_analysis.py similarity index 100% rename from scripts/mem_analysis_malloc.py rename to scripts/mem_analysis.py -- GitLab From ec3c7cb70d6516175695f67024f26453c915d6eb Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 9 Feb 2023 10:46:45 +0100 Subject: [PATCH 5/7] clang format --- scripts/mem_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mem_analysis.py b/scripts/mem_analysis.py index b1dd106fee..96770d43a5 100644 --- a/scripts/mem_analysis.py +++ b/scripts/mem_analysis.py @@ -110,5 +110,5 @@ mng.full_screen_toggle() # mng.window.state('zoomed') # show and save the figure -plt.show() +plt.show(block=True) # plt.savefig(os.path.splitext(args.input)[0] + '.png', dpi=600) -- GitLab From 36f34498e9f51dea727a2518b63122d72128a29e Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 9 Feb 2023 10:47:13 +0100 Subject: [PATCH 6/7] clang format --- lib_debug/wmc_auto.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_debug/wmc_auto.c b/lib_debug/wmc_auto.c index f92f264c5e..112336225f 100644 --- a/lib_debug/wmc_auto.c +++ b/lib_debug/wmc_auto.c @@ -981,7 +981,7 @@ void *mem_alloc( #ifdef MEM_COUNT_DETAILS /* Export heap memory allocation record to the .csv file */ fprintf( fid_csv_filename, "A,%d,%s,%d,%d\n", update_cnt, ptr_record->name, ptr_record->lineno, ptr_record->block_size ); -#endif +#endif if ( ptr_record->frame_allocated != -1 ) { -- GitLab From c9779a351030dc776d294d39bbe944b3aa84b006 Mon Sep 17 00:00:00 2001 From: Vladimir Malenovsky Date: Thu, 9 Feb 2023 11:54:59 +0100 Subject: [PATCH 7/7] fixes saving .png figure in full size --- scripts/mem_analysis.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/scripts/mem_analysis.py b/scripts/mem_analysis.py index 96770d43a5..fa2030c033 100644 --- a/scripts/mem_analysis.py +++ b/scripts/mem_analysis.py @@ -7,15 +7,11 @@ import sys import struct import numpy as np import pandas as pd - -import matplotlib -import tkinter -matplotlib.use('TkAgg') # Qt4Agg TkAgg Gtk3Agg import matplotlib.pyplot as plt # set matplotlib in interactive mode (plots figure and return control) -plt.ion() -plt.close('all') +# plt.ion() +# plt.close('all') parser = argparse.ArgumentParser(description='Analyze memory output file') parser.add_argument('input',type=str,help='Memory analysis file (e.g. mem_analysis.csv)') @@ -30,8 +26,6 @@ Nframes = max(df['frame']) df = df.groupby(df.columns.tolist(),as_index=False).size() df = df.rename(columns={'size': 'count'}) -# df.loc[df['action_type'] == 'D', 'frame'] += 0.9 - # remove column 'action_type' df.drop(columns = ['action_type']) @@ -102,13 +96,15 @@ ax.set_position([box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8] # insert the legend below the graph ax.legend(loc="upper left", bbox_to_anchor=(0, -0.1), ncol=2, borderaxespad=0, fontsize='small') -# magnify to "almost" fullscreen size +# magnify to "almost" fullscreen size (select the alternative for your backend) +fig = plt.gcf() +fig.set_size_inches(15, 9) # we need to set the size manually, otherwise savefig will save the figure in the default size mng = plt.get_current_fig_manager() mng.full_screen_toggle() # mng.frame.Maximize(True) # mng.window.showMaximized() # mng.window.state('zoomed') -# show and save the figure +# show and save the figure (use block=True to wait until Figure is closed manually) +plt.savefig(os.path.splitext(args.input)[0] + '.png', dpi=100, bbox_inches='tight') plt.show(block=True) -# plt.savefig(os.path.splitext(args.input)[0] + '.png', dpi=600) -- GitLab