Merge branch 'ci/fix-prepare-instrumentation-for-split-rendering-basop' into 'main' (03f25119) · Commits · IVAS Codec Public Collaboration / IVAS Codec

ci/complexity_measurements/getWmops.sh

+4 −13

Original line number	Diff line number	Diff line
		@@ -29,7 +29,7 @@
		# the United Nations Convention on Contracts on the International Sales of Goods.

		function usage {
		echo "Usage: $0 \"ivas-format(s)\" \"output-format(s)\" \"mode{full(default)\|mem_only}\" \"repo{float(default)\|basop}\" [wmops_ref_logfile]"
		echo "Usage: $0 \"ivas-format(s)\" \"output-format(s)\" \"mode{full(default)\|mem_only}\" [wmops_ref_logfile]"
		exit 1
		}

		@@ -49,18 +49,9 @@ if [ $# -ge 3 ]; then
		fi
		fi

		repo_arg_script=""
		if [ $# -ge 4 ]; then
		if [ "$4" = "basop" ]; then
		repo_arg_script="--basop"
		elif [ "$4" != "float" ]; then
		usage
		fi
		fi

		wmops_ref_logfile=""
		if [ $# -eq 5 ]; then
		wmops_ref_logfile=$5
		if [ $# -eq 4 ]; then
		wmops_ref_logfile=$4
		fi

		date=$(date +%Y%m%d) # used for log-file file ending
		@@ -91,7 +82,7 @@ if [ "$ivas_format" == "OSBA" ]; then
		fi

		# instrument and build
		./scripts/IvasBuildAndRunChecks.py $mode_arg_script $repo_arg_script -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc} -C $ivas_format $mode_arg -f ${ep} --oc $output_format
		./scripts/IvasBuildAndRunChecks.py $mode_arg_script -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc} -C $ivas_format $mode_arg -f ${ep} --oc $output_format
		ret_val=$?

		# TODO: only split wmops and ram files

scripts/IvasBuildAndRunChecks.py

+0 −6

Original line number	Diff line number	Diff line
		@@ -114,11 +114,6 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript):
		help="pass the '-s' argument to the wmc tool to only measure memory",
		action="store_true",
		)
		self.parser.add_argument(
		"--basop",
		help="build the basop version for instrumentation",
		action="store_true",
		)
		self.parser.add_argument(
		"-e",
		"--enc",
		@@ -199,7 +194,6 @@ class IvasBuildAndRunChecks(IvasScriptsCommon.IvasScript):
		formats_fname=self.args["format_file"],
		max_workers=self.args["max_workers"],
		mem_only=self.args["wmc_tool_mem_only"],
		basop=self.args["basop"],
		)
		else:
		br.add_check(

scripts/prepare_instrumentation.sh

+107 −66

Original line number	Diff line number	Diff line
		@@ -36,14 +36,11 @@ function usage {
		echo " $(basename $0) [OPTIONS]"
		echo
		echo " -m MODE one of [FULL\|MEM_ONLY]"
		echo " -p PROJECT one of [FLOAT\|BASOP]"
		exit
		}


		# default values
		MODE="FULL"
		PROJECT="FLOAT"

		while getopts "m:p:h" OPTIONS; do
		case ${OPTIONS} in
		@@ -53,12 +50,6 @@ while getopts "m:p:h" OPTIONS; do
		usage
		fi
		;;
		p)
		PROJECT=${OPTARG^^}
		if [ "$PROJECT" != "FLOAT" -a "$PROJECT" != "BASOP" ]; then
		usage
		fi
		;;
		h \| *)
		usage
		;;
		@@ -66,24 +57,22 @@ while getopts "m:p:h" OPTIONS; do
		done
		shift $((OPTIND - 1))


		wmc_opt=""
		if [ "$MODE" = "MEM_ONLY" ]; then
		wmc_opt="-s"
		fi


		system=`uname -s`
		if [[ ($system = "Linux") && (`uname -a` =~ (microsoft\|Microsoft\|wsl\|WSL) ) ]]; then
		system=$(uname -s)
		if [[ ($system = "Linux") && ($(uname -a) =~ (microsoft\|Microsoft\|wsl\|WSL)) ]]; then
		system="Linux"
		fi

		coan_exists() {
		type coan &> /dev/null ;
		type coan &>/dev/null
		}

		cppp_exists() {
		type cppp/cppp.pl &> /dev/null ;
		type cppp/cppp.pl &>/dev/null
		}

		if ! (coan_exists \|\| cppp_exists); then
		@@ -93,33 +82,50 @@ if ! (coan_exists \|\| cppp_exists); then
		exit -1
		fi

		targetdir=c-code_instrument
		set -x # echo commands before running

		currdir=`pwd`
		scriptdir=`dirname $0`
		scriptdir=$(dirname $0)
		targetdir=$scriptdir/c-code_instrument
		ifdef_list=ifdef_instrument.list
		sourcedir=$scriptdir/..
		cd $scriptdir

		if [[ ($(git rev-parse --is-inside-work-tree) == true) ]]; then
		sourcedir=$(git rev-parse --show-toplevel)
		fi

		echo "Instrumenting source-code from $sourcedir ..."

		rm -Rf $targetdir
		mkdir $targetdir

		# copy files from source-dir
		cp -R ../lib_* $targetdir
		cp -R ../apps $targetdir
		cp -R ../Makefile $targetdir
		if [ "$PROJECT" = "FLOAT" ]; then
		cp -R $sourcedir/lib_* $targetdir
		cp -R $sourcedir/apps $targetdir
		cp -R $sourcedir/Makefile $targetdir
		if [ -e $sourcedir/CMakeLists.txt ]; then
		cp -R ../CMakeLists.txt $targetdir
		cp -R $sourcedir/CMakeLists.txt $targetdir
		fi
		cp -R ../Workspace_msvc $targetdir
		cp -R $sourcedir/Workspace_msvc $targetdir

		# back up #ifdef-list
		rm -f $ifdef_list
		touch $ifdef_list

		# LC3plus-related stuff -> only in float code
		if [ "$PROJECT" = "FLOAT" ]; then
		# Add LC3plus feature defines to options.h so that they are stripped correctly
		### LC3plus code related modifications
		ISAR=0
		if [ -d $targetdir/lib_lc3plus ]; then
		ISAR=1
		# Floating point code has lib_lc3plus/fft folder while BASOP code does not
		lc3plus_fftdir=""
		LC3PLUS_FFT=0
		if [ -d $targetdir/lib_lc3plus/fft ]; then
		lc3plus_fftdir="$targetdir/lib_lc3plus/fft/*.c"
		LC3PLUS_FFT=1
		fi
		fi

		if [ $ISAR -eq 1 ]; then
		# Generate list of active defines in LC3plus defines.h
		lc3plus_defines=$(
		gcc -E -dM $targetdir/lib_lc3plus/defines.h -I $targetdir/lib_com -I $targetdir/lib_debug \|
		@@ -129,9 +135,8 @@ if [ "$PROJECT" = "FLOAT" ]; then

		# Filter defines that come from outside of the header lib_lc3plus/defines.h
		lc3plus_defines_filtered=""
		while IFS=' \n' read -r line;
		do
		line=`echo $line \| tr -d '\n'`
		while IFS=' \n' read -r line; do
		line=$(echo $line \| tr -d '\n')
		if grep -wqF "$line" "$targetdir/lib_lc3plus/defines.h"; then
		lc3plus_defines_filtered+=$line$'\n'
		fi
		@@ -143,10 +148,45 @@ if [ "$PROJECT" = "FLOAT" ]; then
		#ifndef OPTIONS_H_LC3_DEFINES
		#define OPTIONS_H_LC3_DEFINES
		$lc3plus_defines_filtered
		#endif /* OPTIONS_H_LC3_DEFINES */
		" >>$targetdir/lib_com/options.h

		# only for fixed point code
		if [ $LC3PLUS_FFT -eq 0 ]; then
		# append additional defines for stripping for BASOP
		echo "
		#define ENABLE_HR_MODE
		//#define DYNMEM_COUNT
		#define CR10_A_ATTENUATION_CURVE_SELECTOR
		#define SUBSET_NB
		#define SUBSET_WB
		#define SUBSET_SSWB
		#define SUBSET_SWB
		#define SUBSET_FB
		#define SUBSET_UB
		" >>$targetdir/lib_com/options.h

		# clean out memory tool which breaks wmc_tool instrumentation
		perl -i -0777 -pe 's/Dyn_Mem_Deluxe_In$((.\|\s)?)$;/$1/g' $targetdir/lib_lc3plus/.c
		perl -i -0777 -pe 's/Dyn_Mem_Deluxe_Out\s$[^)]$\s;/;/sg' $targetdir/lib_lc3plus/.c
		perl -i -0777 -pe 's/\sDyn_Mem_In$(.\|\s)?$;//g' $targetdir/lib_lc3plus/.c
		perl -i -0777 -pe 's/Dyn_Mem_Out\s$[^)]$\s;/;/sg' $targetdir/lib_lc3plus/.c
		perl -i -0777 -pe 's/scratchAlign$\s((?:.\|\s)?)\s,\s((?:.\|\s)?)\s$;/(void )(((uintptr_t)($1) + ($2) + 0x3) & ~0x3);/g' $targetdir/lib_lc3plus/.c
		# expand LC3plus specific BASOP macros to avoid instrumentation
		perl -i -0777 -pe 's/cplxMpy32_32_32_2$\s([^,]+?),\s([^,]+?),\s([^,]+?),\s([^,]+?),\s([^,]+?),\s([^,]+?)\s$;/do { $1 = L_sub(Mpy_32_32_0($3, $5), Mpy_32_32_0($4, $6)); $2 = L_add(Mpy_32_32_0($3, $6), Mpy_32_32_0($4, $5)); } while (0);/g' $targetdir/lib_lc3plus/.c
		perl -i -0777 -pe 's/cplxMpy32_32_16_2$\s([^,]+?),\s([^,]+?),\s([^,]+?),\s([^,]+?),\s([^,]+?),\s([^,]+?)\s$;/do { $1 = L_sub(L_shr_pos(Mpy_32_16_lc3plus($3, $5), 1), L_shr_pos(Mpy_32_16_lc3plus($4, $6), 1)); $2 = L_add(L_shr_pos(Mpy_32_16_lc3plus($3, $6), 1), L_shr_pos(Mpy_32_16_lc3plus($4, $5), 1)); } while (0);/g' $targetdir/lib_lc3plus/.c
		perl -i -0777 -pe 's/extractW16/extract_h/g' $targetdir/lib_lc3plus/*.c
		perl -i -0777 -pe 's/(UL_l\|l\|L_\|)(sh[lr](_r)?)(?:_pos){1,2}/$1$2/g' $targetdir/lib_lc3plus/*.c
		fi

		# remove spaces in preprocessor directives
		perl -i -0777 -pe 's/#\s+(if\|else\|elif\|endif\|define)/#$1/g' $targetdir/lib_lc3plus/*.[ch]
		perl -i -0777 -pe 's/#(ifdef\|ifndef)\s{2,}/#$1 /g' $targetdir/lib_lc3plus/*.[ch]

		echo "
		#endif /* OPTIONS_H_LC3_DEFINES */
		" >>$targetdir/lib_com/options.h
		fi # ISAR -eq 1

		# get switches from options.h and append it to $ifdef_list
		parse_options_opt=""
		if coan_exists; then
		@@ -155,7 +195,7 @@ if coan_exists; then
		else
		echo "DEBUGGING" >>$ifdef_list
		fi
		./parse_options_h.sh $parse_options_opt $targetdir/lib_com/options.h >> $ifdef_list
		$scriptdir/parse_options_h.sh $parse_options_opt $targetdir/lib_com/options.h >>$ifdef_list
		if [ $? -ne 0 ]; then
		exit -1
		fi
		@@ -168,40 +208,41 @@ if coan_exists; then
		sed -i "/-UMEM_COUNT_DETAILS/d" $ifdef_list

		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/apps/*.[hc]
		if [ "$PROJECT" = "FLOAT" ]; then
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,rend,util,debug}/!(wmc_auto*).[hc]
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc]
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc]
		else
		# same as first call from if, but without "isar" and "debug" to avoid coan warning
		if [ $ISAR -eq 0 ]; then
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,rend,util}/!(wmc_auto*).[hc]
		else
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_{com,dec,enc,isar,lc3plus,rend,util,debug}/!(wmc_auto*).[hc]
		if [ $LC3PLUS_FFT -eq 1 ]; then
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/fft/!(wmc_auto*).[hc]
		fi
		fi
		coan source --replace --no-transients -E -K --file $ifdef_list $targetdir/lib_lc3plus/!(wmc_auto*).[hc]
		else
		./strip_defines_cppp.sh $targetdir $ifdef_list
		$scriptdir/strip_defines_cppp.sh $targetdir $ifdef_list
		fi
		shopt -u extglob

		# patch code before wmc_tool: replace hexadecimal unsigned long constants (0x...UL) by regular integer constant + cast to unsigned long
		find $targetdir -name ".[ch]" -exec sed -i.bak -e "s/$0x[0-9a-fA-F]$UL/$\(unsigned long$\1\)/" \{\} \;

		# run wmc_tool
		"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/encoder.c" "$targetdir/lib_enc/.c" "$targetdir/lib_com/.c" >> /dev/null
		"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/decoder.c" "$targetdir/lib_dec/.c" "$targetdir/lib_rend/.c" >> /dev/null
		# ISAR post-renderer and lc3plus sources only need to be instrumented in float code
		if [ "$PROJECT" = "FLOAT" ]; then
		# run wmc_tool, exit if the command fails
		set -e
		"$scriptdir/tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/encoder.c" "$targetdir/lib_enc/.c" "$targetdir/lib_com/.c" >>wmc_tool_output.txt 2>&1
		"$scriptdir/tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/decoder.c" "$targetdir/lib_dec/.c" "$targetdir/lib_rend/.c" >>wmc_tool_output.txt 2>&1
		for bak_file in $targetdir/lib_rend/.bak; do mv "$bak_file" "${bak_file%.}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool
		"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/.c" "$targetdir/lib_lc3plus/.c" "$targetdir/lib_lc3plus/fft/*.c" >> /dev/null
		if [ $ISAR -eq 0 ]; then
		"$scriptdir/tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" >>wmc_tool_output.txt 2>&1
		else
		"$scriptdir/tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/.c" "$targetdir/lib_lc3plus/.c" "$lc3plus_fftdir" >>wmc_tool_output.txt 2>&1
		for bak_file in $targetdir/lib_lc3plus/.bak; do mv "$bak_file" "${bak_file%.}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool
		if [ -n "$lc3plus_fftdir" ]; then
		for bak_file in $targetdir/lib_lc3plus/fft/.bak; do mv "$bak_file" "${bak_file%.}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool
		"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/isar_post_rend.c" "$targetdir/lib_isar/.c" "$targetdir/lib_lc3plus/.c" "$targetdir/lib_lc3plus/fft/*.c" >> /dev/null
		else
		for bak_file in $targetdir/lib_rend/.bak; do mv "$bak_file" "${bak_file%.}"; done # restore fresh .c files to avoid time-consuming des-instrumentation of files by the WMC tool
		"tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/renderer.c" "$targetdir/lib_rend/*.c" >> /dev/null
		fi
		# ISAR post rend
		"$scriptdir/tools/$system/wmc_tool" $wmc_opt -m "$targetdir/apps/isar_post_rend.c" "$targetdir/lib_isar/.c" "$targetdir/lib_lc3plus/.c" "$lc3plus_fftdir" >>wmc_tool_output.txt 2>&1
		fi
		set +e

		# automatically enable #define WMOPS in options.h
		sed -i.bak -e "s/\/\[[:space:]]$#define[[:space:]]WMOPS$[[:space:]]\*\//\1/g" $targetdir/lib_com/options.h
		sed -i.bak -e "s/\/\/[[:space:]]$#define[[:space:]]WMOPS$/\1/g" $targetdir/lib_com/options.h

		# return to start dir
		cd "$currdir"

scripts/pyivastest/IvasSvnBuilder.py

+0 −12

Original line number	Diff line number	Diff line
		@@ -227,7 +227,6 @@ class IvasBuilder(IvasBaseClass):
		defines_to_disable=[],
		instrumented=False,
		mem_only=False,
		basop=False,
		):
		"""

		@@ -277,9 +276,6 @@ class IvasBuilder(IvasBaseClass):
		if mem_only:
		instrument_cmd.append("-m")
		instrument_cmd.append("mem_only")
		if basop:
		instrument_cmd.append("-p")
		instrument_cmd.append("basop")
		build_log.write(" ".join(instrument_cmd))
		build_log.write("\n")
		build_result = subprocess.run(
		@@ -649,7 +645,6 @@ class IvasSvnBuilder(IvasBuilder):
		defines_to_disable=[],
		instrumented=False,
		mem_only=False,
		basop=False,
		):
		"""

		@@ -679,7 +674,6 @@ class IvasSvnBuilder(IvasBuilder):
		defines_to_disable=defines_to_disable,
		instrumented=instrumented,
		mem_only=mem_only,
		basop=basop,
		)


		@@ -792,7 +786,6 @@ class IvasBuilderAndRunner(IvasBaseClass):
		max_workers=1,
		timeout=None,
		mem_only=False,
		basop=False,
		):
		"""

		@@ -838,7 +831,6 @@ class IvasBuilderAndRunner(IvasBaseClass):
		self.build_and_run_dict[cfg_name]["instrumented"] = instrumented
		self.build_and_run_dict[cfg_name]["mem_only"] = mem_only
		self.build_and_run_dict[cfg_name]["make_options"] = make_options
		self.build_and_run_dict[cfg_name]["basop"] = basop
		else:
		self.logger.console("Adding config {}".format(cfg_name), logging.INFO)
		run_dir = os.path.join(self.builder.src_dir, cfg_name)
		@@ -884,7 +876,6 @@ class IvasBuilderAndRunner(IvasBaseClass):
		"mem_only": mem_only,
		"make_options": make_options,
		"analyzer": new_analyzer,
		"basop": basop,
		}
		}
		self.build_and_run_dict.update(cfg_dict)
		@@ -1068,7 +1059,6 @@ class IvasBuilderAndRunner(IvasBaseClass):
		defines_to_disable=cfg["defines_to_disable"],
		instrumented=cfg["instrumented"],
		mem_only=cfg["mem_only"],
		basop=cfg["basop"],
		)
		self.build_and_run_dict[cfg_name]["runner"].encoder = self.builder.encoder
		self.build_and_run_dict[cfg_name]["runner"].decoder = self.builder.decoder
		@@ -1295,7 +1285,6 @@ class IvasBuilderAndRunner(IvasBaseClass):
		formats_fname="",
		max_workers=1,
		mem_only=False,
		basop=False,
		):
		"""

		@@ -1329,7 +1318,6 @@ class IvasBuilderAndRunner(IvasBaseClass):
		formats_fname=formats_fname,
		max_workers=max_workers,
		mem_only=mem_only,
		basop=basop,
		)

		# TODO: this is used nowhere, is this still needed?