Merge branch 'complexity-measurement-maintenance' into 'main' (b3044dae) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+6 −9

Original line number	Diff line number	Diff line
		@@ -953,9 +953,6 @@ coverage-test-on-main-scheduled:
		- echo $log_files
		- ls wmops/logs
		- for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done
		- log_files_48=$(cat $public_dir/graphs_wmops_flc_48kHz.js \| grep logFile \| sed "s/.$wmops_newsletter_.\.csv$.*/\1/g")
		- echo $log_files_48
		- for f in $log_files_48; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done
		# copy index page blueprint
		- cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html
		# patch the format in the title
		@@ -969,7 +966,7 @@ coverage-test-on-main-scheduled:
		- .test-job-linux-needs-testv-dir
		tags:
		- test-complexity-measurement
		timeout: 7 hours
		timeout: 3 hours 30 minutes
		stage: test
		artifacts:
		name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"
		@@ -996,7 +993,7 @@ complexity-ism-in-binaural-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 2 hours
		start_in: 1 hour
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1012,7 +1009,7 @@ complexity-sba-hoa3-in-hoa3-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 5 hours
		start_in: 2 hours 30 minutes
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1028,7 +1025,7 @@ complexity-mc-in-7_1_4-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 8 hours
		start_in: 4 hours
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1044,7 +1041,7 @@ complexity-masa-in-7_1_4-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 15 hours
		start_in: 7 hours 30 minutes
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1060,7 +1057,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 17 hours
		start_in: 8 hours 30 minutes
		script:
		- *print-common-info
		- *update-ltv-repo

ci/complexity_measurements/getWmops.sh

+2 −8

Original line number	Diff line number	Diff line
		@@ -52,30 +52,24 @@ config_file="scripts/config/ci_linux_ltv.json"
		# get wmops newsletter
		wmopsFilenameFlcLast=wmops_newsletter_stereo__${commit_sha}_${date}
		wmopsFilenameFlc=${destDir}/wmops/logs/${wmopsFilenameFlcLast}
		wmopsFilenameFlc48kHzLast=wmops_newsletter_stereo_48kHz__${commit_sha}_${date}
		wmopsFilenameFlc48kHz=${destDir}/wmops/logs/${wmopsFilenameFlc48kHzLast}

		# instrument and build
		./scripts/IvasBuildAndRunChecks.py -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc} -C $ivas_format -f ${ep} --oc $output_format
		./scripts/IvasBuildAndRunChecks.py -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc48kHz} -C $ivas_format -R 48 -S 48 -f ${ep} --oc $output_format

		# now get the info on worst case operating point: WMOPS number, enc-operating mode, dec-operating mode
		### WMOPS
		${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcLast}_WMOPS.csv ${commit_sha} ${shortDate} ${fullDate} >> ${destDir}/wmops/log_wmops_all.txt
		${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc48kHz}_WMOPS.csv ${wmopsFilenameFlc48kHzLast}_WMOPS.csv ${commit_sha} ${shortDate} ${fullDate} >> ${destDir}/wmops/log_wmops_48kHz_all.txt

		# now update the webpage
		tcsh ${scriptDir}/genWebpageData_WMOPS.csh ${destDir}/wmops/log_wmops_all.txt ${destDir}/wmops/graphs_wmops_flc.js Graphs_WMOPS
		tcsh ${scriptDir}/genWebpageData_WMOPS.csh ${destDir}/wmops/log_wmops_48kHz_all.txt ${destDir}/wmops/graphs_wmops_flc_48kHz.js Graphs_WMOPS_48kHz

		# per mode graph
		tcsh ${scriptDir}/genWebpageData_WmopPerOperatingpoint.csh ${wmopsFilenameFlc}_WMOPS.csv ${destDir}/wmops/graphs_wmops_flc_perOP.js Graphs_WMOPS_perOP
		tcsh ${scriptDir}/genWebpageData_WmopPerOperatingpoint.csh ${wmopsFilenameFlc48kHz}_WMOPS.csv ${destDir}/wmops/graphs_wmops_flc_perOP_48kHz.js Graphs_WMOPS_perOP_48kHz

		# get memory info for webpage
		### RAM
		${scriptDir}/mergeNewsletterRam.py ${wmopsFilenameFlc48kHz}_SRAM.csv ${wmopsFilenameFlc48kHz}_DRAM.csv > ${wmopsFilenameFlc48kHz}_RAM.csv
		${scriptDir}/parseNewsletterRam.py ${wmopsFilenameFlc48kHz}_SRAM.csv ${wmopsFilenameFlc48kHz}_DRAM.csv ${wmopsFilenameFlc48kHzLast}_RAM.csv ${commit_sha} ${shortDate} ${fullDate} >> ${destDir}/wmops/log_ram_all.txt
		${scriptDir}/mergeNewsletterRam.py ${wmopsFilenameFlc}_SRAM.csv ${wmopsFilenameFlc}_DRAM.csv > ${wmopsFilenameFlc}_RAM.csv
		${scriptDir}/parseNewsletterRam.py ${wmopsFilenameFlc}_SRAM.csv ${wmopsFilenameFlc}_DRAM.csv ${wmopsFilenameFlcLast}_RAM.csv ${commit_sha} ${shortDate} ${fullDate} >> ${destDir}/wmops/log_ram_all.txt

		# generate java script from database
		tcsh ${scriptDir}/genWebpageData_Ram.csh ${destDir}/wmops/log_ram_all.txt ${destDir}/wmops/graphs_ram_flc.js Graphs_RAM

ci/complexity_measurements/index_complexity.html

+0 −332

Original line number	Diff line number	Diff line
		@@ -45,10 +45,8 @@
		<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/flot/0.8.3/jquery.flot.stack.min.js"></script>

		<script type="text/javascript" src="graphs_wmops_flc.js"></script>
		<script type="text/javascript" src="graphs_wmops_flc_48kHz.js"></script>
		<!-- <script type="text/javascript" src="graphs_wmops_basop.js"></script>-->
		<script type="text/javascript" src="graphs_wmops_flc_perOP.js"></script>
		<script type="text/javascript" src="graphs_wmops_flc_perOP_48kHz.js"></script>
		<!-- <script type="text/javascript" src="graphs_wmops_basop_perOP.js"></script>-->
		<script type="text/javascript" src="graphs_ram_flc.js"></script>
		<!-- <script type="text/javascript" src="graphs_ram_basop.js"></script>-->
		@@ -70,14 +68,6 @@

		<li><a href="#sec:graph-wmops_per_op">WMOPS Float per OP</a></li>

		<li><a href="#sec:graph-wmops_48kHz">WMOPS Float, 48kHz</a>
		<span class="symbols">
		<span class="trafficlight"><span id="wmops_48kHz_tl_l">●</span><span id="wmops_48kHz_tl_c">●</span><span id="wmops_48kHz_tl_r">●</span></span>
		<span class="trend"><span id="wmops_48kHz_trend">-</span></span></span>
		</li>

		<li><a href="#sec:graph-wmops_per_op_48kHz">WMOPS Float per OP, 48kHz</a></li>

		<!--
		<li><a href="#sec:graph-wmops-basop">WMOPS BASOP</a>
		<span class="symbols">
		@@ -177,42 +167,6 @@
		<hr />


		<h1 id="sec:graph-wmops_48kHz">IVAS FORMAT - Worst Case WMOPS Performance (Float, 48 kHz)</h1>

		<div class="graph-container">
		<div id="wmops-48kHz-graph"></div>
		</div>

		<div class="graph-container" style="clear: both;">
		<ul class="legend">
		<li style="border-color: #FF8000;"><em>Worst case encoder + decoder performance:</em> Encoder and decoder mode might be different.</li>
		<li style="border-color: #FFC480;"><em>Worst case encoder + decoder performance (rateswitching):</em> Encoder and decoder mode might be different.</li>
		<li style="border-color: #0080FF;"><em>Worst case codec performance:</em> Encoder and decoder modes are identical.</li>
		<li style="border-color: #40C4FF;"><em>Worst case codec performance (rateswitching):</em> Encoder and decoder modes are identical.</li>
		<li style="border-color: #CF4B4B;"><em>Worst case encoder performance</em> </li>
		<li style="border-color: #CF8080;"><em>Worst case encoder performance (rateswitching)</em> </li>
		<li style="border-color: #008040;"><em>Worst case decoder performance</em> </li>
		<li style="border-color: #00F040;"><em>Worst case decoder performance (rateswitching)</em> </li>
		</ul>
		</div>

		<hr />

		<h1 id="sec:graph-wmops_per_op_48kHz">IVAS FORMAT - Worst Case WMOPS Performance
		per Operating Point (Float, 48 kHz)</h1>

		<div class="graph-container">
		<div id="wmops_per_op-48kHz-graph"></div>
		</div>

		<div class="graph-container" style="clear: both; padding-top: 1em">
		<div id="wmops_per_op-48kHz-legend"></div>
		</div>

		<hr />



		<!--
		<h1 id="sec:graph-wmops-basop">IVAS FORMAT - Worst Case WMOPS Performance (BASOP)</h1>

		@@ -372,8 +326,6 @@

		<h1 id="sec:faq">FAQ</h1>
		<dl style="margin-left: 200px; margin-right: 200px;">
		<dt>Q:</dt><dd>What does "native SR" mean? What is the difference between the graphs titled "native SR" and "48 kHz"?</dd>
		<dt>A:</dt><dd>The graphs titled "native SR" run encoder/decoder at the native input/output sampling-rate per operating point. This is 16 kHz for WB, 32 kHz for SWB etc. The graphs titled "48 kHz" use 48 kHz input/output sampling rate, independent of the supported bandwidth, which is set by using the "-max_band" commandline switch.</dd>
		<dt>Q:</dt><dd>What is the meaning of these funny symbols in the navigation box, in the left upper corner of this page?</dd>
		<dt>A:</dt><dd>
		1) Traffic light <span style="color: #202020; background-color: #000000; opacity: 0.8; font-weight: bolder; font-size: 1.5em;"><span style="color: #FF0000;">●</span><span>●</span><span>●</span></span>, <span style="color: #202020; background-color: #000000; opacity: 0.8; font-weight: bolder; font-size: 1.5em;"><span>●</span><span style="color: #FFFF00;">●</span><span>●</span></span> or <span style="color: #202020; background-color: #000000; opacity: 0.8; font-weight: bolder; font-size: 1.5em;"><span>●</span><span>●</span><span style="color: #00FF00;">●</span></span>: !!!CURRENTLY NOT WORKING CORRECTLY AS NO REQUIREMENTS DEFINED YET!!! The traffic light symbols show, whether the last datapoint matches the requirement (green) or not (red). A yellow traffic light means that the requirement is matched, but the score is very close (within a 3% margin) to the requirement.<br />
		@@ -776,287 +728,6 @@ function WMOPS_perOP() {
		}




		function WMOPS_48kHz() {

		var previousPoint = null;

		function drawGraph(elt, graph, max_val) {
		var options = {
		yaxis: {
		min: 0,
		max: max_val,
		tickFormatter: function (v, axis) {
		if (graph.direction == -1)
		return v + " WMOPS";
		return v;
		},
		invert: graph.direction == 1
		},
		xaxis: {
		tickFormatter: function (v, axis) {
		v = Math.round(v);
		if (!(v in graph.runs))
		return '';
		return graph.runs[v].shortDate;
		}
		},
		legend: { show: false },
		grid: {
		hoverable: true,
		clickable: true
		}
		};

		$.plot(elt, graph.displays, options);

		elt.bind("plothover", function (event, pos, item) {
		if (!item) {
		// only remove if not in tooltip anymore
		if ($('#tooltip:hover').length == 0) {
		$("#tooltip").remove();
		}
		previousPoint = null;
		return;
		}

		if (previousPoint &&
		(previousPoint[0] == item.datapoint[0]) &&
		(previousPoint[1] == item.datapoint[1])) {
		return;
		}

		previousPoint = item.datapoint;
		$("#tooltip").remove();
		if(item.series.id != "requirement"){
		var x = item.datapoint[0];
		var y = item.datapoint[1];
		var text = 'Score: ' + y;

		if (graph.direction == -1)
		text += " WMOPS";
		text += "<br>";

		if (x > 0) {
		var thisValue = parseFloat(y);
		var prevValue = parseFloat(item.series.data[x - 1, x - 1][1]);
		var diff = Math.round((thisValue - prevValue) * 100) / 100;
		var pdiff = calcPercentDiff(thisValue, prevValue);
		var better;
		if ((pdiff < 0 && graph.direction == -1) \|\|
		(pdiff > 0 && graph.direction == 1)) {
		better = "worse";
		} else {
		better = "better";
		}
		pdiff = Math.abs(pdiff);
		if (diff === diff) {
		text += String.fromCharCode(916) + ": " + diff;
		if (graph.direction == -1)
		text += " WMOPS";
		text += " (" + pdiff + "% " + better + ")<br>";
		}
		}

		if( item.series.id == "worst case enc/dec" ){
		text += "Worst case enc: " + graph.runs[x].worstCaseEnc + "<br>";
		text += "Worst case dec: " + graph.runs[x].worstCaseDec + "<br>";
		}
		if( item.series.id == "worst case codec" ){
		text += "Worst case codec: " + graph.runs[x].worstCaseCodec + "<br>";
		}
		if( item.series.id == "worst case enc" ){
		text += "Worst case enc: " + graph.runs[x].worstCaseEnc + "<br>";
		}
		if( item.series.id == "worst case dec" ){
		text += "Worst case dec: " + graph.runs[x].worstCaseDec + "<br>";
		}
		if( item.series.id == "worst case enc/dec rs" ){
		text += "Worst case enc rateswitching: " + graph.runs[x].worstCaseEncRs + "<br>";
		text += "Worst case dec rateswitching: " + graph.runs[x].worstCaseDecRs + "<br>";
		}
		if( item.series.id == "worst case codec rs" ){
		text += "Worst case codec rateswitching: " + graph.runs[x].worstCaseCodecRs + "<br>";
		}
		if( item.series.id == "worst case enc rs" ){
		text += "Worst case enc rateswitching: " + graph.runs[x].worstCaseEncRs + "<br>";
		}
		if( item.series.id == "worst case dec rs" ){
		text += "Worst case dec rateswitching: " + graph.runs[x].worstCaseDecRs + "<br>";
		}


		text += "<br>"
		text += "Revision: " + graph.runs[x].revision + "<br>";
		text += "Date: " + graph.runs[x].fullDate + "<br>";
		text += "Fixpoint scal. fac. to reach 138 WMOPS: " + graph.runs[x].fixpointScalingFac + "<br><br>";

		text += "<a href=\"logs/" + graph.runs[x].logFile + "\">Logfile</a><br>";
		} else {
		text = "Complexity requirement: 135 WMOPS";
		}

		showToolTip(item.pageX, item.pageY, text);

		});
		}

		$(document).ready(function () {
		var max = get_max_y_val_for_plotting(Graphs_WMOPS_48kHz.wmops_worstcase.displays, 50);
		drawGraph($("#wmops-48kHz-graph"), Graphs_WMOPS_48kHz.wmops_worstcase, max);
		});

		var refData = Graphs_WMOPS_48kHz.wmops_worstcase.displays[0];
		var testData = Graphs_WMOPS_48kHz.wmops_worstcase.displays[2];
		var testDataRs = Graphs_WMOPS_48kHz.wmops_worstcase.displays[6];
		var nEntries = testData.data.length;

		if( testDataRs.data[nEntries-1][1] > testData.data[nEntries-1][1] ) {
		testData = testDataRs;
		}

		if(testData.data[nEntries-1][1] > refData.data[nEntries-1][1] ) {
		document.getElementById("wmops_48kHz_tl_l").style.color="#FF0000";
		} else if(testData.data[nEntries-1][1] > 0.97 * refData.data[nEntries-1][1] ) {
		document.getElementById("wmops_48kHz_tl_c").style.color="#FFFF00";
		} else {
		document.getElementById("wmops_48kHz_tl_r").style.color="#00FF00";
		}

		if(nEntries > 1) {
		if( testData.data[nEntries-1][1] > 1.01 * testData.data[nEntries-2][1] ) {
		document.getElementById("wmops_48kHz_trend").innerHTML="↑";
		document.getElementById("wmops_48kHz_trend").style.color="#FF0000";
		} else if(testData.data[nEntries-1][1] < 0.99 * testData.data[nEntries-2][1] ) {
		document.getElementById("wmops_48kHz_trend").innerHTML="↓";
		document.getElementById("wmops_48kHz_trend").style.color="#00FF00";
		} else {
		document.getElementById("wmops_48kHz_trend").innerHTML="→";
		document.getElementById("wmops_48kHz_trend").style.color="#FFFFFF";
		}
		}
		}


		function WMOPS_perOP_48kHz() {

		var previousPoint = null;

		function drawGraph(elt, graph, max_val) {
		var options = {
		yaxis: {
		min: 0,
		max: max_val,
		tickFormatter: function (v, axis) {
		if (graph.direction == -1)
		return v + " WMOPS";
		return v;
		},
		invert: graph.direction == 1
		},
		xaxis: {
		ticks: graph.ticks
		},
		legend: { show: true },
		grid: {
		hoverable: true,
		clickable: true
		},
		series: {
		stack: true,
		bars: {
		show: true,
		lineWidth: 2, // in pixels
		barWidth: 0.6, // in units of the x axis
		fill: true,
		fillColor: null,
		align: "center", // "left", "right", or "center"
		horizontal: false,
		zero: true
		}
		}
		};

		$.plot(elt, graph.displays, options);

		elt.bind("plothover", function (event, pos, item) {
		if (!item) {
		// only remove if not in tooltip anymore
		if ($('#tooltip:hover').length == 0) {
		$("#tooltip").remove();
		}
		previousPoint = null;
		return;
		}

		if (previousPoint &&
		(previousPoint[0] == item.datapoint[0]) &&
		(previousPoint[1] == item.datapoint[1])) {
		return;
		}

		previousPoint = item.datapoint;
		$("#tooltip").remove();

		var encData = Graphs_WMOPS_perOP_48kHz.wmops_worstcase_per_op.displays[0];
		var decData = Graphs_WMOPS_perOP_48kHz.wmops_worstcase_per_op.displays[1];

		var x = item.datapoint[0];
		var y = item.datapoint[1];

		var scoreEnc = parseFloat(encData.data[x][1]);
		var scoreDec = parseFloat(decData.data[x][1]);
		var scoreCodec = Math.round((scoreEnc + scoreDec) * 100) / 100;

		var text = "";

		text += "Mode: " + Graphs_WMOPS_perOP_48kHz.wmops_worstcase_per_op.runs[x].operatingPoint + "<br><br>";

		text += 'Score: ' + Math.round(y * 100) / 100;
		if (graph.direction == -1)
		text += " WMOPS";
		if( item.series.id == "worstCaseEnc" ){
		text += " (enc)";
		}
		if( item.series.id == "worstCaseDec" ){
		text += " (enc + dec)";
		}
		text += "<br><br>";

		text += "Worst case enc: " + scoreEnc + " WMOPS<br>";
		text += "Worst case dec: " + scoreDec + " WMOPS<br>";
		text += "Worst case codec: " + scoreCodec + " WMOPS<br><br>";

		var nEntriesWmopsGraph = Graphs_WMOPS_48kHz.wmops_worstcase.runs.length - 1;
		text += "<a href=\"logs/" + Graphs_WMOPS_48kHz.wmops_worstcase.runs[nEntriesWmopsGraph].logFile + "\">Logfile</a><br>";

		showToolTip(item.pageX, item.pageY, text);

		});

		}


		$(document).ready(function () {
		// need to get worst case of enc + dec combined, because values are stacked in the graph
		var max = get_max_y_val_for_plotting(Graphs_WMOPS_48kHz.wmops_worstcase.displays, 50);
		drawGraph($("#wmops_per_op-48kHz-graph"), Graphs_WMOPS_perOP_48kHz.wmops_worstcase_per_op, max);
		});

		var nEntriesWmopsGraph = Graphs_WMOPS_48kHz.wmops_worstcase.runs.length - 1;
		var legend = "<p style=\"text-align: center;\">Numbers derived from revision " +
		Graphs_WMOPS_48kHz.wmops_worstcase.runs[nEntriesWmopsGraph].revision +
		", " +
		Graphs_WMOPS_48kHz.wmops_worstcase.runs[nEntriesWmopsGraph].fullDate +
		"</p>";

		document.getElementById("wmops_per_op-48kHz-legend").innerHTML = legend;

		}



		function RAM() {

		var previousPoint = null;
		@@ -1558,9 +1229,6 @@ function PROM() {
		WMOPS();
		WMOPS_perOP();

		WMOPS_48kHz();
		WMOPS_perOP_48kHz();

		RAM();
		ROM();
		PROM();