Merge remote-tracking branch 'origin/main' into 1438-rtpdump-is-not-rtpdump (016c809f) · Commits · IVAS Codec Public Collaboration / IVAS Codec

scripts/ivas_conformance/README_Package_Gen.md

+17 −21

Original line number	Diff line number	Diff line
		@@ -53,10 +53,7 @@ This folder contains scripts for running IVAS conformance tests.

		## Reference Conformance Package Generation

		<details>
		<summary> <i> Expand for detailed procedure </i> </summary>

		To generate reference conformance package for distribution
		To generate the reference conformance package follow these steps in specified order.

		### Generate Reference Outputs and Readme.txt files

		@@ -64,33 +61,32 @@ To generate reference conformance package for distribution
		sh scripts/ivas_conformance/ivas_be_conf_test_gen.sh
		```

		<details>
		<summary> Example output of reference test generation </summary>
		<pre><code>
		::::::::::::::::::::::::
		------------------------------------------
		Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm
		------------------------------------------
		=================================================
		2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)
		=================================================
		Identified 5430 files from scripts
		Removed 1515 files
		Kept 5422 files
		</code></pre>
		</details>

		### Generate Reference Decoded Outputs for the Reference Encoded files

		```shell
		PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-enc-refs
		```

		### Generate MLD values on Reference/Target platform

		First, copy the CUT binaries to the `CUT_BIN_DIR` folder and execute the IVAS CUT commands on the reference/target platform:

		```shell
		PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR
		```

		Then perform the MLD-based non-BE analysis on the CUT outputs:

		```shell
		PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-mld-ref
		```

		This process creates a new directory at `$PWD/testvec/testv/mld_ref2` containing the updated MLD values.

		### Generate a conformance package zip

		```shell
		zip -r conformance.zip testvec scripts/ivas_conformance/runConformance.py scripts/ivas_conformance/requirements.txt scripts/ivas_conformance/README.md scripts/ivas_conformance/thirdPartyLegalNotices.txt scripts/tools/Darwin/thirdPartyLegalnotices scripts/tools/Darwin/LICENSE_wav-diff.txt scripts/tools/Darwin/wav-diff scripts/tools/Linux/thirdPartyLegalnotices scripts/tools/Linux/LICENSE_wav-diff.txt scripts/tools/Linux/wav-diff scripts/tools/Win32/thirdPartyLegalnotices scripts/tools/Win32/LICENSE_wav-diff.txt scripts/tools/Win32/wav-diff.exe
		```

		</details>

scripts/ivas_conformance/runConformance.py

+544 −101

File changed.

Preview size limit exceeded, changes collapsed.

scripts/parse_commands.py

+22 −8

Original line number	Diff line number	Diff line
		@@ -35,11 +35,27 @@ if __name__ == '__main__':
		for html_report in input:

		with open(html_report,'r') as infile:
		for line in infile.readlines():
		lines = infile.readlines()
		for idx, line in enumerate(lines):
		# parse encoder and decoder commands
		cmds_enc.extend(re.findall(r"REF encoder command:\\n\\t(.*?)\\n", line))
		cmds_dec.extend(re.findall(r"REF decoder command:\\n\\t(.*?)\\n", line))

		# parse renderer commands, including ISAR EXT REND commands (the next line contains IVAS_rend or IVAS_rend_ref)
		cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line))
		next_line = lines[idx + 1] if idx + 1 < len(lines) else ""
		if (
		"Running ISAR EXT REND command" in line
		and ("IVAS_rend" in next_line or "IVAS_rend_ref" in next_line)
		):
		cmds_rend.extend(re.findall(r"Running ISAR EXT REND command\\n(.*?)\\n", line))

		# parse ISAR post renderer commands, including ISAR EXT REND commands (the next line contains ISAR_post_rend or ISAR_post_rend_ref)
		cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line))
		if (
		"Running ISAR EXT REND command" in line
		and ("ISAR_post_rend" in next_line or "ISAR_post_rend_ref" in next_line)
		):
		cmds_isar_post_rend.extend(re.findall(r"Running ISAR EXT REND command\\n(.*?)\\n", line))

		# If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens.
		@@ -165,15 +181,13 @@ if __name__ == '__main__':
		for output in glob.glob(absolute_out.group(1) + '*'):
		output = path.relpath(output).replace('\\','/')
		output = re.sub('tests', CUT_PATH, output)
		diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1')
		diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1\n')
		outfile.write(('; ').join(diff_cmds))
		if isar_out and "ref" in isar_out.group(1):
		outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1')
		outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1\n')
		if isar_md_out and "ref" in isar_md_out.group(1):
		outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n')
		else:
		outfile.write('\n')
		outfile.write('\n\n')
		with open('scripts/script_footer.txt','r') as footer:
		outfile_dec.write(footer.read())
		footer.seek(0)
		@@ -208,7 +222,7 @@ if __name__ == '__main__':
		for output in glob.glob(absolute_out.group(1) + '*'):
		output = path.relpath(output).replace('\\','/')
		output = re.sub('tests', CUT_PATH, output)
		diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/renderer_short/ref',REF_PATH + r'/renderer_short/ref')+' '+ output +' >> $LOG_FILE 2>&1')
		diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/renderer_short/ref',REF_PATH + r'/renderer_short/ref')+' '+ output +' >> $LOG_FILE 2>&1\n')
		outfile.write(('; ').join(diff_cmds))
		outfile.write('\n')
		with open('scripts/script_footer.txt','r') as footer: