pytest: add prepare_pytests.py to complement modified run_pytests.py for local development (688b860c) · Commits · IVAS Codec Public Collaboration / IVAS Codec

tests/README.md

+40 −1

Original line number	Diff line number	Diff line
		@@ -50,7 +50,7 @@ The tests rely on references which need to be generated upfront using reference
		When the reference binaries are named `IVAS_cod_ref(.exe)` and `IVAS_dec_ref(.exe)`, pytest will find and use them.
		When the reference binaries are named differently, you need to specify them via the `--ref_encoder_path` and `--ref_decoder_path` options.

		The tests will used the binaries `IVAS_cod(.exe)` and `IVAS_dec(.exe)` for testing. Please make sure that the binaries have been built before running the tests.
		The tests will use the binaries `IVAS_cod(.exe)` and `IVAS_dec(.exe)` for testing. Please make sure that the binaries have been built before running the tests.
		When different test binaries are to be used, they can be specified via the `--dut_encoder_path` and `--dut_decoder_path` options.
		(DUT: Device Under Test)

		@@ -129,3 +129,42 @@ The custom options are listed as part of the pytest help `pytest -h`.
		--keep_files By default, the DUT output files of successful tests are deleted.
		Use --keep_files to prevent these deletions.
		```

		## Helper scripts

		To help with running the tests during development, two scripts are available in the `tests` folder:

		- prepare_pytests.py
		- run_pytests.py

		The envisioned development workflow is:

		```bash
		# 1. create a new git branch and switch to the branch
		git checkout -b new_branch

		# 2. build the REF binaries (here: example for Linux)
		make -j

		# 3. use the binaries to generate the references for future tests
		# assumption: you want to test your development against the start of the development
		tests/prepare_pytests.py
		# Note: the script will use the binaries IVAS_cod and IVAS_dec in case IVAS_cod_ref and IVAS_dec_ref are not present

		# 3a. (optional) store REF binaries in case you want to re-run the reference generation at a later stage
		cp IVAS_cod IVAS_cod_ref
		cp IVAS_dec IVAS_dec_ref

		# 4. do the development changes
		edit ...

		# 5. build the DUT binaries (here: example for Linux)
		make -j

		# 6. run the tests
		tests/run_pytests.py

		# 7. depending on the test result
		# - either go back to 4.
		# - or commit and push your changes
		```

tests/prepare_pytests.py

0 → 100755

+123 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		__copyright__ = """
		(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository. All Rights Reserved.

		This software is protected by copyright law and by international treaties.
		The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other
		contributors to this repository retain full ownership rights in their respective contributions in
		the software. This notice grants no license of any kind, including but not limited to patent
		license, nor is any license granted by implication, estoppel or otherwise.

		Contributors are required to enter into the IVAS codec Public Collaboration agreement before making
		contributions.

		This software is provided "AS IS", without any express or implied warranties. The software is in the
		development stage. It is intended exclusively for experts who have experience with such software and
		solely for the purpose of inspection. All implied warranties of non-infringement, merchantability
		and fitness for a particular purpose are hereby disclaimed and excluded.

		Any dispute, controversy or claim arising under or in relation to providing this software shall be
		submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in
		accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		__doc__ = """
		Script to prepare the pytest tests.
		"""

		import os
		import sys
		import argparse
		import subprocess
		import platform

		from pathlib import Path
		from create_short_testvectors import create_short_testvectors

		BIN_EXT = ".exe" if platform.system() == "Windows" else ""
		HERE = Path(__file__).parent.resolve()
		DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve())
		DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve())
		DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve())
		DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve())
		REFERENCE_DIR = str(HERE.joinpath("ref").resolve())


		def main(argv):
		"""
		Prepare the pytest tests.
		"""
		# check for python >= 3.7
		if sys.version_info[0] < 3 or sys.version_info[1] < 7:
		sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version())

		parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
		parser.add_argument(
		"--numprocesses",
		action="store",
		default="auto",
		help="Number of processes to use in pytest (default: auto)",
		)

		args = parser.parse_args(argv[1:])

		use_dut_binaries = False

		# check for existing references
		if os.path.exists(REFERENCE_DIR):
		sys.exit(
		f"Found existing references directory {REFERENCE_DIR}.\n"
		"Please delete this directory if you want the references to be recreated."
		)

		# check for DUT binaries
		if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT):
		sys.exit(
		f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}.\n"
		"Please create the binaries."
		)

		# check for REF binaries
		if not os.path.exists(DEFAULT_ENCODER_REF) or not os.path.exists(DEFAULT_DECODER_REF):
		print(f"REF binaries {DEFAULT_ENCODER_REF} and {DEFAULT_DECODER_REF} not found.")
		print("DUT binaries will be used for reference generation.")
		use_dut_binaries = True

		# create references
		print(f"Creating references within the references directory {REFERENCE_DIR}.")
		create_short_testvectors()
		if platform.system() == "Windows":
		base_cmd = ["pytest"]
		else:
		base_cmd = ["python3", "-m", "pytest"]
		base_cmd += [
		"tests",
		"-n",
		args.numprocesses,
		"--update_ref",
		"1",
		]
		if use_dut_binaries:
		base_cmd += [
		"--ref_encoder_path",
		DEFAULT_ENCODER_DUT,
		"--ref_decoder_path",
		DEFAULT_DECODER_DUT,
		]

		result = subprocess.run(base_cmd + ["-m", "create_ref"], check=False)
		result = subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False)
		return result.returncode


		if __name__ == "__main__":
		sys.exit(main(sys.argv))

tests/run_pytests.py

+15 −45

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		__copyright__ = \
		"""
		__copyright__ = """
		(C) 2022 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB,
		Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD.,
		Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange,
		@@ -31,8 +30,7 @@ accordance with the laws of the Federal Republic of Germany excluding its confli
		the United Nations Convention on Contracts on the International Sales of Goods.
		"""

		__doc__ = \
		"""
		__doc__ = """
		Script to run the pytest tests.

		Test prerequisites are checked for and check failures are reported.
		@@ -46,30 +44,22 @@ import subprocess
		import platform
		from pathlib import Path

		sys.path.append('tests/')
		from create_short_testvectors import create_short_testvectors

		BIN_EXT = ".exe" if platform.system() == "Windows" else ""
		HERE = Path(__file__).parent.resolve()
		DEFAULT_ENCODER_DUT = str(HERE.joinpath(f"../IVAS_cod{BIN_EXT}").resolve())
		DEFAULT_DECODER_DUT = str(HERE.joinpath(f"../IVAS_dec{BIN_EXT}").resolve())
		DEFAULT_ENCODER_REF = str(HERE.joinpath(f"../IVAS_cod_ref{BIN_EXT}").resolve())
		DEFAULT_DECODER_REF = str(HERE.joinpath(f"../IVAS_dec_ref{BIN_EXT}").resolve())
		REFERENCE_DIR = str(HERE.joinpath("ref").resolve())


		def main(argv):
		"""
		Run the pytest tests.
		"""
		# check for python >= 3.7
		if sys.version_info[0] < 3 or sys.version_info[1] < 7:
		sys.exit("This script is written for Python >= 3.7. Found: " + platform.python_version())

		parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
		parser.add_argument(
		"--create_only",
		action="store_true",
		default=False,
		help="Create references when needed, but don't run the tests"
		)
		parser.add_argument(
		"--numprocesses",
		action="store",
		@@ -79,38 +69,18 @@ def main(argv):

		args = parser.parse_args(argv[1:])

		# check for DUT binaries
		if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT):
		sys.exit(f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}. Please create the binaries.")

		# check for references
		if os.path.exists(REFERENCE_DIR):
		print(f"Using existing references directory {REFERENCE_DIR}")
		else:
		# check for REF binaries
		print(f"References directory {REFERENCE_DIR} does not exist.")
		if not os.path.exists(DEFAULT_ENCODER_REF) or not os.path.exists(DEFAULT_DECODER_REF):
		sys.exit(f"Need REF binaries {DEFAULT_ENCODER_REF} and {DEFAULT_DECODER_REF}. Please create the binaries.")

		# create references
		print(f"Creating references within the references directory {REFERENCE_DIR}")
		create_short_testvectors()
		if platform.system() == "Windows":
		base_cmd = ["pytest"]
		else:
		base_cmd = ["python3", "-m", "pytest"]
		base_cmd += [
		"tests",
		"-n",
		args.numprocesses,
		"--update_ref",
		"1",
		]
		subprocess.run(base_cmd + ["-m", "create_ref"], check=False)
		subprocess.run(base_cmd + ["-m", "create_ref_part2"], check=False)
		if not os.path.exists(REFERENCE_DIR):
		sys.exit(
		f"References directory {REFERENCE_DIR} not found.\n" "Please create the references."
		)

		if args.create_only:
		return
		# check for DUT binaries
		if not os.path.exists(DEFAULT_ENCODER_DUT) or not os.path.exists(DEFAULT_DECODER_DUT):
		sys.exit(
		f"Need DUT binaries {DEFAULT_ENCODER_DUT} and {DEFAULT_DECODER_DUT}.\n"
		"Please create the binaries."
		)

		# run pytest
		if platform.system() == "Windows":