Merge branch 'draft/lukem-more-lint' into 'meeting/SA99' (1b5d04c4) · Commits · SA3 / SA3 LI

testing/asn_backwards_compat.py

0 → 100644

+137 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		"""
		Module to check backwards compatibility between two ASN.1 files.
		"""

		import logging
		import os
		import sys

		import asn1tools


		# pylint: disable=logging-fstring-interpolation


		def process_error(errors, message):
		"""Log `message` and append `message` to `errors`."""
		logging.info(f"Test Failure: {message}")
		errors.append(message)


		def members_as_tag_dict(atype):
		"""Convert atype['members'] into a dict indexed by members.tag.number."""
		result = {}
		for member in atype.get("members", {}):
		if not member:
		continue
		if not "tag" in member:
		continue
		result[member["tag"]["number"]] = member
		return result


		def compare_files(file1, file2):
		"""Compare ASN.1 `file1` and `file2` for backwards compatibility
		issues introduced in `file2`:
		- Module is the same
		- Each type in the module:
		- Exists
		- Has the same member types
		- Each complex member type has:
		- Same inner type at the tag
		- Warns if the type renamed (not fatal)

		Returns a list of error messages (if any).
		"""
		# pylint: disable=too-many-locals

		errors = []

		f1_asn = asn1tools.parse_files(file1)
		f2_asn = asn1tools.parse_files(file2)

		logging.info(f"Comparing file {file1} with {file2}")
		for module_name, f1_module in f1_asn.items():
		logging.info(f"Checking module {module_name}")
		f2_module = f2_asn.get(module_name)
		if not f2_module:
		process_error(errors, f"Module {module_name} not present")
		continue
		f2_types = f2_module["types"]
		for type_name, f1_type_def in f1_module["types"].items():
		logging.info(f"Checking type {type_name}")
		f2_type_def = f2_types.get(type_name)
		if not f2_type_def:
		process_error(errors, f"Type {type_name} not present")
		continue
		if f1_type_def["type"] != f2_type_def["type"]:
		process_error(
		errors,
		f"Type {type_name} type '{f1_type_def['type']}'"
		f" mismatch with '{f2_type_def['type']}'",
		)
		continue
		f1_tags = members_as_tag_dict(f1_type_def)
		f2_tags = members_as_tag_dict(f2_type_def)
		if not f1_tags:
		if f2_tags:
		process_error(
		errors, f"Type {type_name} contains unexpected members"
		)
		continue
		for tag_number, f1_tag in f1_tags.items():
		logging.info(
		f"Checking {f1_tag['name']} [{tag_number}] {f1_tag['type']}"
		)
		f2_tag = f2_tags.get(tag_number, {})
		if not f2_tag:
		process_error(
		errors,
		f"Type {type_name} tag {tag_number} field '{f1_tag['name']}'"
		f" missing",
		)
		continue
		if f1_tag["type"] != f2_tag["type"]:
		process_error(
		errors,
		f"Type {type_name} tag {tag_number} field '{f1_tag['name']}'"
		f" type '{f1_tag['type']}' mismatch with '{f2_tag['type']}'",
		)
		continue
		if f1_tag["name"] != f2_tag["name"]:
		process_error(
		errors,
		f"Type {type_name} tag {tag_number} field '{f1_tag['name']}'"
		f" renamed to '{f2_tag['name']}'",
		)
		# this is not fatal - continue
		# no other checks for now

		return errors


		def main():
		"""standalone main."""
		loglevel = os.environ.get("LOGLEVEL", "WARNING").upper()
		logging.basicConfig(level=loglevel)

		if len(sys.argv) != 3:
		raise RuntimeError(f"Usage: {sys.argv[0]} file1.asn file2.asn")
		file1 = sys.argv[1]
		file2 = sys.argv[2]
		results = compare_files(file1, file2)
		if results:
		print("-----------------------------")
		print(f"File 1: {file1}")
		print(f"File 2: {file2}")
		print("Errors:")
		for error in results:
		print(f" {error}")
		print("-----------------------------")
		sys.exit(1)


		if __name__ == "__main__":
		main()

testing/asn_lint_exceptions.json

+42 −4

Original line number	Diff line number	Diff line
		{
		"33128/r15/TS33128Payloads.asn" : [
		"Tag 16 IRIEvent field 'mDFCellSiteReport' is not present in XIRIEvent",
		"Tag 5 XIRIEvent field 'unsuccessfulAMProcedure' differs from IRIEvent field 'unsuccessfulRegistrationProcedure'",
		"Tag 10 XIRIEvent field 'unsuccessfulSMProcedure' differs from IRIEvent field 'unsuccessfulSessionProcedure'",
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE"
		],
		"33128/r16/TS33128Payloads.asn" : [
		"Tag 16 IRIEvent field 'mDFCellSiteReport' is not present in XIRIEvent",
		"Tag 5 XIRIEvent field 'unsuccessfulAMProcedure' differs from IRIEvent field 'unsuccessfulRegistrationProcedure'",
		"Tag 10 XIRIEvent field 'unsuccessfulSMProcedure' differs from IRIEvent field 'unsuccessfulSessionProcedure'",
		"Tag 16 missing in XIRIEvent",
		"Enumerations for EstablishmentStatus start at 0, not 1",
		"Enumerations for RequestIndication start at 0, not 1",
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Enumerations for MMSDirection start at 0, not 1",
		"Enumerations for MMSReplyCharging start at 0, not 1",
		"Enumerations for MMStatusExtension start at 0, not 1"
		"Enumerations for MMStatusExtension start at 0, not 1",
		"Tag 2 missing in LALSReport",
		"Tag 6 missing in LALSReport"
		],
		"33128/r17/TS33128Payloads.asn" : [
		"Tag 100 XIRIEvent field 'n9HRPDUSessionInfo' is not present in IRIEvent",
		"Tag 101 XIRIEvent field 's8HRBearerInfo' is not present in IRIEvent",
		"Tag 16 IRIEvent field 'mDFCellSiteReport' is not present in XIRIEvent",
		"Tag 5 XIRIEvent field 'unsuccessfulAMProcedure' differs from IRIEvent field 'unsuccessfulRegistrationProcedure'",
		"Tag 10 XIRIEvent field 'unsuccessfulSMProcedure' differs from IRIEvent field 'unsuccessfulSessionProcedure'",
		"Tag 108 XIRIEvent field 'uDMLocationInformationResult' differs from IRIEvent field 'uDMLocationInformationResultRecord'",
		"Tag 16 missing in XIRIEvent",
		"Tags 113-131 missing in XIRIEvent",
		"Tags 100-101 missing in IRIEvent",
		"Tags 113-131 missing in IRIEvent",
		"Tag 12 missing in SCEFCommunicationPatternUpdate",
		"Enumerations for EstablishmentStatus start at 0, not 1",
		"Enumerations for RequestIndication start at 0, not 1",
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Enumerations for MMSDirection start at 0, not 1",
		"Enumerations for MMSReplyCharging start at 0, not 1",
		"Enumerations for MMStatusExtension start at 0, not 1"
		"Enumerations for MMStatusExtension start at 0, not 1",
		"Tags 4-5 missing in IMSMessage",
		"Tag 6 missing in StartOfInterceptionForActiveIMSSession",
		"Tag 2 missing in LALSReport",
		"Tag 6 missing in LALSReport",
		"Tag 8 missing in MMEStartOfInterceptionWithEPSAttachedUE",
		"Tag 11 missing in MMEStartOfInterceptionWithEPSAttachedUE"
		],
		"33128/r18/TS33128Payloads.asn" : [
		"Tag 100 XIRIEvent field 'n9HRPDUSessionInfo' is not present in IRIEvent",
		"Tag 101 XIRIEvent field 's8HRBearerInfo' is not present in IRIEvent",
		"Tag 16 IRIEvent field 'mDFCellSiteReport' is not present in XIRIEvent",
		"Tag 16 missing in XIRIEvent",
		"Tags 100-101 missing in IRIEvent",
		"Tag 12 missing in SCEFCommunicationPatternUpdate",
		"Enumerations for EstablishmentStatus start at 0, not 1",
		"Enumerations for RequestIndication start at 0, not 1",
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Enumerations for MMSDirection start at 0, not 1",
		"Enumerations for MMSReplyCharging start at 0, not 1",
		"Enumerations for MMStatusExtension start at 0, not 1"
		"Enumerations for MMStatusExtension start at 0, not 1",
		"Tags 4-5 missing in IMSMessage",
		"Tag 6 missing in StartOfInterceptionForActiveIMSSession",
		"Tag 2 missing in LALSReport",
		"Tag 6 missing in LALSReport",
		"Tag 8 missing in MMEStartOfInterceptionWithEPSAttachedUE",
		"Tag 11 missing in MMEStartOfInterceptionWithEPSAttachedUE"
		]
		}

testing/asn_process.py

100644 → 100755

+64 −62

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import logging
		import os
		import json
		from pathlib import Path
		from subprocess import run
		@@ -21,27 +24,20 @@ def syntaxCheckASN (fileList):
		results = {}
		for file in fileList:
		try:
		p = run(['asn1c', '-E', str(file)], capture_output=True)
		if (p.returncode != 0):
		p = run(["asn1c", "-E", str(file)], capture_output=True)
		if p.returncode != 0:
		results[str(file)] = {
		'ok' : False,
		'code' : p.returncode,
		'message' : p.stderr.decode().splitlines()[0]
		"ok": False,
		"code": p.returncode,
		"message": p.stderr.decode().splitlines()[0],
		}
		else:
		results[str(file)] = {
		'ok' : True
		}
		results[str(file)] = {"ok": True}
		except Exception as ex:
		results[str(file)] = {
		'ok' : False,
		'code' : -1,
		'message' : f"{ex!r}"
		}
		results[str(file)] = {"ok": False, "code": -1, "message": f"{ex.__class__.__qualname__}: {ex}"}
		return results



		def compileAllTargets(compileTargets):
		"""
		Attempts to compile a set of compile targets using the pycrate ASN1 tools
		@@ -75,19 +71,18 @@ def compileAllTargets (compileTargets):
		logging.debug(f" Loading {filename}")
		compile_text(fileTexts, filenames=fileNames)
		results[str(firstTarget)] = {
		'ok' : True,
		"ok": True,
		}
		except Exception as ex:
		results[str(firstTarget)] = {
		'ok' : False,
		'code' : -1,
		'message' : f"{ex!r}"
		"ok": False,
		"code": -1,
		"message": f"{ex!r}",
		}
		continue
		return results



		def processResults(results, stageName):
		"""
		Counts the number of errors and writes out the output per filename
		@@ -97,7 +92,7 @@ def processResults (results, stageName):
		:returns: The number of files which had errors
		"""
		print("")
		errorCount = sum([1 for r in results.values() if not r['ok']])
		errorCount = sum([1 for r in results.values() if not r["ok"]])
		logging.info(f"{errorCount} {stageName} errors encountered")

		print(f"{'-':-<60}")
		@@ -105,9 +100,9 @@ def processResults (results, stageName):
		print(f"{'-':-<60}")
		for filename, result in results.items():
		print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}")
		if not result['ok']:
		if isinstance(result['message'], list):
		for thing in result['message']:
		if not result["ok"]:
		if isinstance(result["message"], list):
		for thing in result["message"]:
		print(f" {thing['message']}")
		else:
		print(f" {result['message']}")
		@@ -119,30 +114,33 @@ def processResults (results, stageName):
		return errorCount


		if __name__ == '__main__':
		logging.info('Searching for ASN.1 files')
		def main():
		loglevel = os.environ.get("LOGLEVEL", "WARNING").upper()
		logging.basicConfig(level=loglevel)

		logging.info("Searching for ASN.1 files")
		fileList = list(Path(".").rglob(".asn1")) + list(Path(".").rglob(".asn"))
		logging.info(f'{len(fileList)} ASN.1 files found')
		logging.info(f"{len(fileList)} ASN.1 files found")
		for file in fileList:
		logging.debug(f' {file}')
		logging.debug(f" {file}")

		ignoreList = Path('testing/asn_ignore.txt').read_text().splitlines()
		ignoreList = Path("testing/asn_ignore.txt").read_text().splitlines()
		ignoredFiles = []
		for ignore in ignoreList:
		logging.debug(f'Ignoring pattern {ignore}')
		logging.debug(f"Ignoring pattern {ignore}")
		for file in fileList:
		if ignore in str(file):
		ignoredFiles.append(file)
		logging.debug(f" Ignoring {str(file)} as contains {ignore}")
		ignoredFiles = list(set(ignoredFiles))
		logging.info(f'{len(ignoredFiles)} files ignored')
		logging.info(f"{len(ignoredFiles)} files ignored")
		for file in ignoredFiles:
		logging.debug(f' {file}')
		logging.debug(f" {file}")

		fileList = [file for file in fileList if file not in ignoredFiles]
		logging.info(f'{len(fileList)} files to process')
		logging.info(f"{len(fileList)} files to process")
		for file in fileList:
		logging.debug(f' {file}')
		logging.debug(f" {file}")

		if len(fileList) == 0:
		logging.warning("No files specified")
		@@ -151,33 +149,37 @@ if __name__ == '__main__':
		logging.info("Parsing ASN1 files")
		parseResults = syntaxCheckASN(fileList)
		if processResults(parseResults, "Parsing") > 0:
		exit(-1)
		exit(1)

		logging.info("Getting compile targets")
		compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text())
		compileTargets = json.loads(Path("testing/asn_compile_targets.json").read_text())
		logging.info(f"{len(compileTargets)} compile targets found")

		compileResults = compileAllTargets(compileTargets)
		if processResults(compileResults, "Compiling") > 0:
		exit(-1)
		exit(1)

		logging.info("Linting files")
		ignoreLintingList = Path('testing/asn_ignore_lint.txt').read_text().splitlines()
		ignoreLintingList = Path("testing/asn_ignore_lint.txt").read_text().splitlines()
		ignoredFiles = []
		for ignore in ignoreLintingList:
		logging.debug(f'Ignoring pattern {ignore} for linting')
		logging.debug(f"Ignoring pattern {ignore} for linting")
		for file in fileList:
		if ignore in str(file):
		ignoredFiles.append(file)
		logging.debug(f" Ignoring {str(file)} for linting as contains {ignore}")
		ignoredFiles = list(set(ignoredFiles))
		logging.info(f'{len(ignoredFiles)} files ignored for linting')
		logging.info(f"{len(ignoredFiles)} files ignored for linting")
		for file in ignoredFiles:
		logging.debug(f' {file}')
		logging.debug(f" {file}")
		fileList = [file for file in fileList if file not in ignoredFiles]
		lintExceptions = json.loads(Path('testing/asn_lint_exceptions.json').read_text())
		lintExceptions = json.loads(Path("testing/asn_lint_exceptions.json").read_text())
		lintResults = lint_asn1.lintASN1Files(fileList, lintExceptions)
		if processResults(lintResults, "Linting") > 0:
		exit(-1)
		exit(1)

		exit(0)


		if __name__ == "__main__":
		main()

testing/check_asn_backwards_compat.py

0 → 100644

+56 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import os
		import sys

		import asn_backwards_compat


		def compare_releases(files):
		total_errors = 0
		for idx in range(len(files) - 1):
		file1 = files[idx]
		file2 = files[idx + 1]
		errors = asn_backwards_compat.compare_files(file1, file2)
		if errors:
		print("-----------------------------")
		print(f"File 1: {file1}")
		print(f"File 2: {file2}")
		print("Errors:")
		for error in errors:
		print(f" {error}")
		print("-----------------------------")
		total_errors += len(errors)
		return total_errors


		def main():
		loglevel = os.environ.get("LOGLEVEL", "WARNING").upper()
		logging.basicConfig(level=loglevel)

		error_count = 0

		error_count += compare_releases(
		[
		"33128/r15/TS33128Payloads.asn",
		"33128/r16/TS33128Payloads.asn",
		"33128/r17/TS33128Payloads.asn",
		"33128/r18/TS33128Payloads.asn",
		]
		)

		error_count += compare_releases(
		[
		"33128/r16/TS33128IdentityAssociation.asn",
		"33128/r17/TS33128IdentityAssociation.asn",
		"33128/r18/TS33128IdentityAssociation.asn",
		]
		)

		if error_count:
		print(f"Total errors: {error_count}")
		sys.exit(1)


		if __name__ == "__main__":
		main()

testing/lint_asn1.py

+326 −130

File changed.

Preview size limit exceeded, changes collapsed.