Refactoring stages/jobs (8f8684d7) · Commits · SA3 / SA3 LI

.gitlab-ci.yml

+4 −13

Original line number	Diff line number	Diff line
		image: "mcanterb/asn1test:latest"

		stages:
		- process
		- check

		process_asn:
		stage: process
		image: "mcanterb/asn1test:latest"
		stage: check
		interruptible: true
		script:
		- python3 testing/asn_process.py
		artifacts:
		paths:
		- testing/asn_compile_results.json
		- testing/asn_parse_results.json
		expire_in: 1 day

		check_asn_parse:
		process_xsd:
		image: python:3.9-slim-bullseye
		stage: check
		interruptible: true
		dependencies:
		- process_asn
		script:
		- python3 testing/asn_check_parsing.py
		- python3 --version

testing/asn_check_compile.py

0 → 100644

+19 −0

Original line number	Diff line number	Diff line
		from pathlib import Path
		import json

		if __name__ == "__main__":
		parseResults = json.loads(Path('testing/asn_compile_results.json').read_text())

		print(f"{'-':-<60}")
		print("Parser results:")
		print(f"{'-':-<60}")
		for filename, result in parseResults['results'].items():
		print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}")
		if not result['ok']:
		print(f" {result['message']}")

		print(f"{'-':-<60}")
		print(f"Parser errors: {parseResults['errorCount']}")
		print(f"{'-':-<60}")

		exit(parseResults['errorCount'])
		No newline at end of file

testing/asn_lint_exceptions.json

0 → 100644

+22 −0

Original line number	Diff line number	Diff line
		{
		"33128/r15/TS33128Payloads.asn" : [
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE"
		],
		"33128/r16/TS33128Payloads.asn" : [
		"Enumerations for EstablishmentStatus start at 0, not 1",
		"Enumerations for RequestIndication start at 0, not 1",
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Enumerations for MMSDirection start at 0, not 1",
		"Enumerations for MMSReplyCharging start at 0, not 1",
		"Enumerations for MMStatusExtension start at 0, not 1"
		],
		"33128/r17/TS33128Payloads.asn" : [
		"Enumerations for EstablishmentStatus start at 0, not 1",
		"Enumerations for RequestIndication start at 0, not 1",
		"Enumerations for UDMServingSystemMethod start at 0, not 1",
		"Enumerations for MMSDirection start at 0, not 1",
		"Enumerations for MMSReplyCharging start at 0, not 1",
		"Enumerations for MMStatusExtension start at 0, not 1"
		]
		}
		No newline at end of file

testing/asn_process.py

+77 −60

Original line number	Diff line number	Diff line
		import logging
		import json
		from pathlib import Path

		from subprocess import run
		import lint_asn1
		from py import process

		from pycrate_asn1c.asnproc import *

		ignoreReleases = {'33108' : [f'r{i}' for i in range(5, 17)],
		'33128' : [],
		'testing' : [] }
		import lint_asn1

		def prepareFile(f):
		with open(f) as fh:
		s = fh.read()
		s = s.replace("RELATIVE-OID", "OBJECT IDENTIFIER") # sigh
		return s

		def parseFile(f):
		p = run(['asn1c', '-E', str(f)], capture_output=True)
		if (p.returncode != 0):
		raise Exception(f"Compiler error: {p.returncode} - {p.stderr.decode().splitlines()[0]}")
		return p.stdout.decode()

		def compileFiles(fileList):
		p = run(['asn1c', '-P', '-fcompound-names'] + [str(f) for f in fileList], capture_output=True)
		results = {
		'result' : p.returncode,
		'errors' : [line for line in p.stderr.decode().splitlines() if len(line.strip()) > 0 and not line.startswith('WARNING')],
		'warnings' : [line for line in p.stderr.decode().splitlines() if line.startswith('WARNING') if not "/usr/local/share/asn1c" in line]
		}
		return results
		def syntaxCheckASN (fileList):
		"""
		Performs ASN syntax checking on a list of filenames (or pathlib Paths)

		:param fileList: List of filenames (str or Pathlib Path)
		:returns: Dict with result, return code and message for each filename

		def parseASN (fileList):
		Calls the open-source asn1c compiler with the "syntax only" option.
		As a result, asn1c must be available to run.
		"""
		results = {}
		for file in fileList:
		try:
		@@ -42,7 +27,7 @@ def parseASN (fileList):
		results[str(file)] = {
		'ok' : False,
		'code' : p.returncode,
		'msg' : p.stderr.decode().splitlines()[0]
		'message' : p.stderr.decode().splitlines()[0]
		}
		else:
		results[str(file)] = {
		@@ -52,11 +37,30 @@ def parseASN (fileList):
		results[str(file)] = {
		'ok' : False,
		'code' : -1,
		'msg' : f"{ex!r}"
		'message' : f"{ex!r}"
		}
		return results



		def compileAllTargets (compileTargets):
		"""
		Attempts to compile a set of compile targets using the pycrate ASN1 tools

		:param compileTargets: list of compile targets, each of which is a list of filenames
		:returns: A dict of outcome against the first filename of each compile target. Return code and message are included for failures.

		For each compile target (list of filenames) the first filename is assumed
		to be the "primary" file. This doesn't have any relavance to the compilation,
		but will be used as the identifier when reporting any compile errors.
		The compilation is performed by the pycrate ASN compile functions; errors
		are caught as exceptions and rendered into a list.

		Unfortunately, the pycrate compiler doesn't report line numbers.
		The asn1c compiler does, but doesn't properly handle identifiers with the
		same name in different modules; as this occurs multiple times in TS 33.108,
		we can't use it.
		"""
		results = {}
		for target in compileTargets:
		firstTarget = target[0]
		@@ -84,9 +88,32 @@ def compileAllTargets (compileTargets):
		return results


		if __name__ == '__main__':
		logging.basicConfig(level=logging.INFO)

		def processResults (results, stageName):
		print("")
		errorCount = sum([1 for r in results.values() if not r['ok']])
		logging.info(f"{errorCount} {stageName} errors encountered")

		print(f"{'-':-<60}")
		print(f"{stageName} results:")
		print(f"{'-':-<60}")
		for filename, result in results.items():
		print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}")
		if not result['ok']:
		if isinstance(result['message'], list):
		for thing in result['message']:
		print(f" {thing['message']}")
		else:
		print(f" {result['message']}")

		print(f"{'-':-<60}")
		print(f"{stageName} errors: {errorCount}")
		print(f"{'-':-<60}")

		return errorCount


		if __name__ == '__main__':
		logging.info('Searching for ASN.1 files')
		fileList = list(Path(".").rglob(".asn1")) + list(Path(".").rglob(".asn"))
		logging.info(f'{len(fileList)} ASN.1 files found')
		@@ -111,33 +138,22 @@ if __name__ == '__main__':
		for file in fileList:
		logging.debug(f' {file}')

		if len(fileList) == 0:
		logging.warning ("No files specified")
		exit(0)
		# if len(fileList) == 0:
		# logging.warning ("No files specified")
		# exit(0)

		# logging.info("Parsing ASN1 files")
		# parseResults = syntaxCheckASN(fileList)
		# if processResults(parseResults, "Parsing") > 0:
		# exit(-1)

		logging.info("Parsing ASN1 files")
		parseResults = parseASN(fileList)
		errorCount = sum([1 for r in parseResults.values() if not r['ok']])
		Path('testing/asn_parse_results.json').write_text(json.dumps({
		'errorCount' : errorCount,
		'results' : parseResults
		}))
		logging.info(f"{errorCount} parse errors encountered")

		if (errorCount > 0):
		exit()

		logging.info ("Getting compile targets")
		compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text())
		logging.info (f"{len(compileTargets)} compile targets found")

		compileResults = compileAllTargets(compileTargets)
		errorCount = sum([1 for r in compileResults.values() if not r['ok']])
		Path('testing/asn_compile_results.json').write_text(json.dumps({
		'errorCount' : errorCount,
		'results' : compileResults
		}))
		logging.info(f"{errorCount} compiler encountered")
		# logging.info ("Getting compile targets")
		# compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text())
		# logging.info (f"{len(compileTargets)} compile targets found")

		# compileResults = compileAllTargets(compileTargets)
		# if processResults(compileResults, "Compiling") > 0:
		# exit(-1)

		logging.info ("Linting files")
		ignoreLintingList = Path('testing/asn_ignore_lint.txt').read_text().splitlines()
		@@ -153,8 +169,9 @@ if __name__ == '__main__':
		for file in ignoredFiles:
		logging.debug(f' {file}')
		fileList = [file for file in fileList if file not in ignoredFiles]

		lintResults = lint_asn1.lintASN1Files(fileList)
		Path('testing/asn_lint_results.json').write_text(json.dumps(lintResults))
		lintExceptions = json.loads(Path('testing/asn_lint_exceptions.json').read_text())
		lintResults = lint_asn1.lintASN1Files(fileList, lintExceptions)
		if processResults(lintResults, "Linting") > 0:
		exit(-1)

		exit(0)

testing/lint_asn1.py

+14 −5

Original line number	Diff line number	Diff line
		@@ -167,9 +167,9 @@ def checkD45 (t, context):
		return errors


		def lintASN1File (asnFile):
		print (f"File: {asnFile}")
		def lintASN1File (asnFile, exceptions):
		errors = []
		suppressed = []
		context = {'file' : asnFile}
		try:
		logging.info ("Checking file {0}...".format(asnFile))
		@@ -191,10 +191,17 @@ def lintASN1File (asnFile):
		except ParseError as ex:
		appendFailure(errors, context, { "message" : "ParseError: {0}".format(ex)})
		logging.error("ParseError: {0}".format(ex))
		return errors
		if len(exceptions) > 0:
		suppressed = [error for error in errors if error['message'] in exceptions]
		errors = [error for error in errors if error['message'] not in exceptions]
		return {
		'ok' : len(errors) == 0,
		'message' : errors,
		'suppressed' : suppressed
		}


		def lintASN1Files (fileList):
		def lintASN1Files (fileList, exceptions):
		if len(fileList) == 0:
		logging.warning ("No files specified")
		return {}
		@@ -202,7 +209,9 @@ def lintASN1Files (fileList):
		errorMap = {}
		logging.info("Checking files...")
		for f in fileList:
		errorMap[str(f)] = lintASN1File(str(f))
		unixf = str(f).replace('\\', '/')
		errorMap[str(f)] = lintASN1File(str(f), exceptions[unixf] if unixf in exceptions else [])

		return errorMap