Merge branch 'testing_fixes' into 'main'

ASN.1 Compilation and updated linting

See merge request !48

.gitlab-ci.yml

@@ -4,29 +4,34 @@ before_script:
   - python3 --version
 
 stages:
+  - Merge
   - Syntax
-  - Lint
-  - Compile
+  - CompileAndLint
 
-parseASN1:
-  stage: Syntax
+MergeTest:
+  stage: Merge
   script:
-  - python3 testing/parse_asn1.py
+  - python3 testing/merge_test.py
 
 checkXSD:
   stage: Syntax
   script:
   - python3 testing/check_xsd.py
 
-lintASN1:
-  stage: Lint
+parseASN1:
+  stage: Syntax
   script:
-  - python3 testing/lint_asn1.py
-  allow_failure: true
+  - python3 testing/parse_asn1.py
 
 compileASN1:
-  stage: Compile
+  stage: CompileAndLint
   script:
   - python3 testing/compile_asn1.py
+
+lintASN1:
+  stage: CompileAndLint
+  script:
+  - python3 testing/lint_asn1.py
   allow_failure: true
 
+

testing/check_asn1.py

-import logging
-
-from asn1tools import parse_files, compile_dict, ParseError, CompileError
-from glob import glob
-from pathlib import Path
-
-from pprint import pprint
-
-
-def parseASN1File (asnFile):
-    try:
-        parse_files(asnFile)
-    except ParseError as ex:
-        return [ex]
-    return []
-
-
-def parseASN1Files (fileList):
-    if len(fileList) == 0:
-        logging.warning ("No files specified")
-        return {}
-    errors = {}
-    logging.info("Parsing files...")
-    for f in fileList:
-        ex = parseASN1File(f)
-        if ex: 
-            logging.info (f"  {f}: Failed - {ex!r}")
-        else: 
-            logging.info (f"  {f}: OK")            
-        errors[f] = ex
-    return errors
-
-
-def compileASN1Files (fileList):
-    logging.info("Compiling files...")
-    errors = []
-    try:
-        d = parse_files(fileList)
-        for modulename, module in d.items():
-            # Weird fix because the compiler doesn't like RELATIVE-OID as a type
-            # Not sure if the on-the-wire encoding would be affected or not
-            # but for most checking purposes this doesn't matter
-            module['types']["RELATIVE-OID"] = {'type' : 'OBJECT IDENTIFIER'}
-        c = compile_dict(d)
-    except CompileError as ex:
-        logging.info (f"Compiler  error: {ex}")
-        errors.append(ex)
-    except ParseError as ex:
-        logging.info (f"Parse error: {ex}")
-        errors.append(ex)
-    logging.info ("Compiled OK")
-    return errors
-
-
-def validateASN1Files (fileList):
-    parseErrors = parseASN1Files(fileList)
-#    if len(parseErrors > 0):
-#        logging.info ("Abandonding compile due to parse errors")
-    compileErrors = compileASN1Files(fileList)
-    return parseErrors, compileErrors
-
-
-def validateAllASN1FilesInPath (path):
-    globPattern = str(Path(path)) + '/*.asn1'
-    logging.info("Searching: " + globPattern)
-    schemaGlob = glob(globPattern, recursive=True)
-    return validateASN1Files(schemaGlob)    
-
-
-if __name__ == '__main__':
-    parseErrors, compileErrors = validateAllASN1FilesInPath("./")
-    parseErrorCount = 0
-    print ("ASN.1 Parser checks:")
-    print ("-----------------------------")
-    for filename, errors in parseErrors.items():
-        if len(errors) > 0:
-            parseErrorCount += len(errors)
-            print (f"{filename}: {len(errors)} errors")
-            for error in errors:
-                print ("  " + str(error))
-        else:
-            print (f"{filename}: OK")
-    print ("-----------------------------")
-    print ("ASN.1 Compilation:")
-    print ("-----------------------------")
-    if len(compileErrors) > 0:
-        for error in compileErrors:            
-            print ("  " + str(error))
-    else:
-        print ("Compilation OK")
-    print ("-----------------------------")
-    print (f"{parseErrorCount} parse errors, {len(compileErrors)} compile errors")
-    exit (parseErrorCount + len(compileErrors))
-

testing/compile_asn1.py

-print ("Not implemented yet")
\ No newline at end of file
+import logging
+
+import asn1tools
+from pathlib import Path
+
+from pprint import pprint
+
+ignoreReleases = {'33108' : [f'r{i}' for i in range(5, 17)],
+                  '33128' : [] }
+
+def prepareFile(f):
+    with open(f) as fh:
+        s = fh.read()
+    s = s.replace("RELATIVE-OID", "OBJECT IDENTIFIER") # sigh
+    return s
+
+if __name__ == '__main__':
+    fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn"))
+
+    ignoredFiles = [file for file in fileList if file.parts[1] in ignoreReleases[file.parts[0]]]
+    logging.info(f"Ignoring {len(ignoredFiles)} files")
+    logging.debug(ignoredFiles)
+    
+    fileList = [file for file in fileList if file not in ignoredFiles]
+
+    if len(fileList) == 0:
+        logging.warning ("No files specified")
+        exit(0)
+
+    print ("ASN.1 Compilation checks:")
+    print ("-----------------------------")    
+    logging.info("Parsing files...")
+    errorCount = 0
+    for f in fileList:
+        try:
+            s = prepareFile(str(f))
+            asn1tools.compile_string(s) # this won't work for modules with IMPORTs
+        except asn1tools.ParseError as ex:
+            logging.info (f"  {f}: Failed - {ex!r}")
+            print (f"  {f}: Failed - {ex!r}")
+            errorCount += 1
+            continue
+        print (f"  {f}: OK")
+    print ("-----------------------------")    
+    print (f"Compile errors: {errorCount}")
+    print ("-----------------------------")    
+    exit(errorCount)

testing/dockerfile

-FROM python:3.7
-RUN pip3 install -q asn1tools lxml xmlschema
\ No newline at end of file
+# docker build -t mcanterb/forge-cicd
+# docker push mcanterb/forge-cicd
+
+FROM python:3.8
+RUN apt update && apt-get install -y git
+RUN git config --global user.name "forgeRobot" && git config --global user.email "forgeRobot@example.com"
+RUN pip3 install -q asn1tools lxml xmlschema requests gitpython
\ No newline at end of file

testing/lintingexceptions.py

-exceptedStrings = ["D.4.4: Enumerations for UDMServingSystemMethod start at 0, not 1",
-"D.4.5: Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE"]
\ No newline at end of file
+exceptedStrings = ["D.4.4: Enumerations for UDMServingSystemMethod start at 0, not 1", 
+"D.4.5: Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE", 
+"D.4.4: Enumerations for EstablishmentStatus start at 0, not 1", 
+"D.4.4: Enumerations for MMSDirection start at 0, not 1", 
+"D.4.4: Enumerations for MMSReplyCharging start at 0, not 1", 
+"D.4.4: Enumerations for MMStatusExtension start at 0, not 1"]
+

testing/merge_test.py

+import os
+import pprint
+import requests
+import json
+import subprocess
+
+crCommitBranch = os.environ.get("CI_COMMIT_REF_NAME", "NOTFOUND")
+apiUrl = os.environ.get("CI_API_V4_URL", "https://forge.3gpp.org/rep/api/v4")
+projectId = os.environ.get("CI_PROJECT_ID", "13")
+
+def gapi (query):
+    url = f"{apiUrl}/projects/{projectId}/{query}"
+    r = requests.get(url)
+    return json.loads(r.text)
+
+def do (commandline):
+    #print ("   Attempting: " + commandline)
+    completedProc = subprocess.run(commandline, capture_output=True, shell=True)
+    #print ("   STDOUT > " + ("empty" if completedProc.stdout is None else completedProc.stdout.decode('utf-8')))
+    #print ("   STDERR > " + ("empty" if completedProc.stderr is None else completedProc.stderr.decode('utf-8')))
+    #print (f"   Completed with code {completedProc.returncode}")
+    return (completedProc.returncode == 0,  completedProc.stdout.decode('utf-8'))
+
+print ("Searching for corresponding MR...")
+
+mrs = gapi(f"merge_requests?source_branch={crCommitBranch}")
+if len(mrs) == 0:
+    print ("No MR found... aborting")
+    exit()
+
+if len(mrs) > 1:
+    print (f"{len(mrs)} MRs found, 1 expected - aborting")
+    exit(-1)
+
+mr = mrs[0]
+
+print (f"Found MR {mr['reference']} ({mr['title']})")
+print (f"Target branch is {mr['target_branch']}")
+print ("Searching for open MRs targeting same branch...")
+
+mrs = gapi(f"merge_requests?target_branch={mr['target_branch']}&state=opened")
+mrs = [m for m in mrs if m['reference'] != mr['reference']]
+print (f"{len(mrs)} MRs found")
+
+mergeConflicts = {}
+
+for mr in mrs:
+    source_branch = mr['source_branch']
+    print (source_branch)
+
+    try:
+        do(f"git fetch origin {source_branch}:{source_branch}")
+        success, errStr = do(f"git merge --no-commit {source_branch}")
+        if not success:
+            print ("Merge NOT OK")
+            mergeConflicts[source_branch] = errStr
+        else:
+            print ("Merge OK")
+    except Exception as ex:
+        mergeConflicts[source_branch] = str(ex)
+        raise
+    finally:
+        do("git merge --abort")
+
+print (f"Merge conflicts with following branches: {mergeConflicts}")
+exit(len(mergeConflicts.keys()))
\ No newline at end of file