Commit aa9ca52d authored by norvell's avatar norvell
Browse files

Updated detailed report.

parent 9ad6ef61
Loading
Loading
Loading
Loading
+35 −22
Original line number Diff line number Diff line
@@ -40,6 +40,12 @@ if __name__ == "__main__":
    results_unsorted = {}
    count = {'PASS':0,'FAIL':0,'ERROR':0}

    # Formats, categories and MLD histogram limits
    formats = {'**Unidentified format**':r'.*','Stereo':r'stereo', 'ISM':r'ISM', 'Multichannel':r'Multi-channel', 
               'MASA':r'MASA','SBA':r'SBA', 'OSBA':r'OSBA', 'OMASA':r'OMASA','Renderer':r'renderer'}
    categories = {'Normal operation':r'.*', 'DTX':r'DTX', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching', 'JBM':r'JBM' }
    limits = [0,5,10,20,math.inf]

    for testcase in testcases:
        if testcase.find(".//skipped") is None:
            if testcase.get("file") is None:
@@ -64,43 +70,50 @@ if __name__ == "__main__":
                testresult = 'PASS'

            properties_values = [str(properties_found.get(p)) for p in PROPERTIES]

            # Identify format and category (mode of operation)
            fmt = [f for f in formats if re.search(formats[f], fulltestname, re.IGNORECASE)][-1]
            cat = [c for c in categories if re.search(categories[c], fulltestname, re.IGNORECASE)][-1]

            # For ERROR cases, both a FAIL and an ERROR result is generated. 
            # Here, a FAIL would be overwritten with an ERROR result since it has the same name.
            results_unsorted[fulltestname] = (testresult, properties_values) 
            # Todo: Change to fielded dict for all entries to eliminate confusing literal indices
            results_unsorted[fulltestname] = (fmt, cat, testresult, properties_values) 

    results_sorted = dict(sorted(results_unsorted.items()))

    with open(csv_file, "w") as outfile:
        headerline = ";".join(["testcase","Result"] + PROPERTIES) + "\n"
        headerline = ";".join(["testcase","Format","Category","Result"] + PROPERTIES) + "\n"
        outfile.write(headerline)
        for test in results_sorted:
            count[results_sorted[test][0]] += 1
            line = ";".join([test,results_sorted[test][0]] + results_sorted[test][1]) + "\n"
            count[results_sorted[test][2]] += 1
            line = ";".join([test] + list(results_sorted[test])[0:-1] + results_sorted[test][-1]) + "\n"
            outfile.write(line)

    categories = {'Normal operation':r'.*', 'JBM':r'JBM', 'PLC':r'%', 'Bitrate switching':r'br sw|bitrate switching'}
    limits = [0,5,10,20,math.inf]
    tmp = {}
    for fmt in formats:
        tmp[fmt] = {}
        for cat in categories:
            tmp[fmt][cat] = []

    if summary_file is not None:
        with open(summary_file, "w") as fp:
            for testcase in results_sorted:
                cat = [c for c in categories if re.search(categories[c],testcase)][-1]
                if cat in tmp:
                    tmp[cat].append(results_sorted[testcase][1][0]) # Add MLD score to list
                else:
                    tmp[cat] = [results_sorted[testcase][1][0]]
                (fmt, cat, _, _) = results_sorted[testcase]
                tmp[fmt][cat].append(results_sorted[testcase][-1][0]) # Add MLD score to list
                
            headerline = "Category;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n"
            for fmt in tmp:
                headerline = f"{fmt};\nCategory;0;" + ";".join([f"{str(a)} -- {str(b)}" for (a,b) in zip(limits[0:-1],limits[1:])]) + ";None\n"
                fp.write(headerline)
            for cat in tmp:
                for cat in categories:
                    # Separate 0 and None as special cases
                mld = [float(x) for x in tmp[cat] if x != 'None' and x != '0']
                zero = sum([1 for x in tmp[cat] if x == '0'])
                none = sum([1 for x in tmp[cat] if x == 'None'])                
                    mld = [float(x) for x in tmp[fmt][cat] if x != 'None' and x != '0']
                    zero = sum([1 for x in tmp[fmt][cat] if x == '0'])
                    none = sum([1 for x in tmp[fmt][cat] if x == 'None'])                
                    hist, _ = np.histogram(mld,limits)
                    line = f"{cat}; {str(zero)}; {'; '.join(map(str,hist))}; {str(none)}\n"
                    fp.write(line)
                fp.write("\n")

    print(
        f"Parsed testsuite with {count['PASS']+count['FAIL']+count['ERROR']} tests: {count['PASS']} passes, {count['FAIL']} failures and {count['ERROR']} errors."