Commit 08cd8642 authored by Vladimir Malenovsky's avatar Vladimir Malenovsky
Browse files

Merge branch 'add-debugging-info-to-26-252-be-conformance-scripts' into 'main' 

Resolve "Add debugging info to 26.252 BE conformance scripts"

See merge request !2535
parents 80ecaad9 d8478041

scripts/ivas_conformance/README_Package_Gen.md

+17 −21
Original line number Diff line number Diff line
@@ -53,10 +53,7 @@ This folder contains scripts for running IVAS conformance tests. 


## Reference Conformance Package Generation



<details>

<summary> <i> Expand for detailed procedure </i> </summary>



To generate reference conformance package for distribution

  To generate the reference conformance package follow these steps in specified order.



### Generate Reference Outputs and Readme.txt files
@@ -64,33 +61,32 @@ To generate reference conformance package for distribution 
  sh scripts/ivas_conformance/ivas_be_conf_test_gen.sh

  ```



<details>

<summary> Example output of reference test generation </summary>

<pre><code>

::::::::::::::::::::::::

------------------------------------------

Generated html report: file:///home/dolby/git/ivas-codec/report_cmd.htm

------------------------------------------

=================================================

2571 passed, 538 skipped, 230 xfailed in 377.10s (0:06:17)

=================================================

Identified 5430 files from scripts

Removed 1515 files

Kept 5422 files

</code></pre>

</details>



### Generate Reference Decoded Outputs for the Reference Encoded files



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin  --regenerate-enc-refs

  ```



### Generate MLD values on Reference/Target platform



  First, copy the CUT binaries to the `CUT_BIN_DIR` folder and execute the IVAS CUT commands on the reference/target platform:



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --cut_build_path=CUT_BIN_DIR

  ```



  Then perform the MLD-based non-BE analysis on the CUT outputs:



  ```shell

  PYTHONPATH=scripts python scripts/ivas_conformance/runConformance.py --testvecDir $PWD/testvec --ref_build_path=testvec/bin --regenerate-mld-ref

  ```



  This process creates a new directory at `$PWD/testvec/testv/mld_ref2` containing the updated MLD values. 



### Generate a conformance package zip



  ```shell

  zip -r conformance.zip testvec scripts/ivas_conformance/runConformance.py  scripts/ivas_conformance/requirements.txt scripts/ivas_conformance/README.md scripts/ivas_conformance/thirdPartyLegalNotices.txt scripts/tools/Darwin/thirdPartyLegalnotices  scripts/tools/Darwin/LICENSE_wav-diff.txt scripts/tools/Darwin/wav-diff scripts/tools/Linux/thirdPartyLegalnotices  scripts/tools/Linux/LICENSE_wav-diff.txt scripts/tools/Linux/wav-diff scripts/tools/Win32/thirdPartyLegalnotices  scripts/tools/Win32/LICENSE_wav-diff.txt scripts/tools/Win32/wav-diff.exe

  ```



</details>

scripts/ivas_conformance/runConformance.py

+544 −101

File changed.

Preview size limit exceeded, changes collapsed.

scripts/parse_commands.py

+22 −8
Original line number Diff line number Diff line
@@ -35,11 +35,27 @@ if __name__ == '__main__': 
    for html_report in input:

        

        with open(html_report,'r') as infile:

            for line in infile.readlines():

            lines = infile.readlines()

            for idx, line in enumerate(lines):

                    # parse encoder and decoder commands

                    cmds_enc.extend(re.findall(r"REF encoder command:\\n\\t(.*?)\\n", line))

                    cmds_dec.extend(re.findall(r"REF decoder command:\\n\\t(.*?)\\n", line))



                    # parse renderer commands, including ISAR EXT REND commands (the next line contains IVAS_rend or IVAS_rend_ref)

                    cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line))

                    next_line = lines[idx + 1] if idx + 1 < len(lines) else ""

                    if (

                        "Running ISAR EXT REND command" in line

                        and ("IVAS_rend" in next_line or "IVAS_rend_ref" in next_line)

                    ):

                        cmds_rend.extend(re.findall(r"Running ISAR EXT REND command\\n(.*?)\\n", line))



                    # parse ISAR post renderer commands, including ISAR EXT REND commands (the next line contains ISAR_post_rend or ISAR_post_rend_ref)

                    cmds_isar_post_rend.extend(re.findall(r"Running ISAR post renderer command\\n(.*?)\\n", line))

                    if (

                        "Running ISAR EXT REND command" in line

                        and ("ISAR_post_rend" in next_line or "ISAR_post_rend_ref" in next_line)

                    ):

                        cmds_isar_post_rend.extend(re.findall(r"Running ISAR EXT REND command\\n(.*?)\\n", line))



    # If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens.
@@ -165,15 +181,13 @@ if __name__ == '__main__': 
                for output in glob.glob(absolute_out.group(1) + '*'):

                    output = path.relpath(output).replace('\\','/')

                    output = re.sub('tests', CUT_PATH, output)

                    diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1')

                    diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH, REF_PATH)+' '+output+' >> $LOG_FILE 2>&1\n')

                outfile.write(('; ').join(diff_cmds))

            if isar_out and "ref" in isar_out.group(1):

                outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1')

                outfile.write('$DIFF_BIN '+isar_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_out.group(1)+' >> $LOG_FILE 2>&1\n')

                if isar_md_out and "ref" in isar_md_out.group(1):

                    outfile.write('; $DIFF_BIN '+isar_md_out.group(1).replace(CUT_PATH, REF_PATH)+' '+isar_md_out.group(1)+' >> $LOG_FILE 2>&1\n')

                else:

            outfile.write('\n')

            outfile.write('\n\n')

        with open('scripts/script_footer.txt','r') as footer:

            outfile_dec.write(footer.read())    

            footer.seek(0)
@@ -208,7 +222,7 @@ if __name__ == '__main__': 
                for output in glob.glob(absolute_out.group(1) + '*'):

                    output = path.relpath(output).replace('\\','/')

                    output = re.sub('tests', CUT_PATH, output)

                    diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/renderer_short/ref',REF_PATH + r'/renderer_short/ref')+' '+ output +' >> $LOG_FILE 2>&1')

                    diff_cmds.append('$DIFF_BIN '+output.replace(CUT_PATH + r'/renderer_short/ref',REF_PATH + r'/renderer_short/ref')+' '+ output +' >> $LOG_FILE 2>&1\n')

                outfile.write(('; ').join(diff_cmds))

                outfile.write('\n')            

        with open('scripts/script_footer.txt','r') as footer: