Commit d4038756 authored by norvell's avatar norvell
Browse files

Merge branch 'ci/ivas-conformance-stv' into 'main' 

[CI] Fixes to ivas-conformance job

See merge request !1543
parents 05c5f452 252c5d47

.gitlab-ci.yml

+51 −13
Original line number Diff line number Diff line
@@ -167,6 +167,12 @@ stages: 
  - git pull

  - cd -



.update-ltv-repo-win: &update-ltv-repo-win

  - Push-Location

  - cd $LTV_DIR_WIN

  - git pull

  - Pop-Location



.enable-split-rendering: &enable-split-rendering

# automatically enable #define SPLIT_REND_WITH_HEAD_ROT in options.h, handling both /**/-comment and //-comment

  - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)[[:space:]]*\*\//\1/g" ./lib_com/options.h
@@ -189,6 +195,11 @@ stages: 
    - cp "$LTV_DIR"/*.met scripts/testv/

    - cp "$LTV_DIR"/*.csv scripts/testv/



.copy-ltv-files-to-testv-dir-win: &copy-ltv-files-to-testv-dir-win

    - cp $LTV_DIR_WIN\*.wav scripts\testv

    - cp $LTV_DIR_WIN\*.met scripts\testv

    - cp $LTV_DIR_WIN\*.csv scripts\testv



.activate-Werror-linux: &activate-Werror-linux

  - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile

  - sed -i.bak "s/# \(set(CMAKE_C_FLAGS \"\${CMAKE_C_FLAGS} -Werror\")\)/\1/" CMakeLists.txt
@@ -1360,7 +1371,7 @@ ivas-conformance: 
  tags:

    - ivas-windows

  stage: test

  timeout: "60 minutes"

  timeout: "90 minutes"

  rules:

    - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance'

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -1378,9 +1389,8 @@ ivas-conformance: 
    # Reference creation

    - python tests/create_short_testvectors.py

    - python scripts/prepare_combined_format_inputs.py

    - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files

    - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

    - python -m pytest tests/renderer/test_renderer.py --create_ref --keep_files

    - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files

    - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files



    # Output creation

    - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
@@ -1405,6 +1415,11 @@ ivas-conformance: 
    - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref

    - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref

    - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref

    

    # Remove redundant files 

    - python scripts/cleanup_26252.py



    # Copy test script files

    - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/

    - cp Readme_IVAS_dec.txt testvec

    - cp Readme_IVAS_enc.txt testvec
@@ -1414,6 +1429,7 @@ ivas-conformance: 
    - cp IVAS_dec.exe testvec/bin

    - cp IVAS_rend.exe testvec/bin

    

    

    # Test run generated scripts in testvec

    - cd testvec    

    - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html
@@ -1439,7 +1455,7 @@ ivas-conformance-linux: 
  tags:

    - ivas-linux

  stage: test

  timeout: "60 minutes"

  timeout: "90 minutes"

  rules:

    - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

  allow_failure:
@@ -1455,12 +1471,12 @@ ivas-conformance-linux: 
    # Reference creation

    - python3 tests/create_short_testvectors.py

    - python3 scripts/prepare_combined_format_inputs.py

    - python3 -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files

    - python3 -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

    - python3 -m pytest tests/renderer/test_renderer.py --create_ref --keep_files

    - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py"

    - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files

    - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files



    # Output creation

    - python3 -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html

    - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html

    - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt

    

    # Copy input data and output ref data
@@ -1478,32 +1494,54 @@ ivas-conformance-linux: 
    - cp -r tests/ref testvec/testv/ref

    - cp -r tests/dut/* testvec/testv/ref

    - cp -r tests/renderer/cut testvec/testv/renderer/ref

    

    # Remove redundant files 

    - python3 scripts/cleanup_26252.py

    

    # Copy test script files

    - cp -r tests/conformance-test testvec/

    - cp Readme_IVAS_dec.txt testvec

    - cp Readme_IVAS_enc.txt testvec 

    - cp Readme_IVAS_rend.txt testvec

    - cp Readme_IVAS_JBM_dec.txt testvec

    

    # Create GCOV execs for coverage analysis

    - make clean

    - make GCOV=1 -j

    

    - cp IVAS_cod testvec/bin

    - cp IVAS_dec testvec/bin

    - cp IVAS_rend testvec/bin

        

    # Test run generated scripts in testvec

    - cd testvec   

    - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html

    - exit_code=0    

    - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$?

    - mv report.html ..

    - mv report-junit.xml ..



    # Collect coverage

    - cd -

    - lcov -c -d obj -o coverage.info

    - lcov -r coverage.info "*apps*" -o coverage.info

    - lcov -r coverage.info "*lib_util*" -o coverage.info

    - commit_sha=$(git rev-parse HEAD)

    - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha"



  artifacts:

    name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA"

    expire_in: 1 week

    when: always

    paths:

      - report_cmd.html

      - report-junit.xml

      - report.html

      - Readme_IVAS_dec.txt  

      - Readme_IVAS_enc.txt  

      - Readme_IVAS_rend.txt

      - Readme_IVAS_JBM_dec.txt

      - coverage.info

      - coverage

    expose_as: "Draft IVAS conformance -- Linux"

    reports:

      junit: report-junit.xml

scripts/cleanup_26252.py

0 → 100644
+46 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



from pathlib import Path

import os

import glob



files = ['Readme_IVAS_JBM_dec.txt','Readme_IVAS_dec.txt','Readme_IVAS_enc.txt','Readme_IVAS_rend.txt'];



used_files = []

kept = 0

removed = 0



# Input files explicitly listed in scripts

for file in files:

    with open(file,"r") as fp:

        for line in fp:

            for item in line.split():

                if "$TESTV_PATH" in item or "$REF_PATH" in item:

                    used_files.append(Path(item).name)



# All .dat files in testv (binary render config files listed via txt config file)

for file in glob.glob("testvec/testv/*.dat"):

    used_files.append(Path(file).name)    

                    

# All .met, .csv and .wav files in .txt configuration files

for file in glob.glob("testvec/testv/*.txt"):

    with open(file,"r") as fp:

        for line in fp:        

            if any(x in line for x in [".met",".csv",".wav"]):

                used_files.append(Path(line).name.strip())                    

                    

# Remove duplicates

used_files = list(dict.fromkeys(used_files))



for dirpath, subdirs, files in os.walk("testvec"):

    for file in files:

        if file in used_files:

            kept = kept + 1

        else:

            removed = removed + 1

            os.remove(os.path.join(dirpath, file))

print(f"Identified {len(used_files)} files from scripts")            

print(f"Removed {removed} files")

print(f"Kept {kept} files")

scripts/parse_commands.py

+30 −0
Original line number Diff line number Diff line
@@ -38,6 +38,36 @@ if __name__ == '__main__': 
                    cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line))

                    cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line))



    # If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens.

    if all(not x for x in [cmds_enc, cmds_dec, cmds_rend]):

        for html_report in input:

            with open(html_report,'r') as infile:

                enc_cmd = False

                dec_cmd = False

                rend_cmd = False

                for line in infile.readlines():

                    if enc_cmd:

                        cmds_enc.append(line)

                        enc_cmd = False

                    elif dec_cmd:

                        cmds_dec.append(line)

                        dec_cmd = False

                    elif rend_cmd:

                        cmds_rend.append(line)

                        rend_cmd = False

                    else:

                        if "DUT encoder command" in line:

                            enc_cmd = True

                        elif "DUT decoder command" in line:

                            dec_cmd = True

                        elif "Running command" in line:

                            rend_cmd = True



    # Sort lists to keep deterministic order between runs

    cmds_enc.sort()

    cmds_dec.sort()

    cmds_rend.sort()



    with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile:

        with open('scripts/enc_header.txt','r') as header:

            outfile.write(header.read())