diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7bffc14cb5f748d0fb32420cc232752f17ecbe96..2dd073e563ece23395a8f6bcff54083ec0dc8824 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,6 +167,12 @@ stages: - git pull - cd - +.update-ltv-repo-win: &update-ltv-repo-win + - Push-Location + - cd $LTV_DIR_WIN + - git pull + - Pop-Location + .enable-split-rendering: &enable-split-rendering # automatically enable #define SPLIT_REND_WITH_HEAD_ROT in options.h, handling both /**/-comment and //-comment - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)[[:space:]]*\*\//\1/g" ./lib_com/options.h @@ -189,6 +195,11 @@ stages: - cp "$LTV_DIR"/*.met scripts/testv/ - cp "$LTV_DIR"/*.csv scripts/testv/ +.copy-ltv-files-to-testv-dir-win: ©-ltv-files-to-testv-dir-win + - cp $LTV_DIR_WIN\*.wav scripts\testv + - cp $LTV_DIR_WIN\*.met scripts\testv + - cp $LTV_DIR_WIN\*.csv scripts\testv + .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile - sed -i.bak "s/# \(set(CMAKE_C_FLAGS \"\${CMAKE_C_FLAGS} -Werror\")\)/\1/" CMakeLists.txt @@ -1360,7 +1371,7 @@ ivas-conformance: tags: - ivas-windows stage: test - timeout: "60 minutes" + timeout: "90 minutes" rules: - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -1368,7 +1379,7 @@ ivas-conformance: exit_codes: - 123 script: - - *print-common-info-windows + - *print-common-info-windows - python .\scripts\strip_split_rendering.py - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - cp -force IVAS_cod.exe IVAS_cod_ref.exe @@ -1378,9 +1389,8 @@ ivas-conformance: # Reference creation - python tests/create_short_testvectors.py - python scripts/prepare_combined_format_inputs.py - - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files - - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files - - python -m pytest tests/renderer/test_renderer.py --create_ref --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html @@ -1405,6 +1415,11 @@ ivas-conformance: - cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/dut/* testvec/testv/ref - cp -r -force -ErrorAction Ignore tests/renderer/cut testvec/testv/renderer/ref + + # Remove redundant files + - python scripts/cleanup_26252.py + + # Copy test script files - cp -r -force -ErrorAction Ignore tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - cp Readme_IVAS_enc.txt testvec @@ -1414,6 +1429,7 @@ ivas-conformance: - cp IVAS_dec.exe testvec/bin - cp IVAS_rend.exe testvec/bin + # Test run generated scripts in testvec - cd testvec - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html @@ -1439,7 +1455,7 @@ ivas-conformance-linux: tags: - ivas-linux stage: test - timeout: "60 minutes" + timeout: "90 minutes" rules: - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' allow_failure: @@ -1455,12 +1471,12 @@ ivas-conformance-linux: # Reference creation - python3 tests/create_short_testvectors.py - python3 scripts/prepare_combined_format_inputs.py - - python3 -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files - - python3 -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files - - python3 -m pytest tests/renderer/test_renderer.py --create_ref --keep_files + - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py" + - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files + - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files # Output creation - - python3 -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data @@ -1478,32 +1494,54 @@ ivas-conformance-linux: - cp -r tests/ref testvec/testv/ref - cp -r tests/dut/* testvec/testv/ref - cp -r tests/renderer/cut testvec/testv/renderer/ref + + # Remove redundant files + - python3 scripts/cleanup_26252.py + + # Copy test script files - cp -r tests/conformance-test testvec/ - cp Readme_IVAS_dec.txt testvec - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec + + # Create GCOV execs for coverage analysis + - make clean + - make GCOV=1 -j + - cp IVAS_cod testvec/bin - cp IVAS_dec testvec/bin - cp IVAS_rend testvec/bin - + # Test run generated scripts in testvec - - cd testvec - - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html + - cd testvec + - exit_code=0 + - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$? - mv report.html .. - mv report-junit.xml .. + # Collect coverage + - cd - + - lcov -c -d obj -o coverage.info + - lcov -r coverage.info "*apps*" -o coverage.info + - lcov -r coverage.info "*lib_util*" -o coverage.info + - commit_sha=$(git rev-parse HEAD) + - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha" + artifacts: name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" expire_in: 1 week when: always paths: + - report_cmd.html - report-junit.xml - report.html - Readme_IVAS_dec.txt - Readme_IVAS_enc.txt - Readme_IVAS_rend.txt - Readme_IVAS_JBM_dec.txt + - coverage.info + - coverage expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml diff --git a/scripts/cleanup_26252.py b/scripts/cleanup_26252.py new file mode 100644 index 0000000000000000000000000000000000000000..df6c88ae988a54a845623c15cf1bb2be0f03329f --- /dev/null +++ b/scripts/cleanup_26252.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import os +import glob + +files = ['Readme_IVAS_JBM_dec.txt','Readme_IVAS_dec.txt','Readme_IVAS_enc.txt','Readme_IVAS_rend.txt']; + +used_files = [] +kept = 0 +removed = 0 + +# Input files explicitly listed in scripts +for file in files: + with open(file,"r") as fp: + for line in fp: + for item in line.split(): + if "$TESTV_PATH" in item or "$REF_PATH" in item: + used_files.append(Path(item).name) + +# All .dat files in testv (binary render config files listed via txt config file) +for file in glob.glob("testvec/testv/*.dat"): + used_files.append(Path(file).name) + +# All .met, .csv and .wav files in .txt configuration files +for file in glob.glob("testvec/testv/*.txt"): + with open(file,"r") as fp: + for line in fp: + if any(x in line for x in [".met",".csv",".wav"]): + used_files.append(Path(line).name.strip()) + +# Remove duplicates +used_files = list(dict.fromkeys(used_files)) + +for dirpath, subdirs, files in os.walk("testvec"): + for file in files: + if file in used_files: + kept = kept + 1 + else: + removed = removed + 1 + os.remove(os.path.join(dirpath, file)) +print(f"Identified {len(used_files)} files from scripts") +print(f"Removed {removed} files") +print(f"Kept {kept} files") + + diff --git a/scripts/parse_commands.py b/scripts/parse_commands.py index fe1a1840ade2291536a596d1fd137d9c610445a2..6d10698cfc649cb78e9d42b9bba17449438cb69c 100644 --- a/scripts/parse_commands.py +++ b/scripts/parse_commands.py @@ -38,6 +38,36 @@ if __name__ == '__main__': cmds_dec.extend(re.findall(r"DUT decoder command:\\n\\t(.*?)\\n", line)) cmds_rend.extend(re.findall(r"Running command\\n(.*?)\\n", line)) + # If pytest-html < v4 is used, the parsing will fail and render empty lists. This is a work-around in case that happens. + if all(not x for x in [cmds_enc, cmds_dec, cmds_rend]): + for html_report in input: + with open(html_report,'r') as infile: + enc_cmd = False + dec_cmd = False + rend_cmd = False + for line in infile.readlines(): + if enc_cmd: + cmds_enc.append(line) + enc_cmd = False + elif dec_cmd: + cmds_dec.append(line) + dec_cmd = False + elif rend_cmd: + cmds_rend.append(line) + rend_cmd = False + else: + if "DUT encoder command" in line: + enc_cmd = True + elif "DUT decoder command" in line: + dec_cmd = True + elif "Running command" in line: + rend_cmd = True + + # Sort lists to keep deterministic order between runs + cmds_enc.sort() + cmds_dec.sort() + cmds_rend.sort() + with open(txt_file.replace('.','_enc.'),'w', newline='\n') as outfile: with open('scripts/enc_header.txt','r') as header: outfile.write(header.read())