From 26a125ff7f7dafb2e5d33705a5247d6c5fed2deb Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 20 Mar 2024 15:42:40 +0100 Subject: [PATCH 01/10] Add ivas-conformance-linux job --- .gitlab-ci.yml | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6648a5274..3bf9d3e404 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,7 @@ variables: - 'test-be-release' - 'test-long-self-test' - 'ivas-conformance' + - 'ivas-conformance-linux' GIT_CLEAN_FLAGS: -ffdxq TESTCASE_TIMEOUT_STV_SANITIZERS: 180 TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200 @@ -58,6 +59,10 @@ workflow: variables: IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'trigger' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + variables: + IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH' + stages: - .pre @@ -243,6 +248,8 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance' when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + when: never - when: on_success .rules-merge-request: @@ -1401,6 +1408,87 @@ ivas-conformance: reports: junit: report-junit.xml +ivas-conformance-linux: + tags: + - ivas-linux + stage: test + timeout: "60 minutes" + rules: + - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + allow_failure: + exit_codes: + - 123 + script: + - *print-common-info + # Prepare reference exec, use tests and scripts from reference + - $source_branch_commit_sha = $(git rev-parse HEAD) + - git checkout main # This should be set to a relevant reference + - python3 .\scripts\strip_split_rendering.py + + - make -j + - cp IVAS_cod IVAS_cod_ref + - cp IVAS_dec IVAS_dec_ref + - cp IVAS_rend IVAS_rend_ref + - git restore . + - git checkout $source_branch_commit_sha + + # Reference creation + - python3 tests/create_short_testvectors.py + - python3 scripts/prepare_combined_format_inputs.py + - python3 -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files + - python3 -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python3 -m pytest tests/renderer/test_renderer.py --create_ref --keep_files + + # Output creation + - python3 -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt + + # Copy input data and output ref data + - rm -rf testvec + - mkdir testvec + - mkdir testvec/binauralRenderer_interface + - mkdir testvec/testv + - mkdir testvec/testv/renderer + - mkdir testvec/bin + - cp -r scripts/testv/* testvec/testv + - cp -r scripts/ls_layouts testvec + - cp -r scripts/switchPaths testvec + - cp -r scripts/trajectories testvec + - cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface + - cp -r tests/ref testvec/testv/ref + - cp -r tests/dut/* testvec/testv/ref + - cp -r tests/renderer/cut testvec/testv/renderer/ref + - cp -r tests/conformance-test testvec/ + - cp Readme_IVAS_dec.txt testvec + - cp Readme_IVAS_enc.txt testvec + - cp Readme_IVAS_rend.txt testvec + - cp Readme_IVAS_JBM_dec.txt testvec + - cp IVAS_cod.exe testvec/bin + - cp IVAS_dec.exe testvec/bin + - cp IVAS_rend.exe testvec/bin + + # Test run generated scripts in testvec + - cd testvec + - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html + - mv report.html .. + - mv report-junit.xml .. + + artifacts: + name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + - Readme_IVAS_dec.txt + - Readme_IVAS_enc.txt + - Readme_IVAS_rend.txt + - Readme_IVAS_JBM_dec.txt + expose_as: "Draft IVAS conformance -- Linux" + reports: + junit: report-junit.xml + + test-long-self-test: tags: - ivas-linux-fast -- GitLab From 82a6e4ea2fe8c66d6cc06c10414baeae22eb865f Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 20 Mar 2024 15:46:14 +0100 Subject: [PATCH 02/10] Fix in ivas-conformance-linux job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3bf9d3e404..076daa0f47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1421,7 +1421,7 @@ ivas-conformance-linux: script: - *print-common-info # Prepare reference exec, use tests and scripts from reference - - $source_branch_commit_sha = $(git rev-parse HEAD) + - source_branch_commit_sha=$(git rev-parse HEAD) - git checkout main # This should be set to a relevant reference - python3 .\scripts\strip_split_rendering.py -- GitLab From 28bd62601d0809562c05dff056b23d7aeda1d433 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 20 Mar 2024 15:48:43 +0100 Subject: [PATCH 03/10] Fix in ivas-conformance-linux job --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 076daa0f47..cf61d317d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1423,7 +1423,7 @@ ivas-conformance-linux: # Prepare reference exec, use tests and scripts from reference - source_branch_commit_sha=$(git rev-parse HEAD) - git checkout main # This should be set to a relevant reference - - python3 .\scripts\strip_split_rendering.py + - python3 ./scripts/strip_split_rendering.py - make -j - cp IVAS_cod IVAS_cod_ref -- GitLab From 0fe6cda11867594b57a5acad4b9d690f5e52d50b Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 20 Mar 2024 15:53:16 +0100 Subject: [PATCH 04/10] Fix in ivas-conformance-linux job 3 --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf61d317d9..43c4bb8afd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1423,7 +1423,6 @@ ivas-conformance-linux: # Prepare reference exec, use tests and scripts from reference - source_branch_commit_sha=$(git rev-parse HEAD) - git checkout main # This should be set to a relevant reference - - python3 ./scripts/strip_split_rendering.py - make -j - cp IVAS_cod IVAS_cod_ref -- GitLab From c3a07c598b9e14a6a4385e2b37571320e07cbe15 Mon Sep 17 00:00:00 2001 From: norvell Date: Wed, 20 Mar 2024 15:18:34 +0000 Subject: [PATCH 05/10] Fix in ivas-conformance-linux --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43c4bb8afd..8445f3ab2b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1462,9 +1462,9 @@ ivas-conformance-linux: - cp Readme_IVAS_enc.txt testvec - cp Readme_IVAS_rend.txt testvec - cp Readme_IVAS_JBM_dec.txt testvec - - cp IVAS_cod.exe testvec/bin - - cp IVAS_dec.exe testvec/bin - - cp IVAS_rend.exe testvec/bin + - cp IVAS_cod testvec/bin + - cp IVAS_dec testvec/bin + - cp IVAS_rend testvec/bin # Test run generated scripts in testvec - cd testvec -- GitLab From 7d1a62bcc6562230dcebe8bf4d75bf2ec956bb13 Mon Sep 17 00:00:00 2001 From: norvell Date: Wed, 20 Mar 2024 15:46:00 +0000 Subject: [PATCH 06/10] Use Ericsson windows runner for ivas-conformance for testing --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8445f3ab2b..d997a68996 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1331,7 +1331,7 @@ test-be-to-release: ivas-conformance: tags: - - ivas-windows + - ericsson-windows-runner stage: test timeout: "60 minutes" rules: -- GitLab From fa708c77dbb978fc490a7b0ad77500e2d27f0ded Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Mar 2024 09:34:44 +0100 Subject: [PATCH 07/10] Shorten test case names to be able to run on Windows --- scripts/config/self_test.prm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/config/self_test.prm b/scripts/config/self_test.prm index a9bb024f30..07a35a1a28 100644 --- a/scripts/config/self_test.prm +++ b/scripts/config/self_test.prm @@ -539,7 +539,7 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -dtx -ism +4 testv/stvISM1.csv NULL testv/stvISM_with_no_diegetic_switch.csv testv/stvISM4.csv 256000 48 testv/stv4ISM48n.wav bit ../IVAS_dec BINAURAL 48 bit testv/stv+4ISM48n+non_diegetic_pan.wav_brate_256000-48_DTX_binaural.tst -// 4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, combined render config, directivity configuration with identifiers +// 4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id ../IVAS_cod -ism 4 testv/stvISM1.csv testv/stvISM2.csv testv/stvISM3.csv testv/stvISM4.csv 128000 48 testv/stv4ISM48n.wav bit ../IVAS_dec -render_config testv/rend_config_combined.cfg -dpid 3 0 2 1 BINAURAL_ROOM_REVERB 48 bit testv/stv4ISM48n+combined_render_config_brate_128000-48-binaural_room_reverb.wav @@ -878,7 +878,7 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -sba -1 128000 48 testv/stvFOA48c.wav bit ../IVAS_dec -hrtf ../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin -t testv/headrot_case00_3000_q.csv -render_config testv/rend_config_renderer.cfg BINAURAL_ROOM_REVERB 32 bit testv/stvFOA48c.pcm_planarSBA_12800032-32_BinauralRoomReverb_Config_renderer_Headrot_BinauralFile.tst -// SBA 3OA at 128 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB combined renderer configuration with selected acoustic environment +// SBA 3OA at 128 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB rendconf sel acoustic env ../IVAS_cod -sba 3 128000 48 testv/stv3OA48c.wav bit ../IVAS_dec -render_config testv/rend_config_combined.cfg -aeid 1 BINAURAL_ROOM_REVERB 48 bit testv/stv3OA48c.pcm_SBA_12800048-48_BinauralRoomReverb_Config_renderer_combined_AEID_1.tst -- GitLab From 2128537bb20b73c00dcd66854b8c6a59a5253e9f Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Mar 2024 13:25:19 +0100 Subject: [PATCH 08/10] Add try/catch to Windows conformance job to get report --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d997a68996..46e53459bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1355,12 +1355,12 @@ ivas-conformance: # Reference creation - python tests/create_short_testvectors.py - python scripts/prepare_combined_format_inputs.py - - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files - - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files - - python -m pytest tests/renderer/test_renderer.py --create_ref --keep_files + - try { python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files } catch { echo "Errors encountered!" } + - try { python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files } catch { echo "Errors encountered!" } + - try { python -m pytest tests/renderer/test_renderer.py --create_ref --keep_files } catch { echo "Errors encountered!" } # Output creation - - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html + - try { python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html } catch { echo "Errors encountered!" } - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data -- GitLab From f3a389a8442a9f677d26d42c1f8236fe1c7570b4 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 21 Mar 2024 13:51:33 +0100 Subject: [PATCH 09/10] Add git pull on reference to ensure it is updated --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46e53459bb..6c437cf3b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1344,6 +1344,7 @@ ivas-conformance: # Prepare reference exec, use tests and scripts from reference - $source_branch_commit_sha = $(git rev-parse HEAD) - git checkout main # This should be set to a relevant reference + - git pull # Ensure to get the latest version - python .\scripts\strip_split_rendering.py - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug - cp -force IVAS_cod.exe IVAS_cod_ref.exe @@ -1355,12 +1356,12 @@ ivas-conformance: # Reference creation - python tests/create_short_testvectors.py - python scripts/prepare_combined_format_inputs.py - - try { python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files } catch { echo "Errors encountered!" } - - try { python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files } catch { echo "Errors encountered!" } - - try { python -m pytest tests/renderer/test_renderer.py --create_ref --keep_files } catch { echo "Errors encountered!" } + - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref --keep_files + - python -m pytest tests/codec_be_on_mr_nonselection -v -n auto --update_ref 1 -m create_ref_part2 --keep_files + - python -m pytest tests/renderer/test_renderer.py --create_ref --keep_files # Output creation - - try { python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html } catch { echo "Errors encountered!" } + - python -m pytest tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt # Copy input data and output ref data -- GitLab From a070c68a30ce5c4bba940a92b53b7dd482dd7886 Mon Sep 17 00:00:00 2001 From: norvell Date: Thu, 21 Mar 2024 15:45:46 +0000 Subject: [PATCH 10/10] Enable all Windows runners for ivas-conformance --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c437cf3b3..918aa2631c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1331,7 +1331,7 @@ test-be-to-release: ivas-conformance: tags: - - ericsson-windows-runner + - ivas-windows stage: test timeout: "60 minutes" rules: -- GitLab