diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9ea9542e8c1fc6898caa583ca92c82f17dbbb54..76d7b937aeeda09a8b1c3933cf300b480d75f41a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,12 @@ variables: - 'default' - 'test-be-release' - 'test-long-self-test' + REF_COMMIT: + description: "Ref commit id or ref release tag. To be used by Manual pipelines: 'test-be-release'" + value: '' + CUT_COMMIT: + description: "Cut commit id or cut release tag. To be used by Manual pipeline:'test-be-release'" + value: '' default: @@ -46,7 +52,7 @@ workflow: IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' variables: - IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH' + IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $REF_COMMIT vs $CUT_COMMIT' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' @@ -330,7 +336,7 @@ build-codec-instrumented-linux: extends: - .build-job-linux - .rules-basis - timeout: "6 minutes" + timeout: "7 minutes" script: - *print-common-info - bash ci/build_codec_instrumented_linux.sh @@ -1168,35 +1174,185 @@ codec-comparison-on-main-push: # Manual jobs # --------------------------------------------------------------- -test-be-to-release: +.print-common-info-ref-vs-cut: &print-common-info-ref-vs-cut + - | + echo "Printing common information for build job." + echo "REF_COMMIT: $REF_COMMIT." + echo "CUT_COMMIT: $CUT_COMMIT." + echo "Current job is run on commit $CI_COMMIT_SHA" + echo "Commit time was $CI_COMMIT_TIMESTAMP" + date | xargs echo "System time is" + +.comparison-setup-codec-ref-vs-cut: + &comparison-setup-codec-ref-vs-cut + - git checkout $REF_COMMIT + - $winoutdata = $null + - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /m | tee -variable winoutdata + - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 + - mv IVAS_cod.exe IVAS_cod_ref.exe + - mv IVAS_dec.exe IVAS_dec_ref.exe + - mv IVAS_rend.exe IVAS_rend_ref.exe + - mkdir ref_hrtf_bin + - cp scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/* ref_hrtf_bin + + - git checkout $CUT_COMMIT + - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /m | tee -variable winoutdata + - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 + - mv IVAS_cod.exe IVAS_cod.exe + - mv IVAS_dec.exe IVAS_dec.exe + - mv IVAS_rend.exe IVAS_rend.exe + - mkdir cut_hrtf_bin + - cp scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/* cut_hrtf_bin + + # switch to $CI_COMMIT_SHORT_SHA where the scripts reside + - git checkout $CI_COMMIT_SHORT_SHA + +.rules-manual-pipeline: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE =~ 'default' + +.rules-test-be-release: + extends: .rules-manual-pipeline + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' && $REF_COMMIT && $CUT_COMMIT + +test-be-to-release-stereo: stage: test tags: - - ivas-windows + - test-be-to-release-windows resource_group: ivas-be-to-release-test-resource - timeout: "20 minutes" - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' + timeout: "22 hours" # to be reviewed + extends: + - .rules-test-be-release script: + - *comparison-setup-codec-ref-vs-cut + - mkdir logs - - echo "$CI_COMMIT_BRANCH" - - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata - - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 - - ("& python ci/check_for_warnings.py '$BUILD_OUTPUT'") | Invoke-Expression - - ("exit $LASTEXITCODE") | Invoke-Expression + - mkdir tests\codec_be_to_accepted_release\ref + - mkdir tests\codec_be_to_accepted_release\dut - # path to release candidate refs defined in config.toml + - echo "generate references" - - echo "Placeholder for BE test to release script" - - mkdir logs + # use ref hrtf binaries + - cp ref_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data + + # ref encoder stereo + - $zero_errors_test_enc_const_br=0 + - $zero_failures_test_enc_const_br=0 + + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_stereo.py --ref_encoder_path ./IVAS_cod_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_enc_const_br_stereo_ref.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_stereo_ref.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_ref.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in stereo enc ref generation"} + - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_ref.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in stereo enc ref generation"} + + # ref decoder stereo + - $zero_errors_test_dec_const_br=0 + - $zero_failures_test_dec_const_br=0 + + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_stereo.py --ref_decoder_path ./IVAS_dec_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_dec_const_br_stereo_stereo_ref.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in stereo_stereo dec ref generation"} + - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in stereo_stereo dec ref generation"} + + # use cut hrtf binaries + - cp cut_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data + + # cut encoder stereo + - echo "bitstream comparison" + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_stereo.py --dut_encoder_path ./IVAS_cod.exe --reduction_level reduced --html=logs/report-test_enc_const_br_stereo_cut.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_stereo_cut.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_cut.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in stereo"} + - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_cut.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in stereo ref generation"} + + # cut decoder stereo + - echo "decoder output comparison | cut decoder stereo" + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_stereo.py --dut_decoder_path ./IVAS_dec.exe --reduction_level reduced --html=logs/report-test_dec_const_br_stereo_stereo_cut.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in stereo_stereo dec cut generation"} + - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in stereo_stereo dec cut generation"} + + - IF($zero_errors_test_enc_const_br -ne 0 -or $zero_errors_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - IF($zero_failures_test_enc_const_br -ne 0 -or $zero_failures_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} artifacts: - name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results" + name: "test-be-to-release-stereo--sha-$CI_COMMIT_SHORT_SHA--results" when: always expire_in: 1 week paths: - logs/ - expose_as: "test-be-to-release results" + expose_as: "test-be-to-release stereo results" + reports: + junit: + - logs/report-junit-*.xml + +test-be-to-release-multichannel: + stage: test + tags: + - test-be-to-release-windows + resource_group: ivas-be-to-release-test-resource + timeout: "22 hours" # to be reviewed + extends: + - .rules-test-be-release + script: + - *comparison-setup-codec-ref-vs-cut + - mkdir logs + + - mkdir tests\codec_be_to_accepted_release\ref + - mkdir tests\codec_be_to_accepted_release\dut + + - echo "generate references" + + # use ref hrtf binaries + - cp ref_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data + + # ref encoder multichannel + - $zero_errors_test_enc_const_br=0 + - $zero_failures_test_enc_const_br=0 + + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_multichannel.py --ref_encoder_path ./IVAS_cod_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_enc_const_br_multichannel_ref.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_multichannel_ref.xml --testcase_timeout 360 ") | Invoke-Expression + - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_ref.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in multichannel enc ref generation"} + - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_ref.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in multichannel enc ref generation"} + + # ref decoder multichannel + - $zero_errors_test_dec_const_br=0 + - $zero_failures_test_dec_const_br=0 + + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_multichannel.py --ref_decoder_path ./IVAS_dec_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_dec_const_br_multichannel_multichannel_ref.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in multichannel_multichannel dec ref generation"} + - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in multichannel_multichannel dec ref generation"} + + # use cut hrtf binaries + - cp cut_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data + + # cut encoder multichannel + - echo "cut encoder multichannel" + - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_multichannel.py --dut_encoder_path ./IVAS_cod.exe --reduction_level reduced --html=logs/report-test_enc_const_br_multichannel_cut.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_multichannel_cut.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_cut.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in multichannel"} + - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_cut.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in multichannel ref generation"} + + # cut decoder multichannel + - echo "cut decoder multichannel" + - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_multichannel.py --dut_decoder_path ./IVAS_dec.exe --reduction_level reduced --html=logs/report-test_dec_const_br_multichannel_multichannel_cut.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml --testcase_timeout 360 ") | Invoke-Expression + + - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in multichannel_multichannel dec cut generation"} + - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in multichannel_multichannel dec cut generation"} + + - IF($zero_errors_test_enc_const_br -ne 0 -or $zero_errors_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + - IF($zero_failures_test_enc_const_br -ne 0 -or $zero_failures_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL} + + artifacts: + name: "test-be-to-release-multichannel--sha-$CI_COMMIT_SHORT_SHA--results" + when: always + expire_in: 1 week + paths: + - logs/ + expose_as: "test-be-to-release multichannel results" + reports: + junit: + - logs/report-junit-*.xml test-long-self-test: tags: diff --git a/scripts/tools/Linux/eid-xor b/scripts/tools/Linux/eid-xor new file mode 100755 index 0000000000000000000000000000000000000000..9d820fff7d84a75839e24c160dbec5ef43799461 Binary files /dev/null and b/scripts/tools/Linux/eid-xor differ diff --git a/scripts/tools/Win32/eid-xor.exe b/scripts/tools/Win32/eid-xor.exe new file mode 100755 index 0000000000000000000000000000000000000000..34bd7ab904e4da13e6306df6315f3150e0919a2d --- /dev/null +++ b/scripts/tools/Win32/eid-xor.exe @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1689409062362bad5ac812e27ba5a9c0211fd16f1c335b53f5081adafe1dc422 +size 66048 diff --git a/tests/codec_be_to_accepted_release/decoder/__init__.py b/tests/codec_be_to_accepted_release/decoder/__init__.py index 817d2d14bc55bc919a6824a3668fb52beb392c14..7590611fc8a86d28f8134dde2e2adf8d2ea111c4 100644 --- a/tests/codec_be_to_accepted_release/decoder/__init__.py +++ b/tests/codec_be_to_accepted_release/decoder/__init__.py @@ -30,6 +30,7 @@ the United Nations Convention on Contracts on the International Sales of Goods. import numpy as np import pytest +import platform from contextlib import contextmanager from typing import Union from pathlib import Path @@ -140,9 +141,25 @@ def get_bitstream_and_options( bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192") if processing == BS_PROC_FER_15: ep_path = TESTV_PATH.joinpath("ltv_ep_015.192") + + # get the pathname to the eid-xor tool respecting the running platform + if platform.system() == "Windows": + eid_xor_binary = [ + HERE.joinpath( + "../../scripts/tools/Win32/eid-xor.exe" + ) + ] + elif platform.system() in ["Linux", "Darwin"]: + eid_xor_binary = [ + HERE.joinpath( + "../../scripts/tools/Linux/eid-xor" + ) + ] + else: + assert False, f"eid-xor not available for {platform.system()}" + cmd = [ - # TODO: adapt for windows - "eid-xor", + str(eid_xor_binary), "-fer", "-vbr", str(bitstream), @@ -156,10 +173,26 @@ def get_bitstream_and_options( "../../scripts/dly_error_profiles/dly_error_profile_5.dat" ) tracefile_path = Path(tmp_dir).joinpath(bitstream_out.with_suffix(".trace")) - # TODO: adapt for windows + + # get the pathname to the networkSimulator_g192 tool respecting the running platform + if platform.system() == "Windows": + networkSimulator_g192_binary = [ + HERE.joinpath( + "../../scripts/tools/Win32/networkSimulator_g192.exe" + ) + ] + elif platform.system() in ["Linux", "Darwin"]: + networkSimulator_g192_binary = [ + HERE.joinpath( + "../../scripts/tools/Linux/networkSimulator_g192" + ) + ] + else: + assert False, f"networkSimulator_g192 not available for {platform.system()}" + # TODO: get number of frames per packet from error profile name cmd = [ - "networkSimulator_g192", + str(networkSimulator_g192_binary), str(dly_profile_path), str(bitstream), str(bitstream_out),