Commit 70a5a993 authored by kinuthia's avatar kinuthia
Browse files

add manual pipeline 

- add stereo and multichannel test
parent 1edab21d

.gitlab-ci.yml

+170 −14
Original line number Diff line number Diff line
@@ -20,6 +20,12 @@ variables: 
      - 'default'

      - 'test-be-release'

      - 'test-long-self-test'

  REF_COMMIT:

    description: "Ref commit id or ref release tag. To be used by Manual pipelines: 'test-be-release'"

    value: ''

  CUT_COMMIT:

    description: "Cut commit id or cut release tag. To be used by Manual pipeline:'test-be-release'"

    value: ''





default:
@@ -1168,35 +1174,185 @@ codec-comparison-on-main-push: 
# Manual jobs

# ---------------------------------------------------------------



test-be-to-release:

.print-common-info-ref-vs-cut: &print-common-info-ref-vs-cut

  - |

    echo "Printing common information for build job."

    echo "REF_COMMIT: $REF_COMMIT."

    echo "CUT_COMMIT: $CUT_COMMIT."

    echo "Current job is run on commit $CI_COMMIT_SHA"

    echo "Commit time was $CI_COMMIT_TIMESTAMP"

    date | xargs echo "System time is"



.comparison-setup-codec-ref-vs-cut:

  &comparison-setup-codec-ref-vs-cut

    - git checkout $REF_COMMIT

    - $winoutdata = $null

    - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /m | tee -variable winoutdata

    - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8

    - mv IVAS_cod.exe IVAS_cod_ref.exe

    - mv IVAS_dec.exe IVAS_dec_ref.exe

    - mv IVAS_rend.exe IVAS_rend_ref.exe

    - mkdir ref_hrtf_bin

    - cp scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/* ref_hrtf_bin



    - git checkout $CUT_COMMIT

    - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /m | tee -variable winoutdata

    - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8

    - mv IVAS_cod.exe IVAS_cod.exe

    - mv IVAS_dec.exe IVAS_dec.exe

    - mv IVAS_rend.exe IVAS_rend.exe

    - mkdir cut_hrtf_bin

    - cp scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/* cut_hrtf_bin



    # switch to $CI_COMMIT_SHORT_SHA where the scripts reside

    - git checkout $CI_COMMIT_SHORT_SHA



.rules-manual-pipeline:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE =~ 'default'



.rules-test-be-release:

  extends: .rules-manual-pipeline

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' && $REF_COMMIT && $CUT_COMMIT



test-be-to-release-stereo:

  stage: test

  tags:

    - ivas-windows

  resource_group: ivas-be-to-release-test-resource

  timeout: "20 minutes"

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'

  timeout: "22 hours"  # to be reviewed

  extends:

    - .rules-test-be-release

  script:

    - *comparison-setup-codec-ref-vs-cut

    - mkdir logs



    - echo "$CI_COMMIT_BRANCH"

    - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata

    - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8

    - ("& python ci/check_for_warnings.py '$BUILD_OUTPUT'") | Invoke-Expression

    - ("exit $LASTEXITCODE") | Invoke-Expression

    - mkdir tests\codec_be_to_accepted_release\ref

    - mkdir tests\codec_be_to_accepted_release\dut



    # path to release candidate refs defined in config.toml

    - echo "generate references"



    - echo "Placeholder for BE test to release script"

    - mkdir logs

    # use ref hrtf binaries

    - cp ref_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data



    # ref encoder stereo

    - $zero_errors_test_enc_const_br=0

    - $zero_failures_test_enc_const_br=0



    - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_stereo.py --ref_encoder_path ./IVAS_cod_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_enc_const_br_stereo_ref.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_stereo_ref.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_ref.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in stereo enc ref generation"}

    - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_ref.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in stereo enc ref generation"}



    # ref decoder stereo

    - $zero_errors_test_dec_const_br=0

    - $zero_failures_test_dec_const_br=0



    - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_stereo.py --ref_decoder_path ./IVAS_dec_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_dec_const_br_stereo_stereo_ref.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in stereo_stereo dec ref generation"}

    - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in stereo_stereo dec ref generation"}



    # use cut hrtf binaries

    - cp cut_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data



    # cut encoder stereo

    - echo "bitstream comparison"

    - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_stereo.py --dut_encoder_path ./IVAS_cod.exe --reduction_level reduced --html=logs/report-test_enc_const_br_stereo_cut.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_stereo_cut.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_cut.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in stereo"}

    - IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_cut.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in stereo ref generation"}



    # cut decoder stereo

    - echo "decoder output comparison | cut decoder stereo"

    - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_stereo.py --dut_decoder_path ./IVAS_dec.exe --reduction_level reduced --html=logs/report-test_dec_const_br_stereo_stereo_cut.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in stereo_stereo dec cut generation"}

    - IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in stereo_stereo dec cut generation"}



    - IF($zero_errors_test_enc_const_br -ne 0 -or $zero_errors_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}

    - IF($zero_failures_test_enc_const_b -ne 0 -or $zero_failures_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}



  artifacts:

    name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results"

    name: "test-be-to-release-stereo--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: 1 week

    paths:

      - logs/

    expose_as: "test-be-to-release results"

    expose_as: "test-be-to-release stereo results"

    reports:

      junit:

        - logs/report-junit-*.xml



test-be-to-release-multichannel:

  stage: test

  tags:

    - ivas-windows

  resource_group: ivas-be-to-release-test-resource

  timeout: "22 hours"  # to be reviewed

  extends:

    - .rules-test-be-release

  script:

    - *comparison-setup-codec-ref-vs-cut

    - mkdir logs



    - mkdir tests\codec_be_to_accepted_release\ref

    - mkdir tests\codec_be_to_accepted_release\dut



    - echo "generate references"



    # use ref hrtf binaries

    - cp ref_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data



    # ref encoder multichannel

    - $zero_errors_test_enc_const_br=0

    - $zero_failures_test_enc_const_br=0



    - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_multichannel.py --ref_encoder_path ./IVAS_cod_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_enc_const_br_multichannel_ref.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_multichannel_ref.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_ref.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in multichannel enc ref generation"}

    - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_ref.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in multichannel enc ref generation"}



    # ref decoder multichannel

    - $zero_errors_test_dec_const_br=0

    - $zero_failures_test_dec_const_br=0



    - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_multichannel.py --ref_decoder_path ./IVAS_dec_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_dec_const_br_multichannel_multichannel_ref.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in multichannel_multichannel dec ref generation"}

    - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in multichannel_multichannel dec ref generation"}



    # use cut hrtf binaries

    - cp cut_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data



    # cut encoder multichannel

    - echo "cut encoder multichannel"

    - ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_multichannel.py --dut_encoder_path ./IVAS_cod.exe --reduction_level reduced --html=logs/report-test_enc_const_br_multichannel_cut.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_multichannel_cut.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_cut.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in multichannel"}

    - IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_cut.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in multichannel ref generation"}



    # cut decoder multichannel

    - echo "cut decoder multichannel"

    - ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_multichannel.py --dut_decoder_path ./IVAS_dec.exe --reduction_level reduced --html=logs/report-test_dec_const_br_multichannel_multichannel_cut.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml --testcase_timeout 40 ") | Invoke-Expression



    - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in multichannel_multichannel dec cut generation"}

    - IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in multichannel_multichannel dec cut generation"}



    - IF($zero_errors_test_enc_const_br -ne 0 -or $zero_errors_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}

    - IF($zero_failures_test_enc_const_b -ne 0 -or $zero_failures_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}



  artifacts:

    name: "test-be-to-release-multichannel--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: 1 week

    paths:

      - logs/

    expose_as: "test-be-to-release multichannel results"

    reports:

      junit:

        - logs/report-junit-*.xml



test-long-self-test:

  tags: