Commit c0139534 authored by norvell's avatar norvell
Browse files

Change to Linux conformance

parent 2bc7755e

main-basop.yml

+22 −15
Original line number Diff line number Diff line
@@ -80,9 +80,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

      variables:

        IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

      variables:

        IVAS_PIPELINE_NAME: 'IVAS Conformance: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH'        

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
@@ -152,7 +152,7 @@ workflow: 


.rules-pytest-to-ref-pre:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "ivas-conformance"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "ivas-conformance-linux"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'web'

    - if: $CI_PIPELINE_SOURCE == 'schedule'
@@ -955,7 +955,7 @@ pytest-to-ref-pre: 
# ensure that codec builds on linux

build-codec-linux-make:

  rules:

    - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES

    - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES

      when: never

    - if: $CI_PIPELINE_SOURCE == 'web'

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -975,7 +975,7 @@ build-codec-linux-cmake: 
  rules:

    - if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/

      when: never

    - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance' || $UPDATE_PAGES

    - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' || $UPDATE_PAGES

      when: never

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -1060,7 +1060,7 @@ build-codec-linux-debugging-make: 


build-codec-windows-msbuild:

  rules:

    - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance' || $UPDATE_PAGES

    - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' || $UPDATE_PAGES

      when: never

    - if: $CI_PIPELINE_SOURCE == 'web'

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -3176,23 +3176,28 @@ complexity-osba-in-binaural_room_ir-out: 
    - *complexity-measurements-report-summary

    - exit $ret_val



ivas-conformance:

ivas-conformance-linux:

  extends:

    - .job-windows

  tags:

    - ivas-basop-windows

    - .job-linux

  stage: test

  timeout: "6 hours"

  rules:

    - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance'

    - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

    - if: $MANUAL_PIPELINE_TYPE == "coverage"

    - if: $COVERAGE_TEST

  allow_failure:

    exit_codes:

      - 123

  variables:

    COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_CONFORMANCE"

    COVERAGE_OUTPUT_DIR: "coverage_conformance"

    COVERAGE_TITLE: "conformance test test_26252.py"

  script:

    - *print-common-info-windows

    - ."$CI_PROJECT_DIR/ivas-codec-ci/snippets/ivas-conformance.ps1"

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/ivas-conformance.sh



  artifacts:

    name: "ivas-conformance-$CI_COMMIT_SHORT_SHA"

    name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA"

    expire_in: 1 week

    when: always

    paths:
@@ -3205,7 +3210,9 @@ ivas-conformance: 
      - Readme_IVAS_JBM_dec.txt

      - Readme_IVAS_ISAR_dec.txt

      - Readme_IVAS_ISAR_post_rend.txt

    expose_as: "Draft IVAS conformance"

      - $COVERAGE_OUTPUT_FILE

      - $COVERAGE_OUTPUT_DIR

    expose_as: "Draft IVAS conformance -- Linux"

    reports:

      junit: report-junit.xml

snippets/ivas-conformance.ps1

deleted100644 → 0
+0 −59
Original line number Diff line number Diff line 
MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug

cp -force IVAS_cod.exe IVAS_cod_ref.exe

cp -force IVAS_dec.exe IVAS_dec_ref.exe

cp -force IVAS_rend.exe IVAS_rend_ref.exe

cp -force ISAR_post_rend.exe ISAR_post_rend_ref.exe



# Reference creation

python scripts/prepare_combined_format_inputs.py

$TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py"

python -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



# Copy input data and output ref data

if (Test-Path testvec) {rm -r -force testvec}

if (Test-Path TMP_DEC) {rm -r -force TMP_DEC}

if (Test-Path TMP_ENC) {rm -r -force TMP_ENC}

if (Test-Path TMP_JBM) {rm -r -force TMP_JBM}

if (Test-Path TMP_REND) {rm -r -force TMP_REND}

if (Test-Path TMP_ISAR_POST_REND) {rm -r -force TMP_ISAR_POST_REND}

if (Test-Path TMP_DEC_ISAR) {rm -r -force TMP_DEC_ISAR}

mkdir testvec

mkdir testvec/binauralRenderer_interface

mkdir testvec/testv

mkdir testvec/testv/renderer_short

mkdir testvec/testv/split_rendering

mkdir testvec/bin

cp    -force -ErrorAction Ignore scripts/testv/* testvec/testv

cp -r -force -ErrorAction Ignore scripts/ls_layouts testvec

cp -r -force -ErrorAction Ignore scripts/object_edit testvec

cp -r -force -ErrorAction Ignore scripts/switchPaths testvec

cp -r -force -ErrorAction Ignore scripts/trajectories testvec

cp -r -force -ErrorAction Ignore scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface

cp -r -force -ErrorAction Ignore tests/ref testvec/testv/ref

cp -r -force -ErrorAction Ignore tests/renderer_short/ref testvec/testv/renderer_short/ref

cp -r -force -ErrorAction Ignore tests/split_rendering/ref testvec/testv/split_rendering/ref

cp -r -force -ErrorAction Ignore tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs

cp -r -force -ErrorAction Ignore tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns



# Remove redundant files

python scripts/cleanup_26252.py



# Copy test script files

cp -r -force -ErrorAction Ignore tests/conformance-test testvec/

cp Readme_IVAS_dec.txt testvec

cp Readme_IVAS_enc.txt testvec

cp Readme_IVAS_rend.txt testvec

cp Readme_IVAS_JBM_dec.txt testvec

cp Readme_IVAS_ISAR_dec.txt testvec

cp Readme_IVAS_ISAR_post_rend.txt testvec

cp IVAS_cod.exe testvec/bin

cp IVAS_dec.exe testvec/bin

cp IVAS_rend.exe testvec/bin

cp ISAR_post_rend.exe testvec/bin



# Test run generated scripts in testvec

cd testvec

python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html

mv report.html ..

mv report-junit.xml ..

snippets/ivas-conformance.sh

0 → 100644
+72 −0
Original line number Diff line number Diff line 
#! /bin/bash



set -euxo pipefail





make -j

cp IVAS_cod IVAS_cod_ref

cp IVAS_dec IVAS_dec_ref

cp IVAS_rend IVAS_rend_ref

cp ISAR_post_rend ISAR_post_rend_ref



# Reference creation

python3 scripts/prepare_combined_format_inputs.py

TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py"

python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



# Copy input data and output ref data

rm -rf testvec

mkdir testvec

mkdir testvec/binauralRenderer_interface

mkdir testvec/testv

mkdir testvec/testv/renderer_short

mkdir testvec/testv/split_rendering

mkdir testvec/bin

cp -r scripts/testv/* testvec/testv

cp -r scripts/ls_layouts testvec

cp -r scripts/object_edit testvec

cp -r scripts/switchPaths testvec

cp -r scripts/trajectories testvec

cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface

cp -r tests/ref testvec/testv/ref

cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref

cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref

cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs

cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns



# Remove redundant files

python3 scripts/cleanup_26252.py



# Copy test script files

cp -r tests/conformance-test testvec/

cp Readme_IVAS_dec.txt testvec

cp Readme_IVAS_enc.txt testvec

cp Readme_IVAS_rend.txt testvec

cp Readme_IVAS_JBM_dec.txt testvec

cp Readme_IVAS_ISAR_dec.txt testvec

cp Readme_IVAS_ISAR_post_rend.txt testvec



# Create GCOV execs for coverage analysis

make clean

make GCOV=1 -j



cp IVAS_cod testvec/bin

cp IVAS_dec testvec/bin

cp IVAS_rend testvec/bin

cp ISAR_post_rend testvec/bin



# Test run generated scripts in testvec

cd testvec

exit_code=0

python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$?

mv report.html ..

mv report-junit.xml ..



# Collect coverage

cd -

bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE"



# Check for failures

if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi