Commit 5b73c8b5 authored by TYAGIRIS's avatar TYAGIRIS
Browse files

merge from main

parents 3d76e64f fb4dd681
Loading
Loading
Loading
Loading
Loading
+2 −0
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ scripts/ref/
scripts/test/
scripts/out/
scripts/self_test_summary.txt
tests/dut
tests/ref

# Python files that pop up when running scripts
__pycache__/
+271 −67
Original line number Diff line number Diff line
@@ -2,6 +2,12 @@ variables:
  TESTV_DIR: "/usr/local/testv"
  BUILD_OUTPUT: "build_output.txt"
  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"
  SANITIZER_TESTS: "CLANG1 CLANG2"
  OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"
  OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"
  OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM"
  EXIT_CODE_NON_BE: 123
  EXIT_CODE_FAIL: 1


# This sets when pipelines are created. Jobs have more specific rules to restrict them.
@@ -14,12 +20,12 @@ workflow:
    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main
    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main


stages:
  - maintenance
  - build
  - test
  - compare
  - validate

# ---------------------------------------------------------------
# Generic script anchors
@@ -52,6 +58,8 @@ stages:
  rules:
    - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined)
      when: never
    - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions)
      when: never
    - when: on_success

.rules-merge-request:
@@ -100,6 +108,25 @@ stages:
      - 123


# ---------------------------------------------------------------
# Validation jobs
# ---------------------------------------------------------------

check-if-branch-is-up-to-date-with-main:
  extends:
    - .rules-merge-request
  stage: validate
  needs: []
  tags:
    - ivas-linux
  script:
    - echo $CI_COMMIT_SHA
    - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
    - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME)
    - echo $commits_behind_count
    - if [ $commits_behind_count -eq 0 ]; then exit 0; else exit 1; fi;


# ---------------------------------------------------------------
# Build jobs
# ---------------------------------------------------------------
@@ -217,6 +244,12 @@ msan-on-merge-request-linux:
    - python3 scripts/self_test.py --create | tee test_output.txt
    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true
    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    paths:
      - scripts/ref/logs/
      - test_output.txt
    expose_as: 'Msan selftest results'


# code selftest testvectors with address-sanitizer binaries
@@ -233,10 +266,16 @@ asan-on-merge-request-linux:
    - python3 scripts/self_test.py --create | tee test_output.txt
    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true
    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    paths:
      - scripts/ref/logs/
      - test_output.txt
    expose_as: 'Asan selftest results'


# compare bit exactness between target and source branch
self-test-on-merge-request:
pytest-on-merge-request:
  extends: 
    - .test-job-linux
    - .rules-merge-request
@@ -278,40 +317,53 @@ self-test-on-merge-request:
    - mv IVAS_dec ../IVAS_dec_ref
    - cd ..

    ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)
    ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything)
    - git checkout $source_branch_commit_sha

    ### run selftest
    - ls -altr scripts/testv
    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    ### analyse test output

    # some helper variables - "|| true" to prevent failures from grep not finding anything
    - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true
    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true
    - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true
    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true
    - EXIT_CODE_NON_BE=123
    - EXIT_CODE_FAIL=1

    - selftest_exit_code=0
    - expected_nonbe_1=0
    - expected_nonbe_2=0
    - fail_1=0
    - fail_2=0

    ### prepare pytest
    # create short test vectors
    - python3 tests/create_short_testvectors.py
    # rename test binaries back
    - mv IVAS_cod_test IVAS_cod
    - mv IVAS_dec_test IVAS_dec
    # create references
    - python3 -m pytest tests -v --update_ref 1 -m create_ref
    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2
    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm

    ### run pytest
    - exit_code=0
    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?
    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    # check for crashes during the test, if any happened, fail the test
    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi
    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi

    # check for non bitexact output and store exit code to also always run the SBA pytest
    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi
    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi
    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi
    - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi

    ### run SBA pytest
    ### run pytest for EVS cases
    - exit_code=0
    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?
    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi
    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true
    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;
    # return exit code from selftest if everything went well with the pytest run
    - exit $selftest_exit_code
    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$?
    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true

    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi

    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi
    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi

    # Check results from both tests
    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi
    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi
    - exit 0

  allow_failure:
    exit_codes:
      - 123
@@ -319,13 +371,13 @@ self-test-on-merge-request:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    when: always
    paths:
      - test_output.txt
      - scripts/test/logs/
      - scripts/ref/logs/
      - report-junit.xml
    expose_as: 'Self test results'
      - report-junit-evs.xml
    expose_as: 'pytest results'
    reports:
      junit: report-junit.xml
      junit:
        - report-junit.xml
        - report-junit-evs.xml


# ---------------------------------------------------------------
@@ -341,6 +393,7 @@ be-2-evs-linux:
    - be-2-evs-temp
  stage: test
  needs: [ "build-codec-linux-cmake" ]
  timeout: "20 minutes" # To be revisited
  script:
    - *print-common-info

@@ -401,37 +454,26 @@ codec-comparison-on-main-push:
    ### re-checkout the latest commit in the main branch
    - git checkout $latest_commit

    ### run selftest
    - ls -altr scripts/testv
    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt
    # helper variable - "|| true" to prevent failures from grep not finding anything
    - non_be_flag=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[non[ -]*be\]") || true

    ### analyse test output

    # some helper variables - "|| true" to prevent failures from grep not finding anything
    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true
    - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true
    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true
    - EXIT_CODE_NON_BE=123
    - EXIT_CODE_FAIL=1
    ### prepare pytest
    # create short test vectors
    - python3 tests/create_short_testvectors.py
    # rename test binaries back
    - mv IVAS_cod_test IVAS_cod
    - mv IVAS_dec_test IVAS_dec
    # create references
    - python3 -m pytest tests -v --update_ref 1 -m create_ref
    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - selftest_exit_code=0

    # check for crashes during the test, if any happened, fail the test
    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi

    # check for non bitexact output and store exit code to also always run the SBA pytest
    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi
    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi

    ### run SBA pytest
    ### run pytest
    - exit_code=0
    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?
    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?
    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi
    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true
    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;
    # return exit code from selftest if everything went well with the pytest run
    - exit $selftest_exit_code
  allow_failure:
    exit_codes:
      - 123
@@ -439,26 +481,188 @@ codec-comparison-on-main-push:
    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    when: always
    paths:
      - test_output.txt
      - scripts/test/logs/
      - scripts/ref/logs/
      - report-junit.xml
    expose_as: 'Results of comparison to previous merge commit'
    reports:
      junit: report-junit.xml


sanitizer-test-on-main-scheduled:
  extends: .test-job-linux-needs-testv-dir
# ---------------------------------------------------------------
# Scheduled jobs on main
# ---------------------------------------------------------------
.sanitizer-test-template:
  extends:
    - .test-job-linux-needs-testv-dir
  stage: test
  tags:
    - sanitizer_test_main
  artifacts:
    name: "$CI_JOB_NAME--main--sha-$CI_COMMIT_SHORT_SHA"
    when: always
    paths:
      - ep_015.g192
      # second wildcard is necessary to get encoder and no-PLC run logs
      - "CLANG*/logs*"

sanitizer-test-mono:
  extends: .sanitizer-test-template
  rules:
    # only run in scheduled pipeline that passes this env var
    - if: $SANITIZER_TEST_IN_FMT
    - if: $IS_SANITIZER_TEST_RUN
  script:
    - *print-common-info
    - echo "Running scheduled sanitizer"
    # - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS
    - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS

sanitizer-test-stereo:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 20 minutes
  script:
    - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS

sanitizer-test-stereodmxevs:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 40 minutes
  script:
    - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS

sanitizer-test-ism1:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 1 hour
  script:
    - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

sanitizer-test-ism2:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 1 hour 30 minutes
  script:
    - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

sanitizer-test-ism3:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 2 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

sanitizer-test-ism4:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 2 hours 30 minutes
  script:
    - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

sanitizer-test-mc-5_1:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 3 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

sanitizer-test-mc-5_1_2:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 4 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

sanitizer-test-mc-5_1_4:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 5 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

sanitizer-test-mc-7_1:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 6 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

sanitizer-test-mc-7_1_4:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 7 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

sanitizer-test-masa:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 8 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

sanitizer-test-sba:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 9 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

sanitizer-test-planarsba:
  extends: .sanitizer-test-template
  rules:
    - if: $IS_SANITIZER_TEST_RUN
      when: delayed
      start_in: 10 hours
  script:
    - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

# GCOV/LCOV coverage analysis of self_test suite
coverage-test-on-main-scheduled:
  extends: 
    - .test-job-linux-needs-testv-dir
    - .rules-main-scheduled    
  tags:
    - coverage-test
  stage: test
  rules:
    # only run in scheduled pipeline that passes this env vars
    - if: $COVERAGE_TEST  
  script:
    - *print-common-info
    - make GCOV=1 -j
    - python3 tests/create_short_testvectors.py
    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
    - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
    - lcov -c -d obj -o coverage.info
    - genhtml coverage.info -o coverage
  artifacts:
    name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"
    when: always
    paths:
      - coverage.info
      - coverage

# ---------------------------------------------------------------
# Other jobs
+1 −0
Original line number Diff line number Diff line
@@ -267,6 +267,7 @@
    <ClCompile Include="..\lib_dec\ivas_allrad_dec.c" />
    <ClCompile Include="..\lib_dec\ivas_binauralRenderer.c" />
    <ClCompile Include="..\lib_dec\ivas_binaural_reverb.c" />
    <ClCompile Include="..\lib_dec\ivas_corecoder_dec_reconfig.c" />
    <ClCompile Include="..\lib_dec\ivas_core_dec.c" />
    <ClCompile Include="..\lib_dec\ivas_cpe_dec.c" />
    <ClCompile Include="..\lib_dec\ivas_crend.c" />
+3 −0
Original line number Diff line number Diff line
@@ -581,6 +581,9 @@
    <ClCompile Include="..\lib_dec\ivas_spar_md_dec.c">
      <Filter>dec_ivas_c</Filter>
    </ClCompile>
    <ClCompile Include="..\lib_dec\ivas_corecoder_dec_reconfig.c">
      <Filter>dec_ivas_c</Filter>
    </ClCompile>
  </ItemGroup>
  <ItemGroup>
    <ClInclude Include="..\lib_dec\jbm_jb4_inputbuffer.h">
+1 −0
Original line number Diff line number Diff line
@@ -206,6 +206,7 @@
  </ItemDefinitionGroup>
  <ItemGroup>
    <ClCompile Include="..\lib_enc\ivas_agc_enc.c" />
    <ClCompile Include="..\lib_enc\ivas_corecoder_enc_reconfig.c" />
    <ClCompile Include="..\lib_enc\ivas_core_pre_proc.c" />
    <ClCompile Include="..\lib_enc\ivas_core_pre_proc_front.c" />
    <ClCompile Include="..\lib_enc\ivas_enc_cov_handler.c" />
Loading