Commit 4d10a138 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'kiene/port-short-sanitizer-tests-to-float-pc' into 'float-pc' 

[CI] port remaining CI jobs from float repo to float-pc

See merge request !982
parents 01e19c69 242dcb95

.gitlab-ci.yml

+413 −3
Original line number Diff line number Diff line 
variables:

  TESTV_DIR: "/usr/local/testv"

  LTV_DIR: "/usr/local/ltv"

  EVS_BE_TEST_DIR_FLOAT: "/usr/local/be_2_evs_test"

  EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop"

  REFERENCE_BRANCH: "ivas-float-update"

  BUILD_OUTPUT: "build_output.txt"
@@ -146,6 +147,10 @@ stages: 
.mr-get-target-commit: &mr-get-target-commit # compare to last target branch commit before pipeline was created

  - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H)



# from float

.check-for-testvectors: &check-for-testvectors # check if the testvector files specified in scripts/config/ci_linux*.json are present

  - python3 -m pytest ci/test_vectors_available.py



# From float CI

.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec

  ### build test binaries, initial clean for paranoia reasons
@@ -241,6 +246,10 @@ stages: 
  - git pull

  - cd -



.disable-limiter: &disable-limiter

# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment

  - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*DISABLE_LIMITER\)[[:space:]]*\*\//\1/g" ./lib_com/options.h



.get-commits-behind-count: &get-commits-behind-count

  - echo $CI_COMMIT_SHA

  - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
@@ -266,6 +275,37 @@ stages: 
  - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt"

  - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace '<TreatWarningAsError>false', '<TreatWarningAsError>true' | Set-Content -Path $_.FullName }



# From float repo

# to be reused in MR and LTV-scheduled sanitizer test jobs

# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section

.sanitizer-selftest-anchor: &sanitizer-selftest-anchor

  script:

    - *print-common-info

    - *update-scripts-repo

    - *copy-ltv-files-to-testv-dir

    - make clean

    - make -j CLANG=$CLANG_NUM

    - testcase_timeout=$SELFTEST_SANITY_TIMEOUT

    - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1



    - python3 ci/remove_unsupported_testcases.py $PRM_FILES



    - exit_code20=0

    - exit_code10=0

    - exit_code5=0



    - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi



    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi



.rules-pytest-to-ref-short:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare"
@@ -364,6 +404,13 @@ stages: 
    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never



.rules-merge-request-to-float-pc:

  extends: .rules-basis

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "float-pc"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never



# templates to define stages and platforms

.test-job-linux:

  tags:
@@ -392,6 +439,28 @@ stages: 
    - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi

    - cp -r scripts/testv/* $TESTV_DIR/



.sanitizer-selftest-on-mr:

  stage: test

  extends:

    - .rules-merge-request-to-float-pc

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



.ivas-pytest-anchor: &ivas-pytest-anchor

  stage: test

  needs: ["build-codec-linux-make"]
@@ -810,6 +879,76 @@ clang-format-check: 
    name: "$ARTIFACT_BASE_NAME"

    expose_as: "formatting patch"



# from float

# check for crashes if first received frame on decoder side is an SID

check-first-frame-is-sid:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request-to-float-pc

  tags:

    - ivas-linux

  stage: test

  needs: ["build-codec-linux-make"]

  script:

    - *print-common-info

    - *update-scripts-repo

    - *update-ltv-repo

    # this rm makes check-for-testvectors only check for the signals we actually need in this test

    - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json

    - *check-for-testvectors

    - bash ci/run-first-frame-is-sid-test.sh



# from float

check-bitexactness-hrtf-rom-and-file:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "5 minutes"

  script:

    - *print-common-info

    - *update-scripts-repo

    - make clean

    - make -j

    - python3 tests/create_short_testvectors.py --cut_len 1.0

    # TODO: run full test again once the custom binary files are supported

    - python3 -m pytest -k "not diff_from_rom and not test_binary_file" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html

  artifacts:

    paths:

      - report.html

      - report-junit.xml

    when: always

    name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading"

    expose_as: "logs-hrtf-loading"

    expire_in: "5 days"



# from float

# TODO: this needs setup/addition of tag on runners

.be-2-evs-linux:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  tags:

    - be-2-evs-float

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "20 minutes" # To be revisited

  script:

    - *print-common-info

    - *update-scripts-repo



    - make clean

    - make -j



    # copy over to never change the testvector dir

    - cp -r $EVS_BE_TEST_DIR_FLOAT ./evs_be_test

    - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod

    - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec



    - cd evs_be_test

    - python3 ../ci/run_evs_be_test.py



# ---------------------------------------------------------------

# Build jobs

# ---------------------------------------------------------------
@@ -847,7 +986,6 @@ build-codec-linux-instrumented-make: 
  tags:

    - ivas-linux

  script:

    - *update-scripts-repo

    - *print-common-info

    - *update-scripts-repo

    - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP
@@ -1273,7 +1411,7 @@ be-2-evs-26444: 
    - .test-job-linux

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444"

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' (&& $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main-pc" )

  tags:

    - be-2-evs-basop

  stage: test
@@ -1281,7 +1419,9 @@ be-2-evs-26444: 
  script:

    - *print-common-info

    - *update-scripts-repo

    # needed on BASOP branches only. On ivas-flaot-update + derivates this does nothing as the replaced string does not exist in options.h

    - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h

    - make clean

    - make -j



    # copy over to never change the testvector dir
@@ -1326,8 +1466,10 @@ voip-be-on-merge-request: 
  extends:

    - .test-job-linux-needs-testv-dir

  rules:

    # - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "voip-be-test"

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "float-pc"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "10 minutes"
@@ -1375,6 +1517,221 @@ voip-be-on-merge-request: 
      - smoke_test_output_hrtf.txt

    expose_as: "Smoke test results"



# from float repo

# NOTE: timeout was adjusted to work for float only

# code selftest testvectors with memory-sanitizer binaries

.codec-msan:

  extends:

    - .sanitizer-selftest-on-mr

  tags:

    - ivas-linux

  before_script:

    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=180

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with address-sanitizer binaries

.codec-asan:

  extends:

    - .sanitizer-selftest-on-mr

  tags:

    - ivas-linux

  before_script:

    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=180

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with undefined-behaviour-sanitizer binaries

.codec-usan:

  extends:

    - .sanitizer-selftest-on-mr

  tags:

    - ivas-linux

  before_script:

    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=180

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  <<: *sanitizer-selftest-anchor



# from float

# compare bit-exactness between 5ms and 20 on the branch

# TODO: activate again once all the renderer framesize BE fixes are ported

.pytest-compare-20ms-and-5ms-rendering:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  stage: test

  needs: ["build-codec-linux-make", "build-codec-linux-instrumented-make", "build-codec-sanitizers-linux"]

  script:

    - *print-common-info

    - *update-scripts-repo

    - *disable-limiter

    - make clean

    - make -j

    ### prepare pytest

    - cp IVAS_cod IVAS_cod_ref

    - cp IVAS_dec IVAS_dec_ref



    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    # create references

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1

    ### run pytest

    - exit_code=0

    - exit_code5=0

    - exit_code10=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$?

    - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true

    - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true

    - zero_errors=1

    - if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi

    - if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi

    - if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi

    - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi

    - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    expose_as: "pytest 5ms and 10ms vs 20ms results"

    paths:

      - report-junit-5ms.xml

      - report-5ms.html

      - report-junit-10ms.xml

      - report-10ms.html

    reports:

      junit:

        - report-junit-5ms.xml

        - report-junit-10ms.xml



# from float

# test renderer executable

renderer-smoke-test:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  needs: ["build-codec-linux-make"]

  stage: test

  script:

    - *print-common-info

    - *update-scripts-repo

    - make -j IVAS_rend

    - testcase_timeout=60

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

    expose_as: "renderer make pytest results"

    reports:

      junit:

        - report-junit.xml



# from float

# test renderer executable with cmake + asan

renderer-asan:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  needs: ["build-codec-linux-make"]

  tags: 

    - ivas-linux

  stage: test

  script:

    # was there from float repo. commented out until cmake is there

    # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    # - cmake --build cmake-build -- -j

    - *update-scripts-repo

    - make clean

    - make -j

    - testcase_timeout=180

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout



  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

    expose_as: "renderer asan pytest results"

    reports:

      junit:

        - report-junit.xml



# from float

# test renderer executable with cmake + msan

renderer-msan:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  needs: ["build-codec-linux-make"]

  stage: test

  tags: 

    - ivas-linux

  script:

    # was there from float repo. commented out until cmake is there

    # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    # - cmake --build cmake-build -- -j

    - *update-scripts-repo

    - make clean

    - make -j

    - testcase_timeout=180

    - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout



  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

    expose_as: "renderer msan pytest results"

    reports:

      junit:

        - report-junit.xml



# from float

# test renderer executable with cmake + usan

renderer-usan:

  extends:

    - .test-job-linux

    - .rules-merge-request-to-float-pc

  needs: ["build-codec-linux-make"]

  stage: test

  tags: 

    - ivas-linux

  script:

    # was there from float repo. commented out until cmake is there

    # - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    # - cmake --build cmake-build -- -j

    - *update-scripts-repo

    - make clean

    - make -j

    - testcase_timeout=180

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,log_path=usan_log_catchall python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout

    - grep_exit_code=0

    - touch usan_log_empty # Creates an empty file, this is to avoid "grep: usan_log_*: No such file or directory" in case no USAN failures are reported from pytest

    - grep UndefinedBehaviorSanitizer usan_log_* || grep_exit_code=$?

    - if [ $grep_exit_code != 1 ] ; then echo "Run errors in test_renderer.py with Clang undefined-behavior-sanitizer"; exit 1; fi



  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

    expose_as: "renderer usan pytest results"

    reports:

      junit:

        - report-junit.xml



# compare renderer bitexactness between target and source branch

renderer-pytest-on-merge-request:

  extends:
@@ -1478,6 +1835,59 @@ ivas-pytest-on-merge-request: 
      junit:

        - report-junit.xml



### From Float

# Check interop IVAS_cod_test -> IVAS_dec_ref

ivas-interop-on-merge-request:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request-to-float-pc

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "10 minutes"

  script:

    - *print-common-info

    - *get-commits-behind-count

    - *check-commits-behind-count-in-compare-jobs

    - *merge-request-comparison-setup-codec

    # the next line is dependent on ref-using-main flag in the other tests, but here the flag does not make sense

    - git checkout $source_branch_commit_sha



    - python3 ci/remove_unsupported_testcases.py $PRM_FILES



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    # write to temporary file as workaround for failures observed with piping echo

    - echo $CI_MERGE_REQUEST_TITLE > tmp.txt

    - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true



    ### prepare pytest



    # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec)

    - exit_code=0

    - exit_code2=0

    # set timeout for individual testcase runs to 60 seconds

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true



    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi

    - exit 0



  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "interop test results"

    reports:

      junit:

        - report*-junit.xml



# ---------------------------------------------------------------

# Complexity measurement jobs

# ---------------------------------------------------------------

lib_com/options.h

+2 −0
Original line number Diff line number Diff line
@@ -139,6 +139,8 @@ 


#endif /* DEBUGGING */



/*#define DISABLE_LIMITER*/                     /* disable the limiter - needed for testing Bitexactness between different renderer framings */



/* #################### End DEBUGGING switches ############################ */



/* ################### Start FIXES switches ########################### */

lib_rend/lib_rend.c

+2 −0
Original line number Diff line number Diff line
@@ -336,6 +336,7 @@ static void accumulate2dArrayToBuffer( 
 * In-place saturation control for multichannel buffers with adaptive release time

 *-------------------------------------------------------------------*/



#ifndef DISABLE_LIMITER

/*! r: number of clipped output samples */

static int32_t limitRendererOutput(

    IVAS_LIMITER_HANDLE hLimiter, /* i/o: limiter struct handle                                           */
@@ -380,6 +381,7 @@ static int32_t limitRendererOutput( 


    return numClipping;

}

#endif





/*-------------------------------------------------------------------*