Commit d737c65b authored by norvell's avatar norvell
Browse files

Update .gitlab-ci.yml

parent 3e740a77

.gitlab-ci.yml

+2 −523
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main

    - if: $CI_PIPELINE_SOURCE == 'schedule'



stages:

  - maintenance
@@ -111,533 +112,11 @@ stages: 
# Build jobs

# ---------------------------------------------------------------



build-codec-linux-make:

  extends: 

    - .build-job-with-check-for-warnings

    - .rules-basis

  script:

    - *print-common-info

    - make -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-unittests-linux:

  extends: 

    - .build-job-with-check-for-warnings

    - .rules-basis

  script:

    - *print-common-info

    - make unittests -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-prerenderer-linux:

  extends: 

    - .build-job-with-check-for-warnings

    - .rules-basis

  script:

    - *print-common-info

    - make -C scripts/prerenderer -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-td-object-renderer-standalone-linux:

  extends: 

    - .build-job-with-check-for-warnings

    - .rules-basis

  script:

    - *print-common-info

    - make -C scripts/td_object_renderer/object_renderer_standalone -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-codec-linux-cmake:

  extends: 

    - .build-job-with-check-for-warnings

    - .rules-basis

  script:

    - *print-common-info

    - mkdir build

    - cd build

    - cmake ..

    - cd ..

    - make -C build -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-codec-instrumented-linux:

  extends: 

    - .build-job-linux

    - .rules-basis

  script:

    - *print-common-info

    - bash ci/build_codec_instrumented_linux.sh



# make sure that the codec builds with msan, asan and usan

build-codec-sanitizers-linux:

  extends: 

    - .build-job-linux

    - .rules-basis

  script:

    - *print-common-info

    - bash ci/build_codec_sanitizers_linux.sh





# ---------------------------------------------------------------

# Test jobs for merge requests

# ---------------------------------------------------------------



# test that runs all modes with 1s input signals

codec-smoke-test:

  extends: 

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  timeout: "5 minutes"

  stage: test

  needs: [ "build-codec-linux-cmake" ]

  script:

    - *print-common-info

    - bash ci/smoke_test.sh

    ### analyze for failures

    - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without PLC failed"; exit 1; fi

    - if cat smoke_test_output_plc.txt | grep -c "failed"; then echo "Smoke test with PLC failed"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    paths:

      - out/logs/

      - smoke_test_output.txt

      - smoke_test_output_plc.txt

    expose_as: 'Smoke test results'





# code selftest testvectors with memory-sanitizer binaries

msan-on-merge-request-linux:

  extends: 

    - .test-job-linux

    - .rules-merge-request

  stage: test

  needs: [ "build-codec-sanitizers-linux" ]

  script:

    - *print-common-info

    - make clean

    - make -j CLANG=1

    - python3 scripts/self_test.py --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    paths:

      - scripts/ref/logs/

      - test_output.txt

    expose_as: 'Msan selftest results'





# code selftest testvectors with address-sanitizer binaries

asan-on-merge-request-linux:

  extends: 

    - .test-job-linux

    - .rules-merge-request

  stage: test

  needs: [ "build-codec-sanitizers-linux" ]

  script:

    - *print-common-info

    - make clean

    - make -j CLANG=2

    - python3 scripts/self_test.py --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    paths:

      - scripts/ref/logs/

      - test_output.txt

    expose_as: 'Asan selftest results'





# compare bit exactness between target and source branch

self-test-on-merge-request:

  extends: 

    - .test-job-linux

    - .rules-merge-request

  stage: compare

  needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]

  timeout: "10 minutes"

  script:

    - *print-common-info

    ### build test binaries, initial clean for paranoia reasons

    - make clean

    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_test

    - mv IVAS_dec ../IVAS_dec_test

    - cd ..

    - rm -rf build/*



    ### store the current commit hash

    - source_branch_commit_sha=$(git rev-parse HEAD)



    ### checkout version to compare against

    # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching

    # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later

    - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true

    # needed when depth is lower than the number of commits in the branch

    - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME



    ### compare to last target branch commit before pipeline was created

    - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H)

    - git checkout $target_commit



    ### build reference binaries

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_ref

    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..



    ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)

    - git checkout $source_branch_commit_sha



    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt

    ### analyse test output



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1

    - expected_nonbe_1=0

    - expected_nonbe_2=0

    - expected_nonbe_3=0

    - fail_1=0

    - fail_2=0

    - fail_3=0



    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi



    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi



    # check for non bitexact EVS output

    - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi

    - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi



    ### run SBA pytest

    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    

    - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi

    

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi

    

    # Check results from all three tests

    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi

    - exit 0

    

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - test_output.txt

      - test_output_evs.txt

      - scripts/test/logs/

      - scripts/ref/logs/

      - report-junit.xml

    expose_as: 'Self test results'

    reports:

      junit: report-junit.xml





# ---------------------------------------------------------------

# Test jobs for main branch

# ---------------------------------------------------------------



# check bitexactness to EVS

be-2-evs-linux:

  extends:

    - .test-job-linux

    - .rules-main-push

  tags:

    - be-2-evs-temp

  stage: test

  needs: [ "build-codec-linux-cmake" ]

  timeout: "20 minutes" # To be revisited

  script:

    - *print-common-info



    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - cd ..



    # copy over to never change the testvector dir

    - cp -r $EVS_BE_TEST_DIR ./evs_be_test

    - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod

    - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec



    - cd evs_be_test

    - python3 ../ci/run_evs_be_test.py





codec-comparison-on-main-push:

  extends:

    - .test-job-linux

    - .rules-main-push

  stage: compare

  needs: [ "build-codec-linux-cmake" ]

  timeout: "30 minutes" # To be revisited

  script:

    - *print-common-info

    - latest_commit=$(git rev-parse HEAD) # Latest commit

    - *get-previous-merge-commit-sha # Stored in previous_merge_commit shell variable now

    - echo "Comparing changes from $previous_merge_commit to $latest_commit"

    - git --no-pager diff --stat $previous_merge_commit..$latest_commit



    # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests.

    

    ### build test binaries, initial clean for paranoia reasons

    - make clean

    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_test

    - mv IVAS_dec ../IVAS_dec_test

    - cd ..

    - rm -rf build/*



    ### compare to the previous merge commit in the main branch

    - git fetch origin main

    - git checkout $previous_merge_commit



    ### build reference binaries

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_ref

    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..



    ### re-checkout the latest commit in the main branch

    - git checkout $latest_commit



    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt



    ### analyse test output



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1



    - selftest_exit_code=0



    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi



    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi



    ### run SBA pytest

    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

    # return exit code from selftest if everything went well with the pytest run

    - exit $selftest_exit_code

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - test_output.txt

      - scripts/test/logs/

      - scripts/ref/logs/

      - report-junit.xml

    expose_as: 'Results of comparison to previous merge commit'

    reports:

      junit: report-junit.xml





# ---------------------------------------------------------------

# Scheduled jobs on main

# ---------------------------------------------------------------

.sanitizer-test-template:

  extends:

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  artifacts:

    name: "$CI_JOB_NAME--main--sha-$CI_COMMIT_SHORT_SHA"

    when: always

    paths:

      - ep_015.g192



sanitizer-test-mono:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

  script:

    - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS



sanitizer-test-stereo:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 20 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS



sanitizer-test-stereodmxevs:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 40 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS



sanitizer-test-ism1:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 1 hour

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism2:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 1 hour 30 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism3:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 2 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism4:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 2 hours 30 minutes

  script:

    - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 3 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1_2:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 4 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1_4:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 5 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-7_1:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 6 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-7_1_4:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 7 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-masa:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 8 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-sba:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 9 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-planarsba:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 10 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



# GCOV/LCOV coverage analysis of self_test suite

coverage-test-on-main-scheduled:

  extends: 

    - .test-job-linux-needs-testv-dir  

    - .rules-merge-request    

  tags:

    - coverage-test

  stage: test