Commit 11b4c86c authored by norvell's avatar norvell
Browse files

Merge branch 'Ericsson/ci-split-self-test-into-separate-jobs' into 'main' 

Split merge request pipeline with separate jobs for IVAS and EVS

See merge request !156
parents 53721434 8d19521a

.gitlab-ci.yml

+81 −60
Original line number Diff line number Diff line
@@ -44,6 +44,52 @@ stages: 
.get-previous-merge-commit-sha: &get-previous-merge-commit-sha

  - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H)



.merge_request_comparison_setup: &merge_request_comparison_setup

  ### build test binaries, initial clean for paranoia reasons

  - make clean

  - mkdir build

  - cd build

  - cmake ..

  - make -j

  - mv IVAS_cod ../IVAS_cod_test

  - mv IVAS_dec ../IVAS_dec_test

  - cd ..

  - rm -rf build/*



  ### store the current commit hash

  - source_branch_commit_sha=$(git rev-parse HEAD)



  ### checkout version to compare against

  # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching

  # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later

  - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true

  # needed when depth is lower than the number of commits in the branch

  - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME



  ### compare to last target branch commit before pipeline was created

  - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H)

  - git checkout $target_commit



  ### build reference binaries

  - cd build

  - cmake ..

  - make -j

  - mv IVAS_cod ../IVAS_cod_ref

  - mv IVAS_dec ../IVAS_dec_ref

  - cd ..



  # rename test binaries back

  - mv IVAS_cod_test IVAS_cod

  - mv IVAS_dec_test IVAS_dec



  ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)

  - git checkout $source_branch_commit_sha



.merge-request-comparison-check: &merge-request-comparison-check

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact  cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi

    - exit 0



# ---------------------------------------------------------------

# Job templates
@@ -275,7 +321,7 @@ asan-on-merge-request-linux: 




# compare bit exactness between target and source branch

pytest-on-merge-request:

ivas-pytest-on-merge-request:

  extends: 

    - .test-job-linux

    - .rules-merge-request
@@ -284,85 +330,62 @@ pytest-on-merge-request: 
  timeout: "10 minutes"

  script:

    - *print-common-info

    ### build test binaries, initial clean for paranoia reasons

    - make clean

    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_test

    - mv IVAS_dec ../IVAS_dec_test

    - cd ..

    - rm -rf build/*



    ### store the current commit hash

    - source_branch_commit_sha=$(git rev-parse HEAD)



    ### checkout version to compare against

    # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching

    # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later

    - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true

    # needed when depth is lower than the number of commits in the branch

    - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME



    ### compare to last target branch commit before pipeline was created

    - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H)

    - git checkout $target_commit



    ### build reference binaries

    - cd build

    - cmake ..

    - make -j

    - mv IVAS_cod ../IVAS_cod_ref

    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..



    ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything)

    - git checkout $source_branch_commit_sha

    - *merge_request_comparison_setup



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - expected_nonbe_1=0

    - expected_nonbe_2=0

    - fail_1=0

    - fail_2=0



    ### prepare pytest

    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi

    - *merge-request-comparison-check



  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - report-junit.xml

    expose_as: 'pytest ivas results'

    reports:

      junit:

        - report-junit.xml



evs-pytest-on-merge-request:

  extends: 

    - .test-job-linux

    - .rules-merge-request

  stage: compare

  needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]

  timeout: "10 minutes"

  script:

    - *print-common-info

    - *merge_request_comparison_setup



    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi

    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true



    ### prepare pytest

    # create references

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### run pytest for EVS cases

    - exit_code=0

    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$?

    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi



    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi

    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi



    # Check results from both tests

    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi

    - exit 0

    - *merge-request-comparison-check



  allow_failure:

    exit_codes:
@@ -371,12 +394,10 @@ pytest-on-merge-request: 
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - report-junit.xml

      - report-junit-evs.xml

    expose_as: 'pytest results'

    expose_as: 'pytest evs results'

    reports:

      junit:

        - report-junit.xml

        - report-junit-evs.xml