Commit 8613596d authored by norvell's avatar norvell
Browse files

Added job ivas-interop-on-merge-request

parent aeb35ece
Loading
Loading
Loading
Loading
Loading
+61 −2
Original line number Diff line number Diff line
@@ -731,7 +731,7 @@ ivas-pytest-on-merge-request:
    - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true
    - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true

    ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there
    ### If ref_using_main is not set, checkout the source branch to use scripts and input from there
    - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi

    ### prepare pytest
@@ -766,6 +766,65 @@ ivas-pytest-on-merge-request:
      junit:
        - report-junit.xml


# compare bit exactness between target and source branch
ivas-interop-on-merge-request:
  extends:
    - .test-job-linux
    - .rules-merge-request
  stage: compare
  needs: ["build-codec-linux-cmake", "codec-smoke-test"]
  timeout: "10 minutes"
  script:
    - *print-common-info
    - *get-commits-behind-count
    - *check-commits-behind-count-in-compare-jobs
    - *merge-request-comparison-setup-codec

    # some helper variables - "|| true" to prevent failures from grep not finding anything
    # write to temporary file as workaround for failures observed with piping echo
    - echo $CI_MERGE_REQUEST_TITLE > tmp.txt
    - non_interop_flag=$(grep -c --ignore-case "\[non[ -]*io\]" tmp.txt) || true
    #- ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true # Todo: What to do here?

    ### If ref_using_main is not set, checkout the source branch to use scripts and input from there
    #- if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi

    ### prepare pytest
    # create short test vectors
    - python3 tests/create_short_testvectors.py

    # Run reference creation, using source branch encoder and main decoder (see .merge-request-comparison-setup-codec)
    - exit_code=0
    - exit_code2=0
    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod_test --ref_decoder_path ./IVAS_dec_ref || exit_code=$?
    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report2.html --self-contained-html --junit-xml=report2-junit.xml --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod_test --ref_decoder_path ./IVAS_dec_ref || exit_code2=$?
    - zero_failures=$(cat report-junit.xml report2-junit.xml | grep -c 'failures="0"') || true

    # Todo: I suppose only one junit report can be listed. Select the one with potential failures.
    - if [ $exit_code -eq 1 ]; then cp report-junit.xml report-junit-out.xml; else report2-junit.xml report-junit-out.xml

    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi
    - exit 0

  allow_failure:
    exit_codes:
      - 123
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    paths:
      - report-junit.xml
      - report.html
      - report2-junit.xml
      - report2.html
    expose_as: "interop test results"
    reports:
      junit:
        - report-junit-out.xml

evs-pytest-on-merge-request:
  extends:
    - .test-job-linux