Commit 912d574f authored by norvell's avatar norvell
Browse files

Merge branch 'ci/compare-to-input' into 'main' 

[CI] compare to input

See merge request !651
parents 2470528b a5df33e3

.gitlab-ci.yml

+79 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ variables: 
      - 'default'

      - 'pytest-compare'

      - 'pytest-compare-long'

      - 'pytest-compare-to-input'

      - 'pytest-saturation-smoke-test'

      - 'evs-26444'

      - 'sanitizer-stv'
@@ -66,6 +67,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test'

      variables:

        IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH'
@@ -174,6 +178,14 @@ stages: 
    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never



.rules-pytest-to-input-short:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-to-input"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never



# TODO: only temporary as long the MR encoder tests should not compare to main

.rules-pytest-to-ref-enc-short-temp:

  rules:
@@ -471,6 +483,62 @@ stages: 
      junit:

        - report-junit.xml



.ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "240 minutes"

  variables:



  script:

    - *print-common-info

    - *update-scripts-repo

    - if [ $USE_LTV -eq 1 ]; then

    -    *update-ltv-repo

    -    *copy-ltv-files-to-testv-dir

    -    testcase_timeout=$TESTCASE_TIMEOUT_LTV

    - else

    -    testcase_timeout=$TESTCASE_TIMEOUT_STV

    - fi



    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - if [ $LEVEL_SCALING != "1.0" ];then

    -   *apply-testv-scaling

    - fi

    - *build-reference-and-dut-binaries



    ### run pytest

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH  --compare_to_input || exit_code=$?

    - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true

    - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv

    - python3 scripts/parse_xml_report.py report-junit-dut.xml report-dut.csv

    - python3 scripts/diff_report.py report-ref.csv report-dut.csv report-diff.csv



    - exit 0



  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit-ref.xml

      - report-ref.html

      - report-ref.csv

      - report-junit-dut.xml

      - report-dut.html

      - report-dut.csv

      - report-diff.csv

    expose_as: "pytest compare to input results"

    reports:

      junit:

        - report-junit-ref.xml

        - report-junit-dut.xml



# ---------------------------------------------------------------

# .pre jobs for setting up things

# ---------------------------------------------------------------
@@ -762,6 +830,17 @@ ivas-pytest-compare_to_ref-short-dec-lev+10: 
    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-anchor



ivas-pytest-compare-to-input-short-dec:

  extends:

    - .rules-pytest-to-input-short

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-compare-to-input-anchor



ivas-pytest-dec-msan-short:

  extends:

    - .test-job-linux