Add draft job pytest-compare-to-input-short-dec

      - 'default'

      - 'pytest-compare'

      - 'pytest-compare-long'

      - 'pytest-compare-to-input'

      - 'pytest-saturation-smoke-test'

      - 'evs-26444'

      - 'sanitizer-stv'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against input ref (pass-through only): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test'

      variables:

        IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never

.rules-pytest-to-input-short:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-to-input"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never

# TODO: only temporary as long the MR encoder tests should not compare to main

.rules-pytest-to-ref-enc-short-temp:

  rules:

      junit:

        - report-junit.xml

.ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "240 minutes"

  variables:

  script:

    - *print-common-info

    - *update-scripts-repo

    - if [ $USE_LTV -eq 1 ]; then

    -    *update-ltv-repo

    -    *copy-ltv-files-to-testv-dir

    -    testcase_timeout=$TESTCASE_TIMEOUT_LTV

    - else

    -    testcase_timeout=$TESTCASE_TIMEOUT_STV

    - fi

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - if [ $LEVEL_SCALING != "1.0" ];then

    -   *apply-testv-scaling

    - fi

    - *build-reference-and-dut-binaries

    ### run pytest

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH  --compare_to_input || exit_code=$?

    - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true

    - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv

    - python3 scripts/parse_xml_report.py report-junit-dut.xml report-dut.csv

    # - mkdir $IMAGES_ARTIFACT_NAME

    # - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done

    # - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME

    # - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

    # -    id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)

    # -    echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous"

    # -    curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

    # -    unzip artifacts.zip -d previous_artifacts

    # # This wildcard thingy relies on only one csv file being present per job

    # -    file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv"

    # -    python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME

    # - else

    # # create empty file for artifacts to avoid errors

    # -    touch $PAGES_HTML_ARTIFACT_NAME

    # -    touch $MERGED_CSV_ARTIFACT_NAME

    # - fi

    # - if [ $zero_errors != 1 ]; then

    # -     echo "Run errors encountered!"

    # # TODO: temporary only to not fail MR pipelines on crashes

    # -     if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then

    # -         exit $EXIT_CODE_NON_BE

    # -     else

    # -         exit $EXIT_CODE_FAIL

    # -     fi

    # - fi

    # - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi

    - exit 0

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit-ref.xml

      - report-ref.html

      - report-ref.csv

      - report-junit-dut.xml

      - report-dut.html

      - report-dut.csv

    expose_as: "pytest compare to input results"

    reports:

      junit:

        - report-junit-ref.xml

        - report-junit-dut.xml

# ---------------------------------------------------------------

# .pre jobs for setting up things

# ---------------------------------------------------------------

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-anchor

ivas-pytest-compare-to-input-short-dec:

  extends:

    - .rules-pytest-to-input-short

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-compare-to-input-anchor

ivas-pytest-dec-msan-short:

  extends:

    - .test-job-linux