Test run of long self test

    file: main-basop.yml

  - local: .gitlab-ci/variables.yml

  - local: .gitlab-ci/rules-basis.yml

workflow:

  name: '$IVAS_PIPELINE_NAME'

  rules:

    # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      variables:

        IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'

    ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow

    # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    #   variables:

    #     IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing

      variables:

        IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs'

      variables:

        IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs'

      variables:

        IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx'

      variables:

        IVAS_PIPELINE_NAME: 'Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long-fx-fx'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test'

      variables:

        IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444'

      variables:

        IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer'

      variables:

        IVAS_PIPELINE_NAME: 'Short testvectors sanitizers'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity'

      variables:

        IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test'

      variables:

        IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'renderer-framesize-be'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer framesize BE test on $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough'

      variables:

        IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'        

test-long-self-test:

  extends:

    - .job-linux

  stage: compare

  resource_group: ivas-long-self-test-resource

  timeout: "300 minutes"

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

  allow_failure:

    exit_codes:

      - 123

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    ### store the current commit hash

    - source_branch_commit_sha=$(git rev-parse HEAD)

    ### switch to main

    - git checkout main

    - git pull origin main

    - echo "Building reference codec at commit $(git rev-parse HEAD)"

    ### build main (ref) binaries

    - make clean

    - make -j

    - mv ./IVAS_cod ./IVAS_cod_ref

    - mv ./IVAS_dec ./IVAS_dec_ref

    ### Switch back to branch and rename binaries back

    - git checkout $source_branch_commit_sha

    - echo "Building test codec at commit $source_branch_commit_sha"

    ### build branch binaries

    - make clean

    - make -j

    ### prepare pytest

    # Copy test vectors from LTV to TESTV

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh

    # create references

    - exit_code_ref=0

    - testcase_timeout=900

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$?

    ### run pytest self-test using long test vectors

    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi

    - exit 0

  artifacts:

    name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: 4 mos

    paths:

      - report-junit-ltv.xml

      - report-ltv.html

    expose_as: "test-long-self-test results"

    reports:

      junit:

        - report-junit-ltv.xml