Commit fea98ccb authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into pytest-parallel

parents 7a08a693 dc159dde

includes/default-variables.yml

+3 −0
Original line number Diff line number Diff line
@@ -12,3 +12,6 @@ variables: 
  BASOP_REFERENCE_BRANCH: "ivas-float-update"

  SCALE_FACTOR: "3.162"

  BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch"

  PYTEST_ADDOPTS: ""

  # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases

  DISABLE_HRTF: "false"

main-basop.yml

+174 −25
Original line number Diff line number Diff line
@@ -76,6 +76,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough'

      variables:

        IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

      variables:

        IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
@@ -239,7 +242,7 @@ workflow: 
.ivas-pytest-anchor: &ivas-pytest-anchor

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "360 minutes"

  timeout: "480 minutes"

  variables:

    # keep "mld" in artifact name for backwards compatibility reasons

    CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"
@@ -374,6 +377,9 @@ workflow: 
  before_script:

    - !reference [ .test-job-linux, before_script ]

    - rm -rf tests/dut tests/ref

    - if [ "$DISABLE_HRTF" = "true" ]; then

    -   export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'"

    - fi

  variables:

    USE_LTV: 0
@@ -661,7 +667,7 @@ workflow: 
    - .job-linux

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "600 minutes"

  timeout: "12 hours"

  rules:

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never
@@ -1217,8 +1223,7 @@ renderer-pytest-on-merge-request: 
    - .test-job-linux-needs-testv-dir

    - .test-job-linux-compares-to-fixed-target

    - .rules-merge-request-no-draft

  # TODO: set reasonable timeout, will most likely take less

  timeout: "40 minutes"

  timeout: "90 minutes"

  tags:

    - ivas-basop-linux

  stage: compare
@@ -1317,6 +1322,10 @@ ivas-pytest-on-merge-request: 
    ### If ref_using_target is not set, checkout the source branch to use scripts and input from there

    - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi



    - if [ "$DISABLE_HRTF" = "true" ]; then

    -   export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'"

    - fi



    ### prepare pytest

    # create references

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1
@@ -2138,6 +2147,101 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: 
    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor



# Manual long self-test (not including tests_sba.py) against BASOP main

test-long-self-test:

  extends:

    - .job-linux

  stage: compare

  timeout: "360 minutes"

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

  allow_failure:

    exit_codes:

      - 123

  parallel:

    matrix:

      - LEVEL: 

          - 0dB

          - minus10dB

          - plus10dB

  script:

    - set -euxo pipefail

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh

    - testcase_timeout=$TESTCASE_TIMEOUT_LTV



    - case $LEVEL in 

        "0dB")

          LEVEL_SCALING=1.0

        ;;

        "minus10dB")

          LEVEL_SCALING=0.3162

        ;;

        "plus10dB")

          LEVEL_SCALING=3.162

        ;;

      esac



    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - if [ $LEVEL_SCALING != "1.0" ]; then

    -   bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh

    - fi



    - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")

    - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"



    ### store the current commit hash

    - source_branch_commit_sha=$(git rev-parse HEAD)



    ### switch to main

    - git checkout main

    - git pull origin main

    - echo "Building reference codec at commit $(git rev-parse HEAD)"



    ### build main (ref) binaries

    - make clean

    - make -j

    - mv ./IVAS_cod ./IVAS_cod_ref

    - mv ./IVAS_dec ./IVAS_dec_ref



    ### Switch back to branch and rename binaries back

    - git checkout $source_branch_commit_sha

    - echo "Building test codec at commit $source_branch_commit_sha"



    ### build branch binaries

    - make clean

    - make -j





    ### prepare pytest



    # create references

    - exit_code_ref=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$?



    ### run pytest self-test using long test vectors

    - exit_code=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi

    - exit 0





  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: 4 mos

    paths:

      - report-junit-ltv.xml

      - report-ltv.html

    expose_as: "test-long-self-test results"

    reports:

      junit:

        - report-junit-ltv.xml



ivas-smoke-test-saturation:

  extends:
@@ -2192,48 +2296,34 @@ coverage-test-on-main-scheduled: 
  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref

    # Build DuT binaries with GCOV

    - make clean >> /dev/null

    - make GCOV=1 -j

    - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in the first pytest call

    - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in ref creation call



    - testcase_timeout=$TESTCASE_TIMEOUT_LTV

    - exit_code_dec=0

    - exit_code_enc=0

    - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report-dec.html --self-contained-html --junit-xml=report-junit-dec.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path IVAS_dec || exit_code_dec=$?

    - lcov -c -d obj -o coverage_dec_rend.info # extract coverage of decoder/renderer



    - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection --encoder_only -v  --html=report-enc.html --self-contained-html --junit-xml=report-junit-enc.xml -n auto --testcase_timeout $testcase_timeout --dut_encoder_path IVAS_dec || exit_code_enc=$?

    - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage of encoder/decoder/renderer

    - exit_code=0

    - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$?

    - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage



    # remove apps and lib_util files from coverage

    - lcov -r coverage_dec_rend.info "*apps*" -o coverage_dec_rend.info

    - lcov -r coverage_dec_rend.info "*lib_util*" -o coverage_dec_rend.info

    - lcov -r coverage_enc_dec_rend.info "*apps*" -o coverage_enc_dec_rend.info

    - lcov -r coverage_enc_dec_rend.info "*lib_util*" -o coverage_enc_dec_rend.info



    - commit_sha=$(git rev-parse HEAD)

    - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main enc/dec/rend @ $commit_sha"

    - genhtml coverage_dec_rend.info -o coverage_dec_rend -t "Coverage on main -- dec/rend @ $commit_sha"

    - genhtml coverage_enc_dec_rend.info -o coverage_enc_dec_rend -t "Coverage on main @ $commit_sha"

  artifacts:

    name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"

    when: always

    expire_in: 1 week

    paths:

      - coverage_enc_dec_rend.info

      - coverage_dec_rend.info

      - coverage_enc_dec_rend

      - coverage_dec_rend

      - report-dec.html

      - report-enc.html

      - report.html

    expose_as: "Coverage result"

    reports:

      junit:

        - report-junit-dec.xml

        - report-junit-enc.xml

        - report-junit.xml



# ---------------------------------------------------------------

# EVS 26.444 test job
@@ -2458,6 +2548,65 @@ ivas-long-term-job-logs: 
      - logs

    expose_as: "ivas long term job logs results"



ivas-long-term-job-logs-overview:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs"

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs"

  tags:

    - has-plotly

  needs:

    - ivas-long-term-job-logs

  stage: maintenance

  timeout: "25 minutes"

  script:

    - !reference [ .job-linux, before_script ]

    - set -euxo pipefail

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

    # Create overview page from logs

    - python3 ci/process_long_term_logs.py logs long_term_regression.html

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 4 weeks

    when: always

    paths:

      - long_term_regression.html

    expose_as: "ivas long term job logs overview"



ivas-long-term-regressions:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs"

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs"

  needs:

    - ivas-long-term-job-logs

  stage: maintenance

  timeout: "25 minutes"

  script:

    - !reference [ .job-linux, before_script ]

    - set -euxo pipefail

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh



    # Find regressions

    - for MEASURE in MLD MAX_ABS_DIFF; do

    -   python3 scripts/find_regressions_from_logs.py logs regressions_"$MEASURE".csv  --measure "$MEASURE" --days 7 --ratio_thr 1.1

    -   mkdir regressions_"$MEASURE"

    -   python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv

    -   mv regression*bash regressions_"$MEASURE"

    - done





  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 4 weeks

    when: always

    paths:

      - regressions_MLD.csv

      - regressions_MAX_ABS_DIFF.csv

      - regressions_MLD

      - regressions_MAX_ABS_DIFF

    expose_as: "ivas long term regressions"



# To store backup copy:

# - Prepare accessible folder for backup, e.g. /usr/local/backup

# - Set MANUAL_PIPELINE_TRIGGER to backup-long-term-logs