Commit d68051e4 authored by Archit Tamarapu's avatar Archit Tamarapu
Browse files

split up coverage into two jobs + light formatting

parent 57bb26d3

.gitlab-ci.yml

+128 −125
Original line number Diff line number Diff line
@@ -11,43 +11,43 @@ include: 


# This sets when pipelines are created. Jobs have more specific rules to restrict them.

workflow:

  name: '$IVAS_PIPELINE_NAME'

  name: "$IVAS_PIPELINE_NAME"

  rules:

    # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      variables:

        IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'

        IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

      variables:

        IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Push pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing

      variables:

        IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'

      variables:

        IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Test BE to release pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

      variables:

        IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance'

      variables:

        IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: "Draft IVAS Conformance test: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'trigger'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

      variables:

        IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: "Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'trigger'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping'

      variables:

        IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Check core input clipping: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

      variables:

        IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH"



# ---------------------------------------------------------------

# Generic script anchors
@@ -71,8 +71,8 @@ workflow: 
  - cp ivas-basop/IVAS_cod ./

  - cp ivas-basop/IVAS_dec ./



.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec 

  ### build test binaries, initial clean for paranoia reasons

.merge-request-comparison-setup-codec:

  &merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons

  - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh

  - make clean

  - mkdir build
@@ -160,7 +160,6 @@ workflow: 


    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi





# ---------------------------------------------------------------

# .pre jobs for setting up things

# ---------------------------------------------------------------
@@ -359,7 +358,6 @@ build-codec-instrumented-linux: 
    paths:

      - wmc_tool_output.txt





# make sure that the codec builds with msan, asan and usan

build-codec-sanitizers-linux:

  extends:
@@ -402,7 +400,13 @@ codec-smoke-test: 
  tags:

    - ivas-linux-fast

  stage: test

  needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"]

  needs:

    [

      "build-codec-linux-cmake",

      "build-codec-linux-make",

      "build-codec-instrumented-linux",

      "build-codec-sanitizers-linux",

    ]

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    # LTV update needed as ltv ISM metadata files are used
@@ -481,7 +485,13 @@ pytest-compare-20ms-and-5ms-rendering: 
    - .test-job-linux

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"]

  needs:

    [

      "build-codec-linux-cmake",

      "build-codec-linux-make",

      "build-codec-instrumented-linux",

      "build-codec-sanitizers-linux",

    ]

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh
@@ -583,7 +593,6 @@ renderer-asan: 
  artifacts:

    expose_as: "renderer asan result"





# test renderer executable with cmake + msan

renderer-msan:

  extends:
@@ -595,7 +604,6 @@ renderer-msan: 
  artifacts:

    expose_as: "renderer msan result"





# test renderer executable with cmake + usan

renderer-usan:

  extends:
@@ -607,7 +615,6 @@ renderer-usan: 
  artifacts:

    expose_as: "renderer usan result"





# compare renderer bitexactness between target and source branch

renderer-pytest-on-merge-request:

  extends:
@@ -846,7 +853,6 @@ ivas-pytest-on-merge-request: 
      junit:

        - report-junit.xml





# Check interop IVAS_cod_test -> IVAS_dec_ref

ivas-interop-on-merge-request:

  extends:
@@ -1093,7 +1099,6 @@ check-bitexactness-ext-and-transport-format: 
    expose_as: "logs-ext-sanity-check"

    expire_in: "5 days"





# ---------------------------------------------------------------

# Test jobs for main branch

# ---------------------------------------------------------------
@@ -1230,7 +1235,6 @@ codec-comparison-on-main-push: 
    reports:

      junit: report-junit.xml





# ---------------------------------------------------------------

# Manual jobs

# ---------------------------------------------------------------
@@ -1244,7 +1248,6 @@ test-be-to-release: 
  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'

  script:



    - echo "$CI_COMMIT_BRANCH"

    - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata
@@ -1261,7 +1264,6 @@ test-be-to-release: 
      - logs/

    expose_as: "test-be-to-release results"





ivas-conformance:

  tags:

    - ivas-windows
@@ -1333,7 +1335,6 @@ ivas-conformance: 
    - cp IVAS_rend.exe testvec/bin

    - cp ISAR_post_rend.exe testvec/bin



    

    # Test run generated scripts in testvec

    - cd testvec

    - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html
@@ -1465,7 +1466,6 @@ ivas-conformance-linux: 
    reports:

      junit: report-junit.xml





test-long-self-test:

  extends:

    - .job-linux
@@ -1523,7 +1523,6 @@ test-long-self-test: 
    - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi

    - exit 0





  artifacts:

    name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always
@@ -1972,7 +1971,6 @@ sanitizer-test-omasa-ism4: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS





### --- sanitizer schedule C ---



.sanitizer-test-schedule-C:
@@ -2185,10 +2183,8 @@ sanitizer-test-osba-planar-hoa3-ism4: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS







# GCOV/LCOV coverage analysis of self_test suite

coverage-test-on-main-scheduled:

.coverage-test-template:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-main-scheduled
@@ -2198,10 +2194,10 @@ coverage-test-on-main-scheduled: 
  rules:

    # only run in scheduled pipeline that passes this env vars

    - if: $COVERAGE_TEST

  timeout: 12 hours

  script:

  timeout: 3 hours

  before_script:

    - set -e

    - "trap 'echo \"Command failed at line $LINENO: $BASH_COMMAND\"' ERR"

    - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR'

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh
@@ -2214,7 +2210,25 @@ coverage-test-on-main-scheduled: 
    - cp IVAS_dec IVAS_dec_ref

    - cp IVAS_rend IVAS_rend_ref

    - cp ISAR_post_rend ISAR_post_rend_ref

  after_script:

    - lcov -c -d cmake-build -o $COVERAGE_OUTPUT_FILE

    # remove apps and lib_util files from coverage

    - lcov -r coverage.info "*apps*" -o $COVERAGE_OUTPUT_FILE

    - lcov -r coverage.info "*lib_util*" -o $COVERAGE_OUTPUT_FILE

    - commit_sha=$(git rev-parse HEAD)

    - genhtml $COVERAGE_OUTPUT_FILE -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha"

  artifacts:

    name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"

    when: always

    expire_in: 1 week



coverage-test-on-main-scheduled-stv:

  extends: .coverage-test-template

  variables:

    COVERAGE_OUTPUT_FILE: "coverage_stv.info"

    COVERAGE_OUTPUT_DIR: "coverage_stv"

    COVERAGE_TITLE: "short test vectors"

  script:

    # -- Run short test vector suite to check coverage of that first

    # codec smoke test

    - bash ci/smoke_test.sh coverage
@@ -2245,17 +2259,20 @@ coverage-test-on-main-scheduled: 
    # run LC3plus wrapper unit test

    - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test

    # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job

  artifacts:

    paths:

      - coverage_stv.info

      - coverage_stv



    # -- Extract coverage of short testvectors

    - lcov -c -d cmake-build -o coverage_stv.info

    # -- Remove apps and lib_util files from coverage

    - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info

    - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info



coverage-test-on-main-scheduled-ltv:

  extends: .coverage-test-template

  variables:

    COVERAGE_OUTPUT_FILE: "coverage_ltv.info"

    COVERAGE_OUTPUT_DIR: "coverage_ltv"

    COVERAGE_TITLE: "long test vectors"

  script:

    # -- Add extended coverage of EVS test and long test vectors

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # codec smoke test

    - bash ci/smoke_test.sh coverage

    # default renderer framesize is 20ms

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only
@@ -2272,24 +2289,10 @@ coverage-test-on-main-scheduled: 
    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv --create_ref

    # split rendering test doesn't support LTV - skipped here

    # VoIP BE tests don't support LTV - skipped here



    # -- Extract accumulated coverage of STV + EVS + LTV

    - lcov -c -d cmake-build -o coverage.info

    # remove apps and lib_util files from coverage

    - lcov -r coverage.info "*apps*" -o coverage.info

    - lcov -r coverage.info "*lib_util*" -o coverage.info

    - commit_sha=$(git rev-parse HEAD)

    - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha"

    - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha"

  artifacts:

    name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"

    when: always

    expire_in: 1 week

    paths:

      - coverage.info

      - coverage_stv.info

      - coverage

      - coverage_stv

      - coverage_ltv.info

      - coverage_ltv



# ---------------------------------------------------------------

# Complexity measurement jobs