Commit 72a3e5dd authored by sagnowski's avatar sagnowski
Browse files

Merge branch 'main' into update-wav-diff-v0_4_0

parents 39422bbb 8682b5a1

.gitlab-ci.yml

+208 −127
Original line number Diff line number Diff line
@@ -11,43 +11,46 @@ include: 


# This sets when pipelines are created. Jobs have more specific rules to restrict them.

workflow:

  name: '$IVAS_PIPELINE_NAME'

  name: "$IVAS_PIPELINE_NAME"

  rules:

    # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      variables:

        IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'

        IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

      variables:

        IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Push pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing

      variables:

        IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'

      variables:

        IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Test BE to release pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

      variables:

        IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance'

      variables:

        IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: "Draft IVAS Conformance test: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'trigger'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

      variables:

        IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: "Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'trigger'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping'

      variables:

        IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Check core input clipping: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

      variables:

        IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Pass-through comparison vs input: $CI_COMMIT_BRANCH"

    - if: $MANUAL_PIPELINE_TYPE == 'coverage'

      variables:

        IVAS_PIPELINE_NAME: "Coverage measurement"



# ---------------------------------------------------------------

# Generic script anchors
@@ -160,7 +163,6 @@ workflow: 


    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi





# ---------------------------------------------------------------

# .pre jobs for setting up things

# ---------------------------------------------------------------
@@ -359,7 +361,6 @@ build-codec-instrumented-linux: 
    paths:

      - wmc_tool_output.txt





# make sure that the codec builds with msan, asan and usan

build-codec-sanitizers-linux:

  extends:
@@ -402,7 +403,13 @@ codec-smoke-test: 
  tags:

    - ivas-linux-fast

  stage: test

  needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"]

  needs:

    [

      "build-codec-linux-cmake",

      "build-codec-linux-make",

      "build-codec-instrumented-linux",

      "build-codec-sanitizers-linux",

    ]

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    # LTV update needed as ltv ISM metadata files are used
@@ -481,7 +488,13 @@ pytest-compare-20ms-and-5ms-rendering: 
    - .test-job-linux

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"]

  needs:

    [

      "build-codec-linux-cmake",

      "build-codec-linux-make",

      "build-codec-instrumented-linux",

      "build-codec-sanitizers-linux",

    ]

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh
@@ -577,7 +590,6 @@ renderer-asan: 
  artifacts:

    expose_as: "renderer asan result"





# test renderer executable with cmake + msan

renderer-msan:

  extends:
@@ -589,7 +601,6 @@ renderer-msan: 
  artifacts:

    expose_as: "renderer msan result"





# test renderer executable with cmake + usan

renderer-usan:

  extends:
@@ -601,7 +612,6 @@ renderer-usan: 
  artifacts:

    expose_as: "renderer usan result"





# compare renderer bitexactness between target and source branch

renderer-pytest-on-merge-request:

  extends:
@@ -840,7 +850,6 @@ ivas-pytest-on-merge-request: 
      junit:

        - report-junit.xml





# Check interop IVAS_cod_test -> IVAS_dec_ref

ivas-interop-on-merge-request:

  extends:
@@ -1087,7 +1096,6 @@ check-bitexactness-ext-and-transport-format: 
    expose_as: "logs-ext-sanity-check"

    expire_in: "5 days"





# ---------------------------------------------------------------

# Test jobs for main branch

# ---------------------------------------------------------------
@@ -1226,7 +1234,6 @@ be-2-evs-windows: 
#     reports:

#       junit: report-junit.xml





# ---------------------------------------------------------------

# Manual jobs

# ---------------------------------------------------------------
@@ -1240,7 +1247,6 @@ test-be-to-release: 
  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'

  script:



    - echo "$CI_COMMIT_BRANCH"

    - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m | tee -variable winoutdata
@@ -1257,7 +1263,6 @@ test-be-to-release: 
      - logs/

    expose_as: "test-be-to-release results"





ivas-conformance:

  tags:

    - ivas-windows
@@ -1329,7 +1334,6 @@ ivas-conformance: 
    - cp IVAS_rend.exe testvec/bin

    - cp ISAR_post_rend.exe testvec/bin



    

    # Test run generated scripts in testvec

    - cd testvec

    - python -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html
@@ -1361,9 +1365,15 @@ ivas-conformance-linux: 
  timeout: "90 minutes"

  rules:

    - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

    - if: $MANUAL_PIPELINE_TYPE == "coverage"

    - if: $COVERAGE_TEST

  allow_failure:

    exit_codes:

      - 123

  variables:

    COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_CONFORMANCE"

    COVERAGE_OUTPUT_DIR: "coverage_conformance"

    COVERAGE_TITLE: "conformance test test_26252.py"

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - make -j
@@ -1375,10 +1385,10 @@ ivas-conformance-linux: 
    # Reference creation

    - python3 scripts/prepare_combined_format_inputs.py

    - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py"

    - python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files

    - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files



    # Output creation

    - python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html

    - python3 -m pytest -q $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html

    - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



    # Copy input data and output ref data
@@ -1426,17 +1436,13 @@ ivas-conformance-linux: 
    # Test run generated scripts in testvec

    - cd testvec

    - exit_code=0

    - python3 -m pytest conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$?

    - python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html || exit_code=$?

    - mv report.html ..

    - mv report-junit.xml ..



    # Collect coverage

    - cd -

    - lcov -c -d obj -o coverage.info

    - lcov -r coverage.info "*apps*" -o coverage.info

    - lcov -r coverage.info "*lib_util*" -o coverage.info

    - commit_sha=$(git rev-parse HEAD)

    - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha"

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE"



    # Check for failures

    - if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi
@@ -1455,13 +1461,12 @@ ivas-conformance-linux: 
      - Readme_IVAS_JBM_dec.txt

      - Readme_IVAS_ISAR_dec.txt

      - Readme_IVAS_ISAR_post_rend.txt

      - coverage.info

      - coverage

      - $COVERAGE_OUTPUT_FILE

      - $COVERAGE_OUTPUT_DIR

    expose_as: "Draft IVAS conformance -- Linux"

    reports:

      junit: report-junit.xml





test-long-self-test:

  extends:

    - .job-linux
@@ -1519,7 +1524,6 @@ test-long-self-test: 
    - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi

    - exit 0





  artifacts:

    name: "test-long-self-test--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always
@@ -1968,7 +1972,6 @@ sanitizer-test-omasa-ism4: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - python3 ci/run_scheduled_sanitizer_test.py MASA-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS





### --- sanitizer schedule C ---



.sanitizer-test-schedule-C:
@@ -2181,10 +2184,8 @@ sanitizer-test-osba-planar-hoa3-ism4: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3-ISM4 $OUT_FORMATS_ALL --tests $SANITIZER_TESTS







# GCOV/LCOV coverage analysis of self_test suite

coverage-test-on-main-scheduled:

.coverage-test-template:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-main-scheduled
@@ -2192,53 +2193,133 @@ coverage-test-on-main-scheduled: 
    - ivas-linux-fast

  stage: test

  rules:

    # only run in scheduled pipeline that passes this env vars

    - if: $COVERAGE_TEST

    - if: $MANUAL_PIPELINE_TYPE == "coverage"

  timeout: 3 hours

  script:

  before_script:

    - !reference [.job-linux, before_script]

    - set -e

    - 'trap ''echo "Command failed at line $LINENO: $BASH_COMMAND"'' ERR'

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh

    - make GCOV=1 -j

    - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script

    # compile with coverage enabled

    - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true -DGCOV=ON

    - cmake --build cmake-build -- -j

    # copy executables to ref to be able to run some --create_ref tests

    - cp IVAS_cod IVAS_cod_ref

    - cp IVAS_dec IVAS_dec_ref

    - cp IVAS_rend IVAS_rend_ref

    - cp ISAR_post_rend ISAR_post_rend_ref

  after_script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir cmake-build --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE"

  artifacts:

    name: "main-$COVERAGE_OUTPUT_DIR-sha-$CI_COMMIT_SHORT_SHA"

    when: always

    expire_in: 1 week



    # -- Run short test vector suite to check coverage of that first

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

coverage-merge:

  extends: .rules-main-scheduled

  tags:

    - ivas-linux-fast

  stage: test

  rules:

    - if: $COVERAGE_TEST

    - if: $MANUAL_PIPELINE_TYPE == "coverage"

  needs:

    - job: coverage-test-on-main-scheduled-stv

      artifacts: true

    - job: coverage-test-on-main-scheduled-ltv

      artifacts: true

    - job: ivas-conformance-linux

      artifacts: true

  variables:

    COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_MERGED"

    COVERAGE_OUTPUT_DIR: "coverage-merged"

    COVERAGE_TITLE: "merged total coverage"

  script:

    - echo "Merging coverage reports..."

    - lcov -a $COVERAGE_OUTPUT_FILE_CONFORMANCE -a $COVERAGE_OUTPUT_FILE_STV -a $COVERAGE_OUTPUT_FILE_LTV -o "$COVERAGE_OUTPUT_FILE_MERGED"

    - commit_sha=$(git rev-parse HEAD)

    - genhtml $COVERAGE_OUTPUT_FILE_MERGED -o $COVERAGE_OUTPUT_DIR -t "Coverage on main -- $COVERAGE_TITLE @ $commit_sha"

  artifacts:

    name: "merged-coverage-sha-$CI_COMMIT_SHORT_SHA"

    when: always

    expire_in: 1 week

    paths:

      - $COVERAGE_OUTPUT_FILE

      - $COVERAGE_OUTPUT_DIR



coverage-test-on-main-scheduled-stv:

  extends: .coverage-test-template

  variables:

    COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_STV"

    COVERAGE_OUTPUT_DIR: "coverage_stv"

    COVERAGE_TITLE: "short test vectors"

  script:

    # -- Run short test vector suite to check coverage of that first

    # codec smoke test

    - bash ci/smoke_test.sh coverage

    # default renderer framesize is 20ms

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true

    - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref

    - python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut    

    - lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here

    # remove apps and lib_util files from coverage

    - lcov -r coverage_stv.info "*apps*" -o coverage_stv.info

    - lcov -r coverage_stv.info "*lib_util*" -o coverage_stv.info

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true

    # run renderer smoke test - needed for REFVEC tests

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py

    # run renderer test only with REF creation stage - no CUT needed since executables are identical

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --create_ref

    # run split rendering smoke test and REF creation

    - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py

    - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_split_rendering.py --create_ref

    # VoIP BE test

    - python3 -m pytest -q --tb=no tests/test_be_for_jbm_neutral_dly_profile.py

    # split rendering VoIP BE test

    - python3 -m pytest -n auto -q --tb=no tests/split_rendering/test_voip_be_splitrend_vs_binaural.py

    # TODO first frame SID - this script runs a `make clean` and needs to be checked/adapted

    # - bash ci/run-first-frame-is-sid-test.sh

    # TODO consider HRTF binary loading test, or probably already covered by PRM files

    # - python3 -m pytest -q --tb=no tests/hrtf_binary_loading

    # run BE vs EXT test

    - python3 tests/create_short_testvectors.py --cut_len 1.0

    - python3 -m pytest -q --tb=no tests/test_be_for_ext_outputs.py

    # run LC3plus wrapper unit test

    - scripts/split_rendering/lc3plus_float/ivas_lc3plus_unit_test

    # TODO consider including check-clipping, but this scales ALL testv files. could be added at end of this job

  artifacts:

    paths:

      - $COVERAGE_OUTPUT_FILE

      - $COVERAGE_OUTPUT_DIR



coverage-test-on-main-scheduled-ltv:

  extends: .coverage-test-template

  variables:

    COVERAGE_OUTPUT_FILE: "$COVERAGE_OUTPUT_FILE_LTV"

    COVERAGE_OUTPUT_DIR: "coverage_ltv"

    COVERAGE_TITLE: "long test vectors"

  script:

    # -- Add extended coverage of EVS test and long test vectors

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - bash ci/smoke_test.sh coverage

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true    

    - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py

    - lcov -c -d obj -o coverage.info

    # remove apps and lib_util files from coverage

    - lcov -r coverage.info "*apps*" -o coverage.info

    - lcov -r coverage.info "*lib_util*" -o coverage.info

    - commit_sha=$(git rev-parse HEAD)

    - genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha"

    - genhtml coverage_stv.info -o coverage_stv -t "Coverage on main -- short test vectors @ $commit_sha"

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # default renderer framesize is 20ms

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true

    # run renderer smoke test - needed for REFVEC tests

    - ls -altr scripts/testv

    # LTV smoketest is the same as REF creation stage for renderer

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py --use_ltv

    # split rendering test doesn't support LTV - skipped here

    # VoIP BE tests don't support LTV - skipped here

  artifacts:

    name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"

    when: always

    expire_in: 1 week

    paths:

      - coverage.info

      - coverage_stv.info

      - coverage

      - coverage_stv

      - $COVERAGE_OUTPUT_FILE

      - $COVERAGE_OUTPUT_DIR



# ---------------------------------------------------------------

# Complexity measurement jobs

.gitlab-ci/variables.yml

+5 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ variables: 
      - 'ivas-conformance-linux'

      - 'check-clipping'

      - 'test-branch-vs-input-passthrough'

      - 'coverage'



  PYTEST_ARGS: ""

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv"
@@ -28,3 +29,7 @@ variables: 
  TEST_SUITE: ""

  # note: currently overwrites default value from ci repo

  TESTCASE_TIMEOUT_STV_SANITIZERS: 240

  COVERAGE_OUTPUT_FILE_STV: "coverage-stv.info"

  COVERAGE_OUTPUT_FILE_LTV: "coverage-ltv.info"

  COVERAGE_OUTPUT_FILE_CONFORMANCE: "coverage-conformance.info"

  COVERAGE_OUTPUT_FILE_MERGED: "coverage-merged.info"

ci/index-pages.html

+1 −4
Original line number Diff line number Diff line
@@ -11,9 +11,6 @@ 


  <h2>Test Coverage</h2>



  <ul>

    <li><a href="coverage/index.html">Coverage report</a></li>

    <li><a href="coverage_stv/index.html">Coverage of short test vectors</a></li>

  </ul>

  {}



</body>

ci/setup_pages.py

+27 −8
Original line number Diff line number Diff line
@@ -40,7 +40,10 @@ JOBS_FLOAT_REPO = { 
    "complexity-osba-in-binaural_room_ir-out": "OSBA in, BINAURAL_ROOM_IR out",

    "complexity-StereoDmxEVS-stereo-in-mono-out": "StereoDmxEVS, Stereo in, Mono out",

    # "timeless" jobs (not complexity)

    "coverage-test-on-main-scheduled": "Coverage",

    "coverage-test-on-main-scheduled-stv": "Short test vector coverage (~ merge request pipelines)",

    "coverage-test-on-main-scheduled-ltv": "Long test vector coverage (for test_param_file.py, test_sba.py and test_renderer.py)",

    "ivas-conformance-linux": "Conformance test coverage",

    "coverage-merge": "Merged coverage (all of the above combined)",

}

JOBS_BASOP_REPO = {

    "ivas-pytest-compare_ref-long-dec": "Pytest decoder compare to ref LTV",
@@ -77,6 +80,13 @@ JOBS_FOR_PROJECT_ID = { 
    PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO,

}



ARTIFACT_FOLDER_4_COVERAGE_JOBS = {

    "coverage-test-on-main-scheduled-stv": "coverage_stv",

    "coverage-test-on-main-scheduled-ltv": "coverage_ltv",

    "ivas-conformance-linux": "coverage_conformance",

    "coverage-merge": "coverage-merged",

}



ARTIFACTS = "artifacts.zip"

API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs"

PUBLIC_FOLDER = pathlib.Path("./public").absolute()
@@ -104,15 +114,20 @@ def main(): 


def create_landing_page(jobs, index_html, project_id):

    # dynamically create the complexity links on the landing page

    link_html = list()

    link_html = ["<ul>"]

    link_html_complexity = ["<ul>"]

    link_html_coverage = ["<ul>"]



    for job, link_text in jobs.items():

        if job.startswith("complexity"):

            line = f'<li><a href="{job}-public/index.html">{link_text}</a></li>'

            link_html.append(line)

    link_html.append("</ul>")

    link_html_text = "\n".join(link_html)

            link_html_complexity.append(line)

        elif job.startswith("coverage") or job == "ivas-conformance-linux":

            artifact_folder = ARTIFACT_FOLDER_4_COVERAGE_JOBS[job]

            line = f'<li><a href="{artifact_folder}/index.html">{link_text}</a></li>'

            link_html_coverage.append(line)



    link_html_complexity.append("</ul>")

    link_html_coverage.append("</ul>")



    if project_id == PROJECT_ID_FLOAT_REPO:

        index_template = "index-pages.html"
@@ -126,14 +141,18 @@ def create_landing_page(jobs, index_html, project_id): 
    with open(index_pages_tmpl_path) as f:

        index_pages_tmpl = f.read()



    index_pages_tmpl = index_pages_tmpl.format(link_html_text)

    link_html_complexity_text = "\n".join(link_html_complexity)

    link_html_coverage_text = "\n".join(link_html_coverage)

    index_pages_tmpl = index_pages_tmpl.format(

        link_html_complexity_text, link_html_coverage_text

    )



    with open(index_html, "w") as f:

        f.write(index_pages_tmpl)





def get_artifacts_for_jobs_and_return_num_failed(

    jobs: list, project_id: int, success_only: bool

    jobs: dict, project_id: int, success_only: bool

) -> int:

    """

    Get specified artifact folders for all jobs given and put them into the public folder.

lib_com/options.h

+2 −1
Original line number Diff line number Diff line
@@ -173,6 +173,7 @@ 
#define FIX_1383_HEAD_TRACK_SANITIZER                   /* Nok: issue 1383: Fix head tracking struc values reading in renderer */

#define FIX_1385_INIT_IGF_STOP_FREQ                     /* FhG: Initialize infoIGFStopFreq in init_igf_dec() */

#define FIX_1387_INIT_PRM_SQQ                           /* FhG: initialize pointer prm_sqQ, which might be uninitialized in case of bfi == 1 */

#define FIX_1349_TNS_CRASH                              /* FhG: Fix crash in TNS entropy coding, in case order of joint TNS coding is reduced to 0 */





/* #################### End BE switches ################################## */
@@ -186,7 +187,7 @@ 
#define NONBE_1122_KEEP_EVS_MODE_UNCHANGED              /* FhG: Disables fix for issue 1122 in EVS mode to keep BE tests green. This switch should be removed once the 1122 fix is added to EVS via a CR.  */

#define NONBE_1328_FIX_NON_LINEARITY                    /* VA: Fix possible issue when computing bwe_exc_extended and previous frame were almost 0  */

#define NONBE_1344_REND_MASA_LOW_FS                     /* Nokia: Issue 1344: Fix sanitizer errors when using IVAS_rend to render MASA with lower sampling rates */



#define NONBE_1399_1400_FIX_OBJ_EDIT_ISSUES             /* Nokia: Fix for issues 1399: obj edit broken with MC/SBA output in VOIP, and 1400: negative energy estimate used for gaining. */



/* ##################### End NON-BE switches ########################### */