Commit c517b735 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'mullerfa/basop-smoke-test-fixes' into 'basop-ci-branch' 

[CI] Allow filtering out known-broken smoke test items in BASOP repository

See merge request !1982
parents cbbd1ec2 00d708ce

.gitlab-ci.yml

+63 −38
Original line number Diff line number Diff line
@@ -132,6 +132,12 @@ stages: 
  - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h

  - sed -i.bak -e "s/\/\/\ *\(#define\ *DEBUGGING\ *\)/\1/g" lib_com/options.h



.get-basop-float-reference: &get-basop-float-reference

  - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch

  - make -j -C ivas-basop

  - cp ivas-basop/IVAS_cod ./

  - cp ivas-basop/IVAS_dec ./



.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec 

  ### build test binaries, initial clean for paranoia reasons

  - *disable-debugging-macro
@@ -279,10 +285,24 @@ stages: 
      when: never

    - when: on_success



.rules-merge-request:

.rules-merge-request-to-main:

  extends: .rules-basis

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never



.rules-merge-request-to-basop-ci-branch:

  extends: .rules-basis

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never



.rules-merge-request-to-basop-ci-branch-or-main:

  extends: .rules-basis

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main")

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never
@@ -327,7 +347,7 @@ stages: 
  stage: test

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week
@@ -396,7 +416,7 @@ uninterruptible: 


branch-is-up-to-date-with-main-pre:

  extends:

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: prevalidate

  needs: []

  tags:
@@ -408,7 +428,7 @@ branch-is-up-to-date-with-main-pre: 


check-self-test-names-pre:

  extends:

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: prevalidate

  needs: []

  tags:
@@ -418,7 +438,7 @@ check-self-test-names-pre: 


check-no-duplicates-in-self-tests:

  extends:

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: prevalidate

  needs: []

  tags:
@@ -428,7 +448,7 @@ check-no-duplicates-in-self-tests: 


branch-is-up-to-date-with-main-post:

  extends:

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: postvalidate

  tags:

    - ivas-linux
@@ -440,24 +460,19 @@ branch-is-up-to-date-with-main-post: 
.basop-ci-branch-compat-template:

  extends:

    - .test-job-linux

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "basop-ci-branch"

      changes:

        - tests/**/*

        - scripts/**/*

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

    - .rules-merge-request-to-basop-ci-branch

  tags:

    - ivas-linux-fast

  before_script:

    - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch

    - make -j -C ivas-basop

    - python3 ci/remove_unsupported_testcases.py $PARAM_FILE

    - python3 tests/create_short_testvectors.py



    - *update-ltv-repo

    - *copy-ltv-files-to-testv-dir



    - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true

    - *get-basop-float-reference



    - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi
@@ -579,7 +594,7 @@ build-codec-windows-msbuild: 
codec-smoke-test:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-basop-ci-branch-or-main

  timeout: "20 minutes"

  tags:

    - ivas-linux-fast
@@ -589,7 +604,17 @@ codec-smoke-test: 
    - *print-common-info

    # LTV update needed as ltv ISM metadata files are used

    - *update-ltv-repo



    # for MRs to basop-ci-branch, we want to test with the BASOP float reference build

    # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros

    - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then

    -   python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline

    -   *get-basop-float-reference

    -   bash ci/smoke_test.sh coverage

    - else

    -   bash ci/smoke_test.sh

    - fi



    ### analyze for failures

    - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi

    - ret_val=0
@@ -648,7 +673,7 @@ codec-usan: 
pytest-compare-20ms-and-5ms-rendering:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"]

  script:
@@ -695,7 +720,7 @@ pytest-compare-20ms-and-5ms-rendering: 
renderer-smoke-test:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-make"]

  stage: test

  script:
@@ -717,7 +742,7 @@ renderer-smoke-test: 
renderer-asan:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-cmake"]

  stage: test

  script:
@@ -741,7 +766,7 @@ renderer-asan: 
renderer-msan:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-cmake"]

  stage: test

  script:
@@ -765,7 +790,7 @@ renderer-msan: 
renderer-usan:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-cmake"]

  stage: test

  script:
@@ -793,7 +818,7 @@ renderer-usan: 
renderer-pytest-on-merge-request:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-make"]

  # TODO: set reasonable timeout, will most likely take less

  timeout: "20 minutes"
@@ -845,7 +870,7 @@ renderer-pytest-on-merge-request: 
split-rendering-smoke-test:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-make"]

  stage: test

  script:
@@ -866,7 +891,7 @@ split-rendering-smoke-test: 
lc3-wrapper-unit-test:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-cmake"]

  stage: test

  script:
@@ -878,7 +903,7 @@ lc3-wrapper-unit-test: 
split-rendering-pytest-on-merge-request:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  needs: ["build-codec-linux-make"]

  # TODO: set reasonable timeout, will most likely take less

  timeout: "30 minutes"
@@ -950,7 +975,7 @@ split-rendering-pytest-on-merge-request: 
ivas-pytest-on-merge-request:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: compare

  needs: ["build-codec-linux-cmake", "codec-smoke-test"]

  timeout: "14 minutes"
@@ -1004,7 +1029,7 @@ ivas-pytest-on-merge-request: 
ivas-interop-on-merge-request:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake"]

  timeout: "10 minutes"
@@ -1053,7 +1078,7 @@ ivas-interop-on-merge-request: 
evs-pytest-on-merge-request:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: compare

  needs: ["build-codec-linux-cmake", "codec-smoke-test"]

  timeout: "10 minutes"
@@ -1105,7 +1130,7 @@ evs-pytest-on-merge-request: 
voip-be-on-merge-request:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "10 minutes"
@@ -1118,7 +1143,7 @@ voip-be-on-merge-request: 
clang-format-check:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  variables:

    ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix"

  stage: prevalidate
@@ -1165,7 +1190,7 @@ clang-format-check: 
check-first-frame-is-sid:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

    - .rules-merge-request-to-main

  tags:

    - ivas-linux

  stage: test
@@ -1192,7 +1217,7 @@ check-first-frame-is-sid: 
.lc3plus-ensure-no-code-changes:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: postvalidate

  needs: []

  timeout: "5 minutes"
@@ -1207,7 +1232,7 @@ check-first-frame-is-sid: 
check-bitexactness-hrtf-rom-and-file:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake"]

  timeout: "5 minutes"
@@ -1229,7 +1254,7 @@ check-bitexactness-hrtf-rom-and-file: 
check-bitexactness-ext-and-transport-format:

  extends:

    - .test-job-linux

    - .rules-merge-request

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake"]

  timeout: "5 minutes"
@@ -1256,7 +1281,7 @@ check-bitexactness-ext-and-transport-format: 
# check bitexactness to EVS windows binaries

be-2-evs-windows:

  extends:

    - .rules-merge-request

    - .rules-merge-request-to-main

  tags:

    - ivas-windows

  stage: test

scripts/smoketest-basop-filter.py

0 → 100644
+53 −0
Original line number Diff line number Diff line 
#! /usr/bin/env python3



import argparse

import json

import logging

import sys





logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')

logger = logging.getLogger()





def main():

    parser = argparse.ArgumentParser()



    parser.add_argument("input", help="input JSON", type=argparse.FileType('r'))

    parser.add_argument("output", help="output JSON", type=argparse.FileType('w'), nargs="?")

    parser.add_argument("--inline", "-i", help="patch input inline", action="store_true")



    args = parser.parse_args()



    if not args.inline and not args.output:

        logging.error("You must specify an output file or use --inline")

        return 1



    modes = json.load(args.input)



    for mode in ("MC", "stereo", "OMASA"):

        for encoder_config_name in modes[mode].keys():

            try:

                del modes[mode][encoder_config_name]["dec"]["EXT"]

            except KeyError:

                logger.warning(f"No EXT dec in {mode} -> {encoder_config_name}, skipping")

                continue

            else:

                logger.info(f"Found EXT dec in {mode} -> {encoder_config_name}, removing")



    for encoder_config_name in tuple(modes["OSBA"].keys()):

        if not "FOA" in encoder_config_name:

            continue



        logger.info(f"Removing OSBA FOA encoder config {encoder_config_name}")

        del modes["OSBA"][encoder_config_name]



    if args.inline:

        with open(args.input.name, "w") as f:

            json.dump(modes, f, indent=4)

    else:

        json.dump(modes, args.output, indent=4)





if __name__ == "__main__":

    sys.exit(main())