Commit 7601472d authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'basop-ci-branch' into ci/add-inplace-option-to-create-short-testvectory-script

parents a9261239 37925dc5
Loading
Loading
Loading
Loading
+17 −19
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ workflow:
  - cd -
  - cp ivas-basop/IVAS_cod ./
  - cp ivas-basop/IVAS_dec ./
  - cp ivas-basop/IVAS_rend ./

.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec 
  ### build test binaries, initial clean for paranoia reasons
@@ -251,7 +252,10 @@ branch-is-up-to-date-with-main-post:

    - *get-basop-float-reference

    - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT
    # hack for renderer tests to match the expected naming
    - mv IVAS_rend IVAS_rend_ref

    - python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$TESTCASE_TIMEOUT

  artifacts:
    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
@@ -408,7 +412,7 @@ codec-smoke-test:
    # LTV update needed as ltv ISM metadata files are used
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    # for MRs to basop-ci-branch, we want to test with the BASOP float reference build
    # for MRs to basop-ci-branch-pc, we want to test with the BASOP float reference build
    # per default, the smoke test builds the current repo again with WMOPS activated to catch unbalanced instrumentation macros
    - if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" = "basop-ci-branch" ]; then
    -   python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline
@@ -426,7 +430,7 @@ codec-smoke-test:
    - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi
    - exit $ret_val
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    paths:
@@ -508,7 +512,7 @@ pytest-compare-20ms-and-5ms-rendering:
    - if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi
    - if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    expose_as: "pytest 5ms and 10ms vs 20ms results"
@@ -550,7 +554,7 @@ renderer-smoke-test:
    - .rules-merge-request-to-main
  needs: ["build-codec-linux-cmake"]
  stage: test
  timeout: "1 hour"
  timeout: "30 minutes"
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    expire_in: 1 week
@@ -773,12 +777,6 @@ ivas-pytest-on-merge-request:
  stage: compare
  needs: ["build-codec-linux-cmake", "codec-smoke-test"]
  timeout: "14 minutes"
  parallel:
    matrix:
      # note: keep in sync with list in $TESTS_DIR_CODEC_BE_ON_MR
      - PYTEST_SCRIPT:
          - test_param_file.py
          - test_sba.py
  script:
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
    - commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)"
@@ -796,7 +794,7 @@ ivas-pytest-on-merge-request:

    ### prepare pytest
    # create references
    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1
    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1

    ### Run test using branch scripts and input
    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi
@@ -804,7 +802,7 @@ ivas-pytest-on-merge-request:
    ### run pytest
    - exit_code=0
    - testcase_timeout=60
    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?
    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?
    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - *merge-request-comparison-check
@@ -813,7 +811,7 @@ ivas-pytest-on-merge-request:
    exit_codes:
      - 123
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    paths:
@@ -864,7 +862,7 @@ ivas-interop-on-merge-request:
    exit_codes:
      - 123
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    paths:
@@ -916,7 +914,7 @@ evs-pytest-on-merge-request:
    exit_codes:
      - 123
  artifacts:
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    paths:
@@ -1012,7 +1010,7 @@ check-first-frame-is-sid:
    expose_as: "logs-sidstart"
    expire_in: "5 days"

lc3plus-ensure-no-code-changes:
.lc3plus-ensure-no-code-changes:
  extends:
    - .test-job-linux
    - .rules-merge-request-to-main
@@ -1024,7 +1022,7 @@ lc3plus-ensure-no-code-changes:
    - ./scripts/lc3plus_lib_setup/get_lc3plus.sh

    # Ensure git reports no changes
    - modified_files=$(git status -su lib_lc3plus)
    - modified_files=$(git status -s)
    - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi

check-bitexactness-hrtf-rom-and-file:
@@ -1198,7 +1196,7 @@ codec-comparison-on-main-push:
    exit_codes:
      - 123
  artifacts:
    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
    expire_in: 1 week
    when: always
    paths:
+4 −3
Original line number Diff line number Diff line
@@ -7,7 +7,9 @@

  <h2>Regression tracking</h2>

  <ul>
    <li><a href="long_term_regression.html">Long term regression</a></li>
  </ul>

  <h2>Daily long testvector tests</h2>

@@ -27,8 +29,7 @@
  <h2>Test Coverage</h2>

  <ul>
    <li><a href="coverage_dec_rend/index.html">Coverage report of Decoder and Renderer</a></li>
    <li><a href="coverage_enc_dec_rend/index.html">Coverage report of Encoder, Decoder and Renderer</a></li>
    <li><a href="coverage_enc_dec_rend/index.html">Coverage report</a></li>
  </ul>

</body>
+2 −2
Original line number Diff line number Diff line
@@ -102,7 +102,7 @@ COLUMNS = [
    "testcase",
    "Format",
    "Category",
    "Result",
    "result",
    "MLD",
    "MAX_ABS_DIFF",
    "MIN_SSNR",
@@ -187,7 +187,7 @@ def tr_from_row(row, id_current, id_previous):
        prev = row[f"{c}-{id_previous}"]
        curr = row[f"{c}-{id_current}"]

        if c == "Result":
        if c == "result":
            # print errors in bold red font
            td_tmpl = TD_TMPL_INCREASE if prev == "ERROR" else TD_TMPL_NORMAL
            tr.append(td_tmpl.format(prev))
+21 −70
Original line number Diff line number Diff line
@@ -32,81 +32,39 @@ from pathlib import Path
import argparse

# Enter tag of testcases to remove here WITHOUT the leading //
TESTCASES_MAIN = [
    "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",
    "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out",
    "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id",
    "OSBA planar FOA 1ISM at 256 kbps, 48 kHz in, 48 kHz out, BINAURAL out",
    "Multi-channel 5_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, BINAURAL out, FER at 10%, bandwidth switching",
    "OSBA FOA 4ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, FER at 5%, bandwidth switching",
    "OSBA FOA 2ISM at 64 kbps, 48kHz in, 48kHz out, HOA3 out, bandwidth switching",
    "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",
    "OSBA planar FOA 2ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL out",
    "OSBA planar FOA 1ISM at 256 kbps, 48 kHz in, 48 kHz out, BINAURAL out",
    "SBA 3OA at 128 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB rendconf sel acoustic env",
    "OSBA FOA 4ISM at br sw 13.2 to 512 kbps, 48kHz in, 16kHz out, BINAURAL out (Model from file), FER at 5%, bandwidth switching",
TESTCASES = [
    # self_test.prm
    "stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, EXT out",
    "SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 32kHz in, 32kHz out, DTX on, EXT out",
    "SBA 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out",
    "SBA 3OA bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, random FER at 5%, EXT out",
    "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out",
    "Multi-channel 5_1_2 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, EXT out",
    "Multi-channel 5_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 32kHz out, EXT out",
    "Multi-channel 7_1 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 16kHz out, EXT out",
    "Multi-channel 7_1_4 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out",
    "SBA planar 2OA bitrate switching from 13.2 kbps to 128 kbps, 32kHz in, 32kHz out, DTX on, EXT out",
    "SBA planar FOA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out",
    "SBA 2OA bitrate switching from 13.2 kbps to 512 kbps, 32kHz in, 32kHz out, EXT out",
    "SBA planar 3OA bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, random FER at 5%, EXT out",
    "Multi-channel 5_1 at 512 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (CREND)",
    "Multi-channel 5_1 at 64 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (FastConv)",
    "Multi-channel 5_1 at 32 kbps, 48kHz in 48kHz out, BINAURAL_ROOM_REVERB out custom acoustic environment with a sequence (ParamBin)",
    "MASA 1dir 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration",
    "MASA 1dir 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration",
    "MASA 1TC at 256kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out custom configuration",
    "MASA 1TC at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, HR custom configuration",
    "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",
    "Multi-channel 7_1_4 bitrate switching, 48kHz in, 48kHz out, BINAURAL out, HR, JBM Prof 5",
    "Multi-channel 5_1_2 bitrate switching from 24.4 kbps to 256 kbps, 48kHz in, 48kHz out, EXT out",
    "Multi-channel 7_1 bitrate switching, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out, HR, JBM Prof 5",
    "3 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 32 kHz out, DTX, BINAURAL_ROOM_IR out,  JBM Prof 5",
    # object editing starts here
    "OMASA 2Dir2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",
    "OMASA 2Dir2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM",
    "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",
    "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM",
    "OMASA 2Dir2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC",
    "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5",
    "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC",
    "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC, FER at 5%",
    "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM",
    "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing",
    "OMASA 2Dir2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",
    "OMASA 2Dir2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",
    # the next four lines are basically duplicates of the last four, but without "Dir", because the ltv MASA cases do not have dedicated testvectors for numbers of directions
    "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM",
    "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC",
    "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",
    "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",
    "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, DISC",
    "OSBA 2OA 3ISM at 128 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",
    "OSBA 2OA 3ISM at bitrate switching 13.2 to 512 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",
    "4 ISM with metadata at 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, PARAM_ISM",
    "4 ISM with metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, DISC",
    "3 ISM with metadata at 24.4 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, PARAM_ISM",
    "3 ISM with metadata at 384 kbps, 48 kHz in, 48 kHz out, FOA out, object editing, JBM Prof 5, DISC",
    "4 ISM with metadata bitrate switching from 32 kbps to 48 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_IR out, object editing",
    "4 ISM with metadata bitrate switching from 48 kbps to 32 kbps, 48 kHz in, 48 kHz out, BINAURAL out, object editing, JBM Prof 5",
]
TESTCASES_MAIN_PC = [
    "SBA at 128 kbps, 32kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out, Config renderer, HR",
    "SBA at 128 kbps, 32kHZ in, 16kHz out, BINAURAL_ROOM_REVERB out (Model from file), HR",
    "Planar SBA at 128 kbps, 48kHZ in, 32kHz out, BINAURAL_ROOM_REVERB out (Model from file), Config renderer, HR",
    "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out default configuration",
    "Multi-channel 5_1 at 80 kbps, 48kHz in, 32kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR",
    "OSBA planar 2OA 4ISM at 512 kbps, 48 kHz in, 48 kHz out, BINAURAL ROOM REVERB (Model from file) out",
    "SBA 3OA 4ISM at 96 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out custom configuration",
    "Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config recreation, HR",
    "Multi-channel 5_1_2 at 64 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out Config renderer, HR",
    "OSBA 3OA 4ISM at 512 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out",
    "OSBA 3ISM 3OA at bitrate switching 13.2 to 512 kbps, 48kHz in, 32kHz out, BINAURAL ROOM REVERB out",
    "Multi-channel 5_1 bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL_ROOM_REVERB out",
    "OSBA 2OA 2ISM bitrate switching, 32kHz in, 48kHz out, BINAURAL_ROOM_REVERB out, JBM Prof 5",
]
TESTCASES_MAIN_PC_REENABLE = [
    "4 ISM with extended metadata at 128 kbps, 48 kHz in, 48 kHz out, BINAURAL_ROOM_REVERB out, rendconf dir w id",
    "OSBA 3OA 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_IR out, object editing, DISC",
    # self_test_ltv.prm
    "OMASA 2TC 4ISM at br sw techs 13.2 to 512 kbps start 80 kbps, 48kHz in, 48kHz out, EXT out",
    "OMASA 2TC 2ISM at 96 kbps, 48kHz in, 48kHz out, FOA out, object editing, JBM Prof 5, DISC",
    "OMASA 2TC 4ISM at 80 kbps, 48kHz in, 48kHz out, BINAURAL out, default object editing, 1SEP-PARAM",
    "OMASA 2TC 4ISM at 256 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing, JBM Prof 5, DISC",
    "OMASA 2TC 2ISM br sw techs 13.2 to 512 kbps start 48 kbps, 48kHz in, 48kHz out, BINAURAL out, object editing",
]


def remove_testcases(cfg: Path, testcases: list):
    """
    Go through file line by line and copy all testcases except the given ones
@@ -132,16 +90,9 @@ def remove_testcases(cfg: Path, testcases: list):
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("cfg_files", nargs="+", type=Path)
    parser.add_argument("--use-main-pc-set", action="store_true")
    args = parser.parse_args()

    testcases = TESTCASES_MAIN
    if args.use_main_pc_set:
        # Add further unsupported test cases
        testcases.extend(TESTCASES_MAIN_PC)
        # Re-enable test cases that are now supported
        for case in TESTCASES_MAIN_PC_REENABLE:
            testcases.remove(case)
    testcases = TESTCASES

    for f in args.cfg_files:
        remove_testcases(f, testcases)
+0 −4
Original line number Diff line number Diff line
@@ -134,9 +134,7 @@ typedef enum
    IVAS_ERR_BITSTREAM_READER_INVALID_FORMAT,
    IVAS_ERR_NO_FILE_OPEN,
    IVAS_ERR_SAMPLING_RATE_UNKNOWN,
#ifdef FIX_1370_EXTERNAL_ORIENTATION_CHECK
    IVAS_ERR_EXTERNAL_ORIENTATION_INVALID_FORMAT,
#endif

    /*----------------------------------------*
     *    renderer (lib_rend only)            *
@@ -274,10 +272,8 @@ static inline const char *ivas_error_to_string( ivas_error error_code )
            return "Invalid input format";
        case IVAS_ERR_INVALID_INDEX:
            return "Invalid index";
#ifdef FIX_1370_EXTERNAL_ORIENTATION_CHECK
        case IVAS_ERR_EXTERNAL_ORIENTATION_INVALID_FORMAT:
            return "Euler angles were detected in the input but only Quaternions are supported";
#endif
        default:
            break;
    }
Loading