Commit 2debdcb3 authored by Dominik Weckbecker's avatar Dominik Weckbecker 💬
Browse files

Merge branch '752-osba-mct-misconfiguration-causes-crashes-at-256kbps' into b_20230817_jbm-for-osba

parents dec0b022 51ad3aa6

.gitlab-ci.yml

+53 −82
Original line number Diff line number Diff line
@@ -38,10 +38,11 @@ workflow: 
stages:

  - .pre

  - maintenance

  - prevalidate

  - build

  - test

  - compare

  - validate

  - postvalidate

  - deploy



# ---------------------------------------------------------------
@@ -133,6 +134,14 @@ stages: 
  - sed -i.bak -e "s/\/\*[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)[[:space:]]*\*\//\1/g" ./lib_com/options.h

  - sed -i.bak -e "s/\/\/[[:space:]]*\(#define[[:space:]]*SPLIT_REND_WITH_HEAD_ROT\)/\1/g"        ./lib_com/options.h



.get-commits-behind-count: &get-commits-behind-count

  - echo $CI_COMMIT_SHA

  - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

  - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME)



.check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs

  - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run 'git merge origin/main' to update."; exit 1; fi



# ---------------------------------------------------------------

# Job templates

# ---------------------------------------------------------------
@@ -179,11 +188,13 @@ stages: 
.build-job-linux:

  stage: build

  timeout: "4 minutes"

  needs: []

  tags:

    - ivas-linux



.build-job-windows:

  stage: build

  needs: []

  timeout: "4 minutes"

  tags:

    - ivas-windows
@@ -236,19 +247,28 @@ uninterruptible: 
# Validation jobs

# ---------------------------------------------------------------



check-if-branch-is-up-to-date-with-main:

branch-is-up-to-date-with-main-pre:

  extends:

    - .rules-merge-request

  stage: validate

  stage: prevalidate

  needs: []

  tags:

    - ivas-linux

  script:

    - echo $CI_COMMIT_SHA

    - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

    - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME)

    - *get-commits-behind-count

    - echo $commits_behind_count

    - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, run 'git merge origin/main' to update."; exit 1; fi;



branch-is-up-to-date-with-main-post:

  extends:

    - .rules-merge-request

  stage: postvalidate

  tags:

    - ivas-linux

  script:

    - *get-commits-behind-count

    - echo $commits_behind_count

    - if [ $commits_behind_count -eq 0 ]; then exit 0; else exit 1; fi;

    - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run 'git merge origin/main' to update." exit 1; fi;



# ---------------------------------------------------------------

# Build jobs
@@ -530,6 +550,8 @@ renderer-pytest-on-merge-request: 
  stage: compare

  script:

    - *print-common-info

    - *get-commits-behind-count

    - *check-commits-behind-count-in-compare-jobs



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    # write to temporary file as workaround for failures observed with piping echo
@@ -611,6 +633,8 @@ split-rendering-pytest-on-merge-request: 
  stage: compare

  script:

    - *print-common-info

    - *get-commits-behind-count

    - *check-commits-behind-count-in-compare-jobs



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    # write to temporary file as workaround for failures observed with piping echo
@@ -674,6 +698,8 @@ ivas-pytest-on-merge-request: 
  timeout: "10 minutes"

  script:

    - *print-common-info

    - *get-commits-behind-count

    - *check-commits-behind-count-in-compare-jobs

    - *merge-request-comparison-setup-codec



    # some helper variables - "|| true" to prevent failures from grep not finding anything
@@ -726,6 +752,8 @@ evs-pytest-on-merge-request: 
  timeout: "10 minutes"

  script:

    - *print-common-info

    - *get-commits-behind-count

    - *check-commits-behind-count-in-compare-jobs

    - *merge-request-comparison-setup-codec



    # some helper variables - "|| true" to prevent failures from grep not finding anything
@@ -770,8 +798,8 @@ voip-be-on-merge-request: 
  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  stage: compare # Or should it be test? Comparison is done within one git revision

  needs: ["build-codec-linux-make", codec-smoke-test]

  stage: test

  needs: ["build-codec-linux-make", "codec-smoke-test"]

  timeout: "10 minutes"

  script:

    - *print-common-info
@@ -783,7 +811,7 @@ clang-format-check: 
    - .rules-merge-request

  variables:

    ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix"

  stage: validate

  stage: prevalidate

  needs: []

  timeout: "5 minutes"

  script:
@@ -830,7 +858,7 @@ check-first-frame-is-sid: 
  tags:

    - ivas-linux

  stage: test

  # needs: ["build-codec-linux-cmake"]

  needs: ["build-codec-linux-cmake"]

  script:

    - *print-common-info

    - *update-ltv-repo
@@ -841,13 +869,16 @@ check-first-frame-is-sid: 
    - make -j



    # TODO: for some MASA modes, we currently do not have testvectors that actually trigger DTX

    # SBA is run separately to use shorter part of file

    # SBA modes are run separately to use shorter part of file

    - exit_code_no_sba=0

    - exit_code_sba=0

    - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -v MASA | grep -v SBA)

    - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 0:20 || exit_code_no_sba=$?

    - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep SBA)

    - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 70:80 || exit_code_sba=$?

    - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -v MASA | grep -vE "FOA|HOA" )

    - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 200 -U 0:20 || exit_code_no_sba=$?

    - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -E "HOA")

    - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 70:80 || exit_code_sba=$?

    # need to do FOA separately as VAD apparently behaves differently there

    - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep "FOA")

    - scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 || exit_code_sba=$?

    - if [ $exit_code_no_sba -ne 0 ] || [ $exit_code_sba -ne 0 ]; then exit 1; fi

  artifacts:

    paths:
@@ -857,50 +888,11 @@ check-first-frame-is-sid: 
    expose_as: "logs-sidstart"

    expire_in: "5 days"



selection-test-processing:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  tags:

    - processing-scripts-linux

  stage: test

  needs: ["build-codec-linux-make"]

  script:

    # get processing script code

    - git clone https://forge.3gpp.org/rep/ivas-codec-pc/ivas-processing-scripts.git --single-branch -b main



    - mkdir -p ivas-processing-scripts/tests/data/testv

    - cp -r scripts/testv/* ivas-processing-scripts/tests/data/testv/



    # copy binaries into local bin dir, those should take precendence over PATH

    - cp $PROCESSING_SCRIPTS_BIN_DIR/* ivas-processing-scripts/ivas_processing_scripts/bin/

    # for testing with native binaries

    - rm ivas-processing-scripts/ivas_processing_scripts/bin/IVAS*.exe

    - rm ivas-processing-scripts/ivas_processing_scripts/bin/EVS*.exe



    # build codec and put into bin dir

    - make -j

    - cp ./IVAS_* ivas-processing-scripts/ivas_processing_scripts/bin/



    # patch the use_windows_codec_binaries key (weird folding is needed so colons are accepted)

    - >

      sed -i "s/use_windows_codec_binaries: true/use_windows_codec_binaries: false/" ivas-processing-scripts/experiments/selection/*/config/*.yml



    # run experiments test

    - cd ivas-processing-scripts

    - python3 -m pytest tests/test_experiments.py::test_generate_test_items -n auto | tee log.txt

  artifacts:

    paths:

      - ivas-processing-scripts/experiments/selection/*/proc_output/*.log

      - log.txt

    when: on_failure

    expire_in: 1 week



lc3plus-ensure-no-code-changes:

  extends:

    - .test-job-linux

    - .rules-merge-request

  stage: validate

  stage: postvalidate

  needs: []

  timeout: "5 minutes"

  script:
@@ -1256,15 +1248,15 @@ sanitizer-test-mc-7_1_4: 
    - .sanitizer-test-template

  timeout: 6 hours



sanitizer-test-sba:

sanitizer-test-hoa3:

  extends: .sanitizer-test-schedule-C

  rules:

    - if: $SANITIZER_SCHEDULE_C

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

    - python3 ci/run_scheduled_sanitizer_test.py HOA3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-planarsba:

sanitizer-test-planarhoa3:

  extends: .sanitizer-test-schedule-C

  rules:

    - if: $SANITIZER_SCHEDULE_C
@@ -1272,7 +1264,7 @@ sanitizer-test-planarsba: 
      start_in: 6 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

    - python3 ci/run_scheduled_sanitizer_test.py PlanarHOA3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



### --- sanitizer schedule D ---
@@ -1457,7 +1449,7 @@ complexity-sba-hoa3-in-hoa3-out: 
    - *print-common-info

    - *update-ltv-repo

    - *complexity-measurements-setup

    - in_format=SBA

    - in_format=HOA3

    - out_format=HOA3

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format"

    - ret_val=$?
@@ -1522,27 +1514,6 @@ complexity-StereoDmxEVS-stereo-in-mono-out: 
# Other jobs

# ---------------------------------------------------------------



upload-selection-BE-log:

  rules:

    - if: $UPLOAD_SELECTION_BE_RESULTS && $CI_PIPELINE_SOURCE == 'trigger'

      when: always

  timeout: 5 minutes

  tags:

    - ericsson-windows-runner

  script:

    - cp -r $SELECTION_BE_RESULT ./selection-BE-result

    - Get-Content -Path selection-BE-result/public_log--sha-*.txt

    - $has_failed_debugging_active = (Select-String -Path selection-BE-result/public_log--sha-*.txt -Pattern '^FAILED tests' -CaseSensitive).Line

    - Get-Content -Path selection-BE-result/public_log_DEBUGGING_inactive--sha-*.txt

    - $has_failed_debugging_inactive = (Select-String -Path selection-BE-result/public_log_DEBUGGING_inactive--sha-*.txt -Pattern '^FAILED tests' -CaseSensitive).Line

    - If($has_failed_debugging_active -or $has_failed_debugging_inactive) {exit -1}

  artifacts:

    paths:

      - selection-BE-result/public_log--sha-*.txt

      - selection-BE-result/public_log_DEBUGGING_inactive--sha-*.txt

    when: always

    expire_in: 1 week



# job that sets up gitlab pages website

pages:

  stage: deploy

ci/smoke_test.sh

+1 −1
Original line number Diff line number Diff line
@@ -102,7 +102,7 @@ echo "\n======================= 5. JBM, modes with EXT =======================\n 
./scripts/runIvasCodec.py  $verbosity_cmd -m $modes_with_ext_out -p $cfg $duration_arg $WORKERS --decoder_only --jbm_file $dly_profile --oc BINAURAL BINAURAL_ROOM_IR mono stereo FOA HOA3 5_1 7_1_4 | tee -a smoke_test_output_jbm_noEXT.txt



# run all modes with binaural output using external files

modes_with_bin_out="SBA PlanarSBA MASA MC ISM1 ISM2 ISM3 ISM4"

modes_with_bin_out="FOA HOA2 HOA3 PlanarFOA PlanarHOA2 PlanarHOA3 MASA MC ISM1 ISM2 ISM3 ISM4"

bin_out_modes="BINAURAL BINAURAL_ROOM_IR"



echo "\n======================= 6. binaural out with HRTF files - WB =======================\n\n"

lib_dec/ivas_corecoder_dec_reconfig.c

+15 −0
Original line number Diff line number Diff line
@@ -60,6 +60,9 @@ ivas_error ivas_corecoder_dec_reconfig( 
    int16_t n, sce_id, cpe_id, output_frame;

    int16_t nSCE_existing, nCPE_existing;

    int32_t ivas_total_brate;

#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

    int16_t nchan_transport_real;

#endif

    MC_MODE last_mc_mode;

    DECODER_CONFIG_HANDLE hDecoderConfig;

    ivas_error error;
@@ -82,6 +85,14 @@ ivas_error ivas_corecoder_dec_reconfig( 
        last_mc_mode = MC_MODE_NONE;

    }



#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

    nchan_transport_real = st_ivas->nchan_transport;



    if ( st_ivas->ism_mode == ISM_SBA_MODE_DISC )

    {

        nchan_transport_real += st_ivas->nchan_ism;

    }

#endif

    /*-----------------------------------------------------------------*

     * Allocate, initialize, and configure SCE/CPE/MCT handles

     *-----------------------------------------------------------------*/
@@ -105,7 +116,11 @@ ivas_error ivas_corecoder_dec_reconfig( 
        }

    }



#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

    if ( nchan_transport_real == nchan_transport_old && st_ivas->nSCE == nSCE_old && st_ivas->nCPE == nCPE_old )

#else

    if ( st_ivas->nchan_transport == nchan_transport_old && st_ivas->nSCE == nSCE_old && st_ivas->nCPE == nCPE_old )

#endif

    {

        for ( sce_id = 0; sce_id < st_ivas->nSCE; sce_id++ )

        {

lib_dec/ivas_sba_dec.c

+3 −0
Original line number Diff line number Diff line
@@ -563,6 +563,9 @@ ivas_error ivas_sba_dec_reconfigure( 
            st_ivas->nCPE += ( st_ivas->nchan_ism + 1 ) >> 1;

            nCPE_old = st_ivas->nCPE;

            nchan_transport_old = st_ivas->nchan_transport;

#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

            nchan_transport_old += st_ivas->nchan_ism;

#endif

        }

    }

lib_enc/ivas_corecoder_enc_reconfig.c

+20 −5
Original line number Diff line number Diff line
@@ -64,7 +64,9 @@ ivas_error ivas_corecoder_enc_reconfig( 
    int16_t i, nb_bits;

    Indice temp_ind_list[MAX_NUM_IND_TEMP_LIST];

    int16_t nb_bits_tot;



#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

    int16_t nchan_transport_real, nchan_transport_old_real;

#endif

    ENCODER_CONFIG_HANDLE hEncoderConfig;

    ivas_error error;
@@ -81,11 +83,24 @@ ivas_error ivas_corecoder_enc_reconfig( 
        len_inp_memory += NS2SA( hEncoderConfig->input_Fs, IVAS_FB_ENC_DELAY_NS );

    }



#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

    nchan_transport_old_real = nchan_transport_old;

    nchan_transport_real = st_ivas->nchan_transport;



    if ( st_ivas->ism_mode == ISM_SBA_MODE_DISC )

    {

        nchan_transport_real += st_ivas->hEncoderConfig->nchan_ism;

    }

#endif



    /*-----------------------------------------------------------------*

     * Switching between SCE(s)/CPE(s)/MCT

     *-----------------------------------------------------------------*/



#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

    if ( nchan_transport_real == nchan_transport_old_real && st_ivas->nSCE == nSCE_old && st_ivas->nCPE == nCPE_old ) /* in McMASA, nchan_transport may be the same, but nSCE/nCPE differs */

#else

    if ( st_ivas->nchan_transport == nchan_transport_old && st_ivas->nSCE == nSCE_old && st_ivas->nCPE == nCPE_old ) /* in McMASA, nchan_transport may be the same, but nSCE/nCPE differs */

#endif

    {

        for ( sce_id = 0; sce_id < st_ivas->nSCE; sce_id++ )

        {
@@ -117,14 +132,14 @@ ivas_error ivas_corecoder_enc_reconfig( 
    }

    else

    {

#ifndef NONBE_FIX_752_OSBA_MISCONFIG_MCT

        int16_t nchan_transport_real, nchan_transport_old_real;





        nchan_transport_old_real = nchan_transport_old;

        nchan_transport_real = st_ivas->nchan_transport;



#ifdef NONBE_FIX_752_OSBA_MISCONFIG_MCT

        nchan_transport_real += st_ivas->hEncoderConfig->nchan_ism;

#endif



        /* in SCE+CPE McMASA nchan_transport is still 2, fix the numbers */

        if ( hEncoderConfig->ivas_format == MC_FORMAT && last_mc_mode == MC_MODE_MCMASA )

        {