Commit 02f7bf23 authored by vaclav's avatar vaclav
Browse files

Merge branch 'main' into 114-bitrate-switching-in-sba

parents d63cde2d 56fea2c2

.gitignore

+2 −0
Original line number Diff line number Diff line
@@ -46,6 +46,8 @@ scripts/ref/ 
scripts/test/

scripts/out/

scripts/self_test_summary.txt

tests/dut

tests/ref



# Python files that pop up when running scripts

__pycache__/

.gitlab-ci.yml

+59 −72
Original line number Diff line number Diff line
@@ -6,6 +6,8 @@ variables: 
  OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"

  OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"

  OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM"

  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1





# This sets when pipelines are created. Jobs have more specific rules to restrict them.
@@ -253,7 +255,7 @@ asan-on-merge-request-linux: 




# compare bit exactness between target and source branch

self-test-on-merge-request:

pytest-on-merge-request:

  extends: 

    - .test-job-linux

    - .rules-merge-request
@@ -295,54 +297,51 @@ self-test-on-merge-request: 
    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..



    ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)

    ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything)

    - git checkout $source_branch_commit_sha



    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt

    ### analyse test output



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1

    - expected_nonbe_1=0

    - expected_nonbe_2=0

    - expected_nonbe_3=0

    - fail_1=0

    - fail_2=0

    - fail_3=0



    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi

    ### prepare pytest

    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi

    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi



    # check for non bitexact EVS output

    - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi

    - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi



    ### run SBA pytest

    ### run pytest for EVS cases

    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$?

    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi

    - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi



    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi

    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi

    - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi



    # Check results from all three tests

    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi

    # Check results from both tests

    - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi

    - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi

    - exit 0



  allow_failure:
@@ -352,14 +351,13 @@ self-test-on-merge-request: 
    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - test_output.txt

      - test_output_evs.txt

      - scripts/test/logs/

      - scripts/ref/logs/

      - report-junit.xml

    expose_as: 'Self test results'

      - report-junit-evs.xml

    expose_as: 'pytest results'

    reports:

      junit: report-junit.xml

      junit:

        - report-junit.xml

        - report-junit-evs.xml





# ---------------------------------------------------------------
@@ -436,37 +434,26 @@ codec-comparison-on-main-push: 
    ### re-checkout the latest commit in the main branch

    - git checkout $latest_commit



    ### run selftest

    - ls -altr scripts/testv

    - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt

    # helper variable - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[non[ -]*be\]") || true



    ### analyse test output

    ### prepare pytest

    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # rename test binaries back

    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2



    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true

    - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true

    - EXIT_CODE_NON_BE=123

    - EXIT_CODE_FAIL=1



    - selftest_exit_code=0



    # check for crashes during the test, if any happened, fail the test

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi



    # check for non bitexact output and store exit code to also always run the SBA pytest

    - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi

    - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi



    ### run SBA pytest

    ### run pytest

    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

    # return exit code from selftest if everything went well with the pytest run

    - exit $selftest_exit_code

  allow_failure:

    exit_codes:

      - 123
@@ -474,9 +461,6 @@ codec-comparison-on-main-push: 
    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    when: always

    paths:

      - test_output.txt

      - scripts/test/logs/

      - scripts/ref/logs/

      - report-junit.xml

    expose_as: 'Results of comparison to previous merge commit'

    reports:
@@ -497,6 +481,8 @@ codec-comparison-on-main-push: 
    when: always

    paths:

      - ep_015.g192

      # second wildcard is necessary to get encoder and no-PLC run logs

      - "CLANG*/logs*"



sanitizer-test-mono:

  extends: .sanitizer-test-template
@@ -645,9 +631,10 @@ coverage-test-on-main-scheduled: 
  script:

    - *print-common-info

    - make GCOV=1 -j

    - ./scripts/self_test.py --create -t 1

    - ./scripts/self_test.py --create -t 1 scripts/config/self_test_evs.prm

    - ./scripts/ivas_pytests/self_test_b.py --create_only --numprocesses 1 --encref IVAS_cod --decref IVAS_dec --encdut IVAS_cod --decdut IVAS_dec

    - python3 tests/create_short_testvectors.py

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - lcov -c -d obj -o coverage.info

    - genhtml coverage.info -o coverage

  artifacts:

ci/run_scheduled_sanitizer_test.py

+11 −3
Original line number Diff line number Diff line
@@ -93,12 +93,20 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True) 
    subprocess.call(GENPATT_CMD.split())



    # cleanup to avoid script errors

    # we want "logs" and "dec" subfolders to be empty -> delete and recreate them

    cleanup_folders = ["logs", "dec"]

    # we want "logs" and "dec" subfolders to be empty -> delete "dec" and rename "log"

    # to keep the log files from the first run with no frame losses

    folders_to_delete = ["dec"]

    folders_to_backup = ["logs"]

    for t in tests:

        for fol in cleanup_folders:

        for fol in folders_to_delete:

            for fi in pathlib.Path(t).joinpath(fol).iterdir():

                fi.unlink()

        for fol in folders_to_backup:

            path = pathlib.Path(t).joinpath(fol)

            new_name = pathlib.Path(str(path) + "_noPLC")

            path.rename(new_name)

            # need empty log folder to avoid crashes

            path.mkdir()



    cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]

    print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))

lib_com/bitstream.c

+0 −32
Original line number Diff line number Diff line
@@ -1820,11 +1820,7 @@ ivas_error preview_indices( 
                break;

        }

    }

#ifdef ALIGN_SID_SIZE

    else if ( total_brate == IVAS_SID_5k2 )

#else

    else if ( total_brate == IVAS_SID_4k4 )

#endif

    {

        /* read SID format */

        st_ivas->sid_format = 0;
@@ -1888,16 +1884,6 @@ ivas_error preview_indices( 
                return IVAS_ERROR( IVAS_ERR_INTERNAL_FATAL, "Invalid value %c found in SID format field.", st_ivas->sid_format );

        }

    }

#ifndef ALIGN_SID_SIZE

    else if ( total_brate == IVAS_SID_5k )

    {

        /* SBA SID frame */

        st_ivas->sid_format = SID_SBA_1TC;

        st_ivas->ivas_format = SBA_FORMAT;

        st_ivas->sba_mode = SBA_MODE_SPAR;

        st_ivas->element_mode_init = IVAS_SCE;

    }

#endif



    /* only read element mode from active frames */

    if ( is_DTXrate( total_brate ) == 0 )
@@ -1988,11 +1974,7 @@ ivas_error preview_indices( 


            st_ivas->sba_analysis_order = ivas_sba_get_analysis_order( total_brate, st_ivas->sba_order );



#ifdef HARMONIZE_SBA_NCHAN_TRANSPORT

            ivas_sba_config( total_brate, st_ivas->sba_analysis_order, -1, &( st_ivas->nchan_transport ), st_ivas->sba_planar, &( st_ivas->nSCE ), &( st_ivas->nCPE ), &( st_ivas->element_mode_init ) );

#else

            ivas_sba_config( total_brate, st_ivas->sba_analysis_order, -1, &( st_ivas->nchan_transport ), st_ivas->sba_planar, &( st_ivas->nSCE ), &( st_ivas->nCPE ), &( st_ivas->element_mode_init ), st_ivas->sba_mode );

#endif

        }

    }
@@ -2072,19 +2054,7 @@ ivas_error read_indices( 
        }

        else if ( k == SIZE_IVAS_BRATE_TBL )

        {

#ifdef ALIGN_SID_SIZE

            return IVAS_ERROR( IVAS_ERR_INTERNAL_FATAL, "Error, illegal bitrate (%d) in the  G.192 frame ! Exiting ! \n", total_brate );

#else

            /*temp change for SPAR DTX*/

            if ( total_brate == IVAS_SID_5k )

            {

                st_ivas->element_mode_init = -1;

            }

            else

            {

                return IVAS_ERROR( IVAS_ERR_INTERNAL_FATAL, "Error, illegal bitrate (%d) in the  G.192 frame ! Exiting ! \n", total_brate );

            }

#endif

        }

        else

        {
@@ -2978,7 +2948,6 @@ void evs_dec_previewFrame( 
}





#ifdef ALIGN_SID_SIZE

void dtx_read_padding_bits(

    DEC_CORE_HANDLE st,

    int16_t num_bits )
@@ -2990,6 +2959,5 @@ void dtx_read_padding_bits( 
    get_next_indice( st, num_bits );

    st->total_brate = tmp;

}

#endif



#undef WMC_TOOL_MAN

lib_com/ivas_cnst.h

+2 −31
Original line number Diff line number Diff line
@@ -165,14 +165,10 @@ typedef enum 
#define HEAD_ROTATION_HOA_ORDER                 3                           /* HOA 3rd order */

#define MAX_CICP_CHANNELS                       16                          /* max channels for loudspeaker layouts (16 for custom layouts)*/

#define MAX_OUTPUT_CHANNELS                     16                          /* Maximum number of output channels (HOA 3rd order) */

#ifndef FIX_CREND_CHANNELS

#define IVAS_MAX_NUM_CH                         16                          /* == MAX_OUTPUT_CHANNELS */

#endif



#define FOA_CHANNELS                            4                           /* number of FOA channels */



#define BINAURAL_CHANNELS                       2                           /* number of channels for binaural output configuration */

#define CPE_CHANNELS                            2                           /* number of CPE (stereo) channels */

#define FOA_CHANNELS                            4                           /* number of FOA channels */

#define MAX_NUM_OBJECTS                         4                           /* max. number of audio objects */



#define MAX_SCE                                 MAX_NUM_OBJECTS             /* max. number of SCEs */
@@ -197,12 +193,7 @@ typedef enum 
/*----------------------------------------------------------------------------------*

 * IVAS Bitrates

 *----------------------------------------------------------------------------------*/

#ifdef ALIGN_SID_SIZE

#define IVAS_SID_5k2                            5200 /* SID frame bitrate */

#else

#define IVAS_SID_4k4                            4400 /* SID frame bitrate */

#define IVAS_SID_5k                             5000 /* SBA SID frame bitrate */

#endif

#define IVAS_13k2                               13200

#define IVAS_16k4                               16400

#define IVAS_24k4                               24400
@@ -220,13 +211,8 @@ typedef enum 


#define IVAS_BRATE_MAX                          IVAS_512k



#ifdef ALIGN_SID_SIZE

#define SIZE_IVAS_BRATE_TBL                     16

#define IVAS_NUM_ACTIVE_BRATES                 (SIZE_IVAS_BRATE_TBL - 2)

#else

#define SIZE_IVAS_BRATE_TBL                     17

#define IVAS_NUM_ACTIVE_BRATES                 (SIZE_IVAS_BRATE_TBL - 3)

#endif



/*----------------------------------------------------------------------------------*

 * IVAS modes : IVAS SCE, IVAS CPE modes (DFT, TD, MDCT stereo)
@@ -842,9 +828,7 @@ typedef enum { 
 *----------------------------------------------------------------------------------*/

// VE: this should be renamed to e.g. N_SPATIAL_SUBFRAMES

#define MAX_PARAM_SPATIAL_SUBFRAMES             4                           /* Maximum number of subframes for parameteric spatial coding */

#ifdef FIX_I106_TDREND_5MS

#define L_SPATIAL_SUBFR_48k                     (L_FRAME48k / MAX_PARAM_SPATIAL_SUBFRAMES)

#endif





/*----------------------------------------------------------------------------------*
@@ -869,20 +853,7 @@ typedef enum 
 * DirAC Constants 

 *----------------------------------------------------------------------------------*/



#ifdef FIX_DIRAC_CHANNELS

#define DIRAC_MAX_ANA_CHANS                     FOA_CHANNELS                /* Maximum number of channels for DirAC analysis */

#else

#define DIRAC_MAX_ANA_CHANS                     4                           /* Maximum number of channels for DirAC analysis */

#endif



#ifndef HARMONIZE_SBA_NCHAN_TRANSPORT

#define DIRAC_MAX_TRANS_CHANS                   8                           /* Maximum number of transport channels for DirAC */

#endif



#define DIRAC_MIN_BITRATE_8_TRANS_CHAN          IVAS_384k

#define DIRAC_MIN_BITRATE_6_TRANS_CHAN          IVAS_256k

#define DIRAC_MIN_BITRATE_4_TRANS_CHAN          IVAS_160k                   /* minimum bitrate for sending 4 transport channels (FOA) */

#define DIRAC_MIN_BITRATE_2_TRANS_CHAN          IVAS_48k                    /* minimum bitrate for sending 2 transport channels (Stereo) */



#define DIRAC_NUM_DIMS                          3                           /* number of directions to estimate (X,Y,Z) */

#define DIRAC_MAX_NBANDS                        12                          /* Maximum number of frequency bands for the DirAC Side Parameter decoding */
@@ -952,7 +923,7 @@ typedef enum 


#define SPAR_CONFIG_BW                          FB



#define IVAS_SPAR_MAX_CH                        (2*IVAS_MAX_SBA_ORDER + 2) /* FOA + planar HOA */

#define IVAS_SPAR_MAX_CH                        (FOA_CHANNELS + 2 * ( IVAS_MAX_SBA_ORDER - 1 )) /* FOA + planar HOA */



#define IVAS_SPAR_P_LOWERTRI                    ((IVAS_SPAR_MAX_CH - 1) * (IVAS_SPAR_MAX_CH - 2)) >> 1

#define IVAS_SPAR_MAX_C_COEFF                   (IVAS_SPAR_MAX_CH - IVAS_SPAR_MAX_DMX_CHS) * ( IVAS_SPAR_MAX_DMX_CHS - 1)