Commit 52d1dc86 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into... 

Merge branch 'main' into 598-be-re-insert-assert-to-catch-out-of-memory-access-in-the-ind_list-buffer
parents 6024e22a 649241d1

.gitignore

+3 −4
Original line number Diff line number Diff line
@@ -42,10 +42,9 @@ scripts/out/ 
scripts/self_test_summary.txt

scripts/cppp/

binary/

tests/renderer/cut

tests/renderer/ref

tests/dut

tests/ref

tests/**/[c|d]ut

tests/**/ref

tests/*/testv

scripts/testv/*_cut*.pcm

# default reference binary name

IVAS_cod_ref

.gitlab-ci.yml

+19 −16
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ variables: 
  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1

  PROCESSING_SCRIPTS_BIN_DIR: "/test-bin"

  TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection"



default:

  interruptible: true # Make all jobs by default interruptible
@@ -498,7 +499,7 @@ renderer-pytest-on-merge-request: 
      - 123

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"

    expire_in: 2 week

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml
@@ -531,15 +532,15 @@ ivas-pytest-on-merge-request: 
    # create short test vectors

    - python3 tests/create_short_testvectors.py

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -549,7 +550,7 @@ ivas-pytest-on-merge-request: 
      - 123

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 2 week

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml
@@ -579,14 +580,14 @@ evs-pytest-on-merge-request: 


    ### prepare pytest

    # create references

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest for EVS cases

    - exit_code=0

    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$?

    - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -596,7 +597,7 @@ evs-pytest-on-merge-request: 
      - 123

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 2 week

    expire_in: 1 week

    when: always

    paths:

      - report-junit-evs.xml
@@ -655,7 +656,7 @@ clang-format-check: 


    - exit $format_problems

  artifacts:

    expire_in: 2 days

    expire_in: 1 day

    paths:

      - tmp-formatting-fix/

    when: on_failure
@@ -840,15 +841,15 @@ codec-comparison-on-main-push: 
    - mv IVAS_cod_test IVAS_cod

    - mv IVAS_dec_test IVAS_dec

    # create references

    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2



    ### re-checkout the latest commit here, if ref_using_main is set

    - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
@@ -858,7 +859,7 @@ codec-comparison-on-main-push: 
      - 123

  artifacts:

    name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 2 week

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml
@@ -1120,14 +1121,15 @@ coverage-test-on-main-scheduled: 
  rules:

    # only run in scheduled pipeline that passes this env vars

    - if: $COVERAGE_TEST

  timeout: 2 hours

  script:

    - *print-common-info

    - make GCOV=1 -j

    - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script  

    - python3 tests/create_short_testvectors.py

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - bash ci/smoke_test.sh coverage

    - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py

    - bash ci/ivas_voip_be_test.sh coverage
@@ -1320,6 +1322,7 @@ pages: 
  artifacts:

    paths:

      - public

    expire_in: 1 day



# Pull state of a branch on 3GPP repo, push to a mirror repo.

pull-from-3gpp-forge:

Workspace_msvc/lib_util.vcxproj

+0 −2
Original line number Diff line number Diff line
@@ -145,7 +145,6 @@ 
    <ClCompile Include="..\lib_util\cmdl_tools.c" />

    <ClCompile Include="..\lib_util\evs_rtp_payload.c" />

    <ClCompile Include="..\lib_util\g192.c" />

    <ClCompile Include="..\lib_util\head_rotation_file_reader.c" />

    <ClCompile Include="..\lib_util\vector3_pair_file_reader.c" />

    <ClCompile Include="..\lib_util\hrtf_file_reader.c" />

    <ClCompile Include="..\lib_util\ism_file_reader.c" />
@@ -171,7 +170,6 @@ 
    <ClInclude Include="..\lib_util\evs_rtp_payload.h" />

    <ClInclude Include="..\lib_util\g192.h" />

    <ClInclude Include="..\lib_util\vector3_pair_file_reader.h" />

    <ClInclude Include="..\lib_util\head_rotation_file_reader.h" />

    <ClInclude Include="..\lib_util\hrtf_file_reader.h" />

    <ClInclude Include="..\lib_util\ism_file_reader.h" />

    <ClInclude Include="..\lib_util\ism_file_writer.h" />

lib_com/bitstream.c

+136 −0
Original line number Diff line number Diff line
@@ -169,6 +169,131 @@ Word16 rate2EVSmode( 
 * Re-allocate the list of indices

 *-------------------------------------------------------------------*/



#ifdef FIX_MEM_REALLOC_IND_LIST

ivas_error ind_list_realloc(

    INDICE_HANDLE old_ind_list,    /* i  : pointer to the beginning of the old buffer of indices */

    const int16_t max_num_indices, /* i  : new maximum number of allowed indices in the list */

    Encoder_Struct *st_ivas        /* i  : IVAS encoder structure                  */

)

{

    int16_t i, n, ch, n_channels, ind_list_pos, is_metadata, ivas_max_num_indices;

    INDICE_HANDLE new_ind_list;

    BSTR_ENC_HANDLE hBstr;



    if ( st_ivas == NULL )

    {

        return IVAS_ERR_OK;

    }



    /* get the pointer to the beginning of the old buffer of indices (either metadata or core coders) */

    if ( old_ind_list == st_ivas->ind_list_metadata )

    {

        is_metadata = 1;

        ivas_max_num_indices = st_ivas->ivas_max_num_indices_metadata;

    }

    else

    {

        is_metadata = 0;

        ivas_max_num_indices = st_ivas->ivas_max_num_indices;

    }



    /* allocate new buffer of indices */

    if ( ( new_ind_list = (INDICE_HANDLE) malloc( max_num_indices * sizeof( Indice ) ) ) == NULL )

    {

        return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for buffer of indices!\n" ) );

    }



    /* move indices from the old list to the new list */

    for ( i = 0; i < min( max_num_indices, ivas_max_num_indices ); i++ )

    {

        if ( old_ind_list[i].nb_bits > -1 )

        {

            new_ind_list[i].id = old_ind_list[i].id;

            new_ind_list[i].value = old_ind_list[i].value;

        }

        new_ind_list[i].nb_bits = old_ind_list[i].nb_bits;

    }



    /* reset nb_bits of all other indices to -1 */

    for ( ; i < max_num_indices; i++ )

    {

        new_ind_list[i].nb_bits = -1;

    }



    /* update parameters in all SCE elements */

    for ( n = 0; n < st_ivas->nSCE; n++ )

    {

        /* get the pointer to hBstr */

        if ( is_metadata )

        {

            hBstr = st_ivas->hSCE[n]->hMetaData;

        }

        else

        {

            hBstr = st_ivas->hSCE[n]->hCoreCoder[0]->hBstr;

        }



        if ( hBstr != NULL )

        {

            /* get the current position inside the old list */

            ind_list_pos = (int16_t) ( hBstr->ind_list - old_ind_list );



            /* set pointers in the new list */

            *( hBstr->ivas_ind_list_zero ) = new_ind_list;

            hBstr->ind_list = &new_ind_list[ind_list_pos];



            /* set the new maximum number of indices */

            *( hBstr->ivas_max_num_indices ) = max_num_indices;

        }

    }



    /* update parameters in all CPE elements */

    for ( n = 0; n < st_ivas->nCPE; n++ )

    {

        /* get the pointer to hBstr */

        if ( is_metadata )

        {

            n_channels = 1;

        }

        else

        {

            n_channels = CPE_CHANNELS;

        }



        for ( ch = 0; ch < n_channels; ch++ )

        {

            if ( is_metadata )

            {

                hBstr = st_ivas->hCPE[n]->hMetaData;

            }

            else

            {

                hBstr = st_ivas->hCPE[n]->hCoreCoder[ch]->hBstr;

            }



            if ( hBstr != NULL )

            {

                /* get the current position inside the old list */

                ind_list_pos = (int16_t) ( hBstr->ind_list - old_ind_list );



                /* set pointers in the new list */

                *( hBstr->ivas_ind_list_zero ) = new_ind_list;

                hBstr->ind_list = &new_ind_list[ind_list_pos];



                /* set the new maximum number of indices */

                *( hBstr->ivas_max_num_indices ) = max_num_indices;

            }

        }

    }



    /* free the old list */

    free( old_ind_list );



    return IVAS_ERR_OK;

}



#else



ivas_error ind_list_realloc(

    BSTR_ENC_HANDLE hBstr,        /* i/o: encoder bitstream handle                          */

    const int16_t max_num_indices /* i  : new maximum number of allowed indices in the list */
@@ -216,6 +341,8 @@ ivas_error ind_list_realloc( 
    return IVAS_ERR_OK;

}



#endif





/*-----------------------------------------------------------------------*

 * get_ivas_max_num_indices()
@@ -468,6 +595,7 @@ int16_t get_core_max_num_indices( 
    const int32_t total_brate /* i  : total bitrate             */

)

{



    /* set the maximum number of indices in the core coder */

    if ( core == ACELP_CORE || core == AMR_WB_CORE )

    {
@@ -871,7 +999,11 @@ ivas_error check_ind_list_limits( 
#endif



        /* reallocate the buffer of indices with increased limit */

#ifdef FIX_MEM_REALLOC_IND_LIST

        if ( ( error = ind_list_realloc( *hBstr->ivas_ind_list_zero, *( hBstr->ivas_max_num_indices ) + STEP_MAX_NUM_INDICES, hBstr->st_ivas ) ) != IVAS_ERR_OK )

#else

        if ( ( error = ind_list_realloc( hBstr, *( hBstr->ivas_max_num_indices ) + STEP_MAX_NUM_INDICES ) ) != IVAS_ERR_OK )

#endif

        {

            return error;

        }
@@ -904,7 +1036,11 @@ ivas_error check_ind_list_limits( 
#endif



                /* no available empty slot -> need to re-allocate the buffer */

#ifdef FIX_MEM_REALLOC_IND_LIST

                if ( ( error = ind_list_realloc( *hBstr->ivas_ind_list_zero, *( hBstr->ivas_max_num_indices ) + STEP_MAX_NUM_INDICES, hBstr->st_ivas ) ) != IVAS_ERR_OK )

#else

                if ( ( error = ind_list_realloc( hBstr, *( hBstr->ivas_max_num_indices ) + STEP_MAX_NUM_INDICES ) ) != IVAS_ERR_OK )

#endif

                {

                    return error;

                }

lib_com/options.h

100644 → 100755
+21 −3
Original line number Diff line number Diff line
@@ -127,16 +127,20 @@ 
/*#define DEBUG_BINAURAL_FILTER_DESIGN*/        /* debugging of Crend binaural filter design */

/*#define DEBUG_AGC_ENCODER_CMD_OPTION*/        /* Ability to force enable or disable AGC behaviour in DIRAC/SPAR via command line option */

#define DEBUG_JBM_CMD_OPTION                    /* ability for telling the decoder the frontend fetch size and to not delay compensate for bad frames at the beginning */



#define VARIABLE_SPEED_DECODING                 /* variable speed decoding employing the JBM functioniality; move to DEBUGGING after build for disabled is fixed */

#define DEBUG_IND_LIST_MEMORY                   /* raise assert() when ind_list[] runs out of memory */

#endif



/* #################### End DEBUGGING switches ############################ */



/* ################# Start DEVELOPMENT switches ######################## */



/* keep as part of options.h */

#define BASOP_NOGLOB                                    /* Disable global symbols in BASOPs, Overflow/Carry in BASOPs disabled, additional BASOPs in case of Overflow */





/* ################# Start BE DEVELOPMENT switches ######################## */

/* only BE switches wrt operation points tested in selection */



/*#define FIX_I4_OL_PITCH*/                             /* fix open-loop pitch used for EVS core switching */
@@ -160,9 +164,23 @@ 
#define FIX_575_LOW_OVERLAP_PLC_RECOVERY                /* FhG: Issue 575 fix for PLC and transistion to TCX5*/

#define ISM_FB_16k4                                     /* VA: Issue: 579: change BW from SWB to FB in NxISM conditions to match the EVS codec */

#define FIX_580_PARAMMC_ENER_BURSTS                     /* FhG: issue 580: energy bursts due to ILD holding when energy relations change too much */

#define FIX_570_SF_EXT_ORIENTATION

#define FIX_593_STL_INCLUDE                             /* FhG: Issue 593: correct include of stl.h in lib_enc/ivas_stereo_eclvq_enc.c */

#define FIX_583_CLANG_TRANS_DET                         /* FhG: Issue 583: clang left shift on ramp_up_flag in transient detector */

#define NONBE_FIX_589_JBM_TC_OFFSETS                    /* FhG: issue 589: wrong offset into the TC buffers is used in some rendering paths in the JBM main rendering function */

#define FIX_MEM_REALLOC_IND_LIST                        /* VA: issue 601: failure of the automatic memory re-allocation mechanism when ind_list[] buffer is depleted in MASA mode with 2 TC*/

#define FIX_581_CLANG_OFFSET_TO_NULL                    /* FhG: issue 581: fix CLANG error about applying an offset to a NULL pointer */



/* ################## End BE DEVELOPMENT switches ######################### */





/* #################### Start NON-BE CR switches ########################## */

/* any switch which is non-be wrt operation points tested in selection */

/* all switches in this category should start with "CR_" */





/* ##################### End NON-BE CR switches ########################### */



/* ################## End DEVELOPMENT switches ######################### */

/* clang-format on */



#endif