From f91e7283c98ebb1896765342243d76b9b44e2932 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Thu, 22 Jun 2023 11:41:02 +0200 Subject: [PATCH 01/19] Checking in a few draft chunks of code --- lib_com/options.h | 2 + lib_dec/ivas_jbm_dec.c | 75 ++++++++++++++++++++++++++++++++ lib_dec/ivas_mc_paramupmix_dec.c | 32 ++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/lib_com/options.h b/lib_com/options.h index e51736a79f..46bb929d94 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -151,6 +151,8 @@ #define FIX_545_ASSERT /* VA: fix issue 545, replace assert() with warning message when hitting memory limit in the buffer of indices */ #endif +#define JBM_PARAMUPMIX + #define FIX_383_CLEAN_UP /* Dlb : Clean up of unused functions */ #define FIX_532_ISM_MD_INACTIVE /* VA: issue 532: improve MD coding in ISM inactive frames */ #define FIX_547_NAN_IGF_DEC /* FhG: issue 547: fix possible nan in IGF decoder */ diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 6871747db6..a0e3cb243b 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -350,6 +350,81 @@ ivas_error ivas_jbm_dec_tc( } } } +#ifdef JBM_PARAMUPMIX + else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) + { + ivas_lfe_dec( st_ivas->hLFE, st, output_frame, st_ivas->bfi, output_lfe_ch ); + + ivas_mc_paramupmix_dec_read_BS( st_ivas, st, st_ivas->hMCParamUpmix, &nb_bits_metadata[0] ); + + if ( ( error = ivas_mct_dec( st_ivas, output, output_frame, nb_bits_metadata[0] ) ) != IVAS_ERR_OK ) + { + return error; + } + + mvr2r( output_lfe_ch, output[LFE_CHANNEL], output_frame ); + + ivas_mc_paramupmix_dec( st_ivas, output ); + + /* HP filtering */ + for ( n = 0; n < st_ivas->nchan_transport; n++ ) + { + if ( n != LFE_CHANNEL ) + { + hp20( output[n], output_frame, st_ivas->mem_hp20_out[n], output_Fs ); + } + } + + if ( st_ivas->transport_config != st_ivas->intern_config && ( st_ivas->intern_config == AUDIO_CONFIG_FOA || st_ivas->intern_config == AUDIO_CONFIG_HOA2 || st_ivas->intern_config == AUDIO_CONFIG_HOA3 ) ) + { + ivas_mc2sba( st_ivas->hTransSetup, p_output, p_output, output_frame, st_ivas->hIntSetup.ambisonics_order, GAIN_LFE ); + } + + /* Rendering */ + if ( output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO ) + { + ivas_ls_setup_conversion( st_ivas, audioCfg2channels( AUDIO_CONFIG_5_1_2 ), output_frame, p_output, p_output ); + } +#if 0 + /* Rendering */ + if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) + { + if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig, + st_ivas->hCombinedOrientationData, + &st_ivas->hIntSetup, st_ivas->hEFAPdata, p_output, output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } + + ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output ); + } + else if ( st_ivas->renderer_type == RENDERER_MC ) + { + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO ) || ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_STEREO ) ) + { + ivas_ls_setup_conversion( st_ivas, audioCfg2channels( AUDIO_CONFIG_5_1_2 ), output_frame, p_output, p_output ); + } + else + { + ivas_ls_setup_conversion( st_ivas, MC_PARAMUPMIX_MAX_INPUT_CHANS, output_frame, p_output, p_output ); + } + } + else if ( st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) + { + ivas_mc2sba( st_ivas->hIntSetup, p_output, p_output, output_frame, st_ivas->hOutSetup.ambisonics_order, 0.f ); + } + else if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) + { + if ( ( ivas_td_binaural_renderer( st_ivas, p_output, output_frame ) ) != IVAS_ERR_OK ) + { + return error; + } + + ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output ); + } +#endif + } +#endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) { /* read Parametric MC parameters from the bitstream */ diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 6447c2f456..709ec3d531 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -307,6 +307,10 @@ ivas_error ivas_mc_paramupmix_dec_open( int16_t nchan_transport; uint16_t i; ivas_error error; +#ifdef JBM_PARAMUPMIX + int16_t num_channels_internal; + num_channels_internal = MC_PARAMUPMIX_MAX_TRANSPORT_CHANS; +#endif error = IVAS_ERR_OK; @@ -356,6 +360,34 @@ ivas_error ivas_mc_paramupmix_dec_open( set_zero( hMCParamUpmix->pcm_delay[i], 240 ); } +#ifdef JBM_PARAMUPMIX + /* allocate transport channels*/ + if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) + { + int16_t nchan_to_allocate; + int16_t nchan_tc; + TC_BUFFER_MODE buffer_mode; + + buffer_mode = TC_BUFFER_MODE_RENDERER; + nchan_tc = ivas_jbm_dec_get_num_tc_channels( st_ivas ); + nchan_to_allocate = num_channels_internal; + if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_STEREO || st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO ) + { + buffer_mode = TC_BUFFER_MODE_BUFFER; + nchan_tc = st_ivas->hDecoderConfig->nchan_out; + nchan_to_allocate = nchan_tc; + } + else if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + nchan_to_allocate = 2 * BINAURAL_CHANNELS; + } + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif + st_ivas->hMCParamUpmix = hMCParamUpmix; return error; -- GitLab From 41bdcfbb47a625c584e1ea062e4a8523b1ae0e97 Mon Sep 17 00:00:00 2001 From: Michael Schug Date: Fri, 23 Jun 2023 14:50:50 +0200 Subject: [PATCH 02/19] small changes to let the decoder run with -vs command line (without meaningful output): increased tc buffer channels to 12, disabled full ivas_mc_paramumix_dec function for jbm --- lib_dec/ivas_jbm_dec.c | 8 +++++++- lib_dec/ivas_mc_paramupmix_dec.c | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 9690f9a61b..b7cbd47e58 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -381,7 +381,7 @@ ivas_error ivas_jbm_dec_tc( mvr2r( output_lfe_ch, output[LFE_CHANNEL], output_frame ); - ivas_mc_paramupmix_dec( st_ivas, output ); + //ivas_mc_paramupmix_dec( st_ivas, output ); /* HP filtering */ for ( n = 0; n < st_ivas->nchan_transport; n++ ) @@ -919,6 +919,12 @@ ivas_error ivas_jbm_dec_render( ivas_binaural_add_LFE( st_ivas, *nSamplesRendered, st_ivas->hTcBuffer->tc, p_output ); } } +#ifdef JBM_PARAMUPMIX + else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) + { + *nSamplesRendered = min( st_ivas->hTcBuffer->n_samples_available, nSamplesAskedLocal ); + } +#endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) { ivas_param_mc_dec_render( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, p_output ); diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 709ec3d531..62bf88caf9 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -370,7 +370,7 @@ ivas_error ivas_mc_paramupmix_dec_open( buffer_mode = TC_BUFFER_MODE_RENDERER; nchan_tc = ivas_jbm_dec_get_num_tc_channels( st_ivas ); - nchan_to_allocate = num_channels_internal; + nchan_to_allocate = nchan_tc; if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_STEREO || st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO ) { buffer_mode = TC_BUFFER_MODE_BUFFER; -- GitLab From 446ffb36f67698191cafedd2662111329b0229d7 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 26 Jun 2023 15:36:51 +0200 Subject: [PATCH 03/19] Check-in beginning of ivas_jbm_dec_feed_tc_to_renderer operation --- lib_dec/ivas_jbm_dec.c | 6 +++ lib_dec/ivas_mc_paramupmix_dec.c | 88 ++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index b7cbd47e58..b5e2cbdbbe 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -670,6 +670,12 @@ ivas_error ivas_jbm_dec_feed_tc_to_renderer( { ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); } +#ifdef JBM_PARAMUPMIX + else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) + { + ivas_param_upmix_dec_decorr_subframes( st_ivas ); + } +#endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) { ivas_param_mc_dec_digest_tc( st_ivas, (uint8_t) n_render_timeslots, p_data_f ); diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 62bf88caf9..30ab30653d 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -427,7 +427,95 @@ void ivas_mc_paramupmix_dec_close( return; } +#ifdef JBM_PARAMUPMIX +/*------------------------------------------------------------------------- + * ivas_param_upmix_dec_decorr_subframes() + * + * + *------------------------------------------------------------------------*/ +static void paramupmix_td_decorr_process_jbm( + ivas_td_decorr_state_t *hTdDecorr[], /* i/o: SPAR Covar. decoder handle */ + float *pcm_in[], /* i : input audio channels */ + float **pp_out_pcm, /* o : output audio channels */ + const int16_t output_frame /* i : output frame length */ +) +{ + int16_t j, k; + int16_t offset; + float in_duck_gain[L_FRAME48k], out_duck_gain[L_FRAME48k]; + + offset = (int16_t) ( output_frame * FRAMES_PER_SEC * IVAS_DECORR_PARM_LOOKAHEAD_TAU ); + + /* Look-ahead delay */ + for ( k = 0; k < MC_PARAMUPMIX_COMBINATIONS; k++ ) + { + mvr2r( pcm_in[k], pp_out_pcm[k], output_frame ); + delay_signal( pp_out_pcm[k], output_frame, hTdDecorr[k]->look_ahead_buf, offset ); + + /* In ducking gains */ + if ( hTdDecorr[k]->ducking_flag ) + { + ivas_td_decorr_get_ducking_gains( hTdDecorr[k]->pTrans_det, pcm_in[k], in_duck_gain, out_duck_gain, output_frame, 0 ); + + for ( j = 0; j < output_frame; j++ ) + { + pp_out_pcm[k][j] = pp_out_pcm[k][j] * in_duck_gain[j]; + } + } + + /* All pass delay section */ + ivas_td_decorr_APD_iir_filter( &hTdDecorr[k]->APD_filt_state[0], pp_out_pcm[k], hTdDecorr[k]->num_apd_sections, output_frame ); + + /* Out ducking gains */ + if ( hTdDecorr[k]->ducking_flag ) + { + for ( j = 0; j < output_frame; j++ ) + { + pp_out_pcm[k][j] = pp_out_pcm[k][j] * out_duck_gain[j]; + } + } + } + + return; +} + +void ivas_param_upmix_dec_decorr_subframes( + Decoder_Struct *st_ivas /* i/o: IVAS decoder handle */ +) +{ + MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; + int16_t i, ch; + int32_t output_Fs; + int16_t output_frame; + float Pcm_decorr[MC_PARAMUPMIX_COMBINATIONS][L_FRAME48k]; /* decorrelated channels */ + float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ + float *p_tc[MC_PARAMUPMIX_MAX_INPUT_CHANS]; + + hMCParamUpmix = st_ivas->hMCParamUpmix; + assert( hMCParamUpmix ); + + push_wmops( "ivas_param_upmix_dec_decorr_subframes" ); + + output_Fs = st_ivas->hDecoderConfig->output_Fs; + output_frame = (int16_t) ( output_Fs / FRAMES_PER_SEC ); + for ( ch = 0; ch < MC_PARAMUPMIX_MAX_INPUT_CHANS; ch++ ) + { + p_tc[ch] = st_ivas->hTcBuffer->tc[ch]; + } + + for ( i = 0; i < MC_PARAMUPMIX_COMBINATIONS; i++ ) + { + pPcm_temp[i] = p_tc[8+i]; /* decorrelated */ + } + + paramupmix_td_decorr_process_jbm( hMCParamUpmix->hTdDecorr, &( p_tc[4] ), pPcm_temp, output_frame ); + + pop_wmops(); + + return; +} +#endif /*****************************************************************************************/ /* local functions */ -- GitLab From a7aa869de840c9e4cab0011793d0595a48ab2b03 Mon Sep 17 00:00:00 2001 From: Michael Schug Date: Tue, 27 Jun 2023 16:42:57 +0200 Subject: [PATCH 04/19] upmix working with subframes used in jbm decoder path; fixed number of tc buffer channels to 12 also for other than 7_1_4 configurations --- lib_com/ivas_prot.h | 16 ++ lib_dec/ivas_jbm_dec.c | 116 ++++++------ lib_dec/ivas_mc_paramupmix_dec.c | 297 ++++++++++++++++++++++++++++++- lib_dec/ivas_stat_dec.h | 3 + 4 files changed, 372 insertions(+), 60 deletions(-) diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 34226e640c..6014794be5 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -3877,6 +3877,22 @@ void ivas_mc_paramupmix_dec_read_BS( int16_t *nb_bits /* o : number of bits written */ ); +#ifdef JBM_PARAMUPMIX +void ivas_mc_paramupmix_dec_digest_tc( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ +); + +void ivas_mc_paramupmix_dec_render( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ + float *input_f[], /* i: core-coder transport channels */ + float *output_f[] /* i/o: synthesized core-coder transport channels */ +); +#endif + void ivas_param_mc_metadata_open( const MC_LS_SETUP mc_ls_setup, /* i : MC ls setup */ const int16_t lfe_index, /* i : channel index of LFE */ diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index b5e2cbdbbe..c86739c323 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -381,65 +381,23 @@ ivas_error ivas_jbm_dec_tc( mvr2r( output_lfe_ch, output[LFE_CHANNEL], output_frame ); - //ivas_mc_paramupmix_dec( st_ivas, output ); - - /* HP filtering */ - for ( n = 0; n < st_ivas->nchan_transport; n++ ) - { - if ( n != LFE_CHANNEL ) - { - hp20( output[n], output_frame, st_ivas->mem_hp20_out[n], output_Fs ); - } - } - - if ( st_ivas->transport_config != st_ivas->intern_config && ( st_ivas->intern_config == AUDIO_CONFIG_FOA || st_ivas->intern_config == AUDIO_CONFIG_HOA2 || st_ivas->intern_config == AUDIO_CONFIG_HOA3 ) ) - { - ivas_mc2sba( st_ivas->hTransSetup, p_output, p_output, output_frame, st_ivas->hIntSetup.ambisonics_order, GAIN_LFE ); - } - - /* Rendering */ - if ( output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO ) - { - ivas_ls_setup_conversion( st_ivas, audioCfg2channels( AUDIO_CONFIG_5_1_2 ), output_frame, p_output, p_output ); - } -#if 0 /* Rendering */ - if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) - { - if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig, - st_ivas->hCombinedOrientationData, - &st_ivas->hIntSetup, st_ivas->hEFAPdata, p_output, output_Fs ) ) != IVAS_ERR_OK ) - { - return error; - } - - ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output ); - } - else if ( st_ivas->renderer_type == RENDERER_MC ) + if ( st_ivas->renderer_type == RENDERER_MC ) { - if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO ) || ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_STEREO ) ) + if ( output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO ) { + /* HP filtering */ + for ( n = 0; n < st_ivas->nchan_transport; n++ ) + { + if ( n != LFE_CHANNEL ) + { + hp20( output[n], output_frame, st_ivas->mem_hp20_out[n], output_Fs ); + } + } ivas_ls_setup_conversion( st_ivas, audioCfg2channels( AUDIO_CONFIG_5_1_2 ), output_frame, p_output, p_output ); } - else - { - ivas_ls_setup_conversion( st_ivas, MC_PARAMUPMIX_MAX_INPUT_CHANS, output_frame, p_output, p_output ); - } } - else if ( st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) - { - ivas_mc2sba( st_ivas->hIntSetup, p_output, p_output, output_frame, st_ivas->hOutSetup.ambisonics_order, 0.f ); - } - else if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) - { - if ( ( ivas_td_binaural_renderer( st_ivas, p_output, output_frame ) ) != IVAS_ERR_OK ) - { - return error; - } - ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output ); - } -#endif } #endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) @@ -673,7 +631,7 @@ ivas_error ivas_jbm_dec_feed_tc_to_renderer( #ifdef JBM_PARAMUPMIX else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) { - ivas_param_upmix_dec_decorr_subframes( st_ivas ); + ivas_mc_paramupmix_dec_digest_tc( st_ivas, (uint8_t) n_render_timeslots ); } #endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) @@ -928,7 +886,51 @@ ivas_error ivas_jbm_dec_render( #ifdef JBM_PARAMUPMIX else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) { - *nSamplesRendered = min( st_ivas->hTcBuffer->n_samples_available, nSamplesAskedLocal ); + ivas_mc_paramupmix_dec_render( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, p_tc, p_output ); + + /* HP filtering */ + for ( n = 0; n < st_ivas->nchan_transport; n++ ) + { + if ( n != LFE_CHANNEL ) + { + hp20( p_output[n], *nSamplesRendered, st_ivas->mem_hp20_out[n], output_Fs ); + } + } + + if ( st_ivas->transport_config != st_ivas->intern_config && ( st_ivas->intern_config == AUDIO_CONFIG_FOA || st_ivas->intern_config == AUDIO_CONFIG_HOA2 || st_ivas->intern_config == AUDIO_CONFIG_HOA3 ) ) + { + ivas_mc2sba( st_ivas->hTransSetup, p_output, p_output, *nSamplesRendered, st_ivas->hIntSetup.ambisonics_order, GAIN_LFE ); + } + + /* Rendering */ + if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) + { + if ( ( error = ivas_rend_crendProcessSubframe( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig, + st_ivas->hCombinedOrientationData, + &st_ivas->hIntSetup, st_ivas->hEFAPdata, st_ivas->hTcBuffer, p_output, p_output, *nSamplesRendered, output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } + + ivas_binaural_add_LFE( st_ivas, *nSamplesRendered, p_output, p_output ); + } + else if ( st_ivas->renderer_type == RENDERER_MC ) + { + ivas_ls_setup_conversion( st_ivas, MC_PARAMUPMIX_MAX_INPUT_CHANS, *nSamplesRendered, p_output, p_output ); + } + else if ( st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) + { + ivas_mc2sba( st_ivas->hIntSetup, p_output, p_output, *nSamplesRendered, st_ivas->hOutSetup.ambisonics_order, 0.f ); + } + else if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) + { + if ( ( ivas_td_binaural_renderer( st_ivas, p_output, *nSamplesRendered ) ) != IVAS_ERR_OK ) + { + return error; + } + + ivas_binaural_add_LFE( st_ivas, *nSamplesRendered, p_output, p_output ); + } } #endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) @@ -1489,6 +1491,12 @@ int16_t ivas_jbm_dec_get_num_tc_channels( num_tc = st_ivas->hOutSetup.nchan_out_woLFE + st_ivas->hOutSetup.num_lfe; } } +#ifdef JBM_PARAMUPMIX + else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) + { + num_tc = MC_PARAMUPMIX_MAX_INPUT_CHANS; + } +#endif else if ( st_ivas->mc_mode == MC_MODE_MCMASA ) { if ( st_ivas->hOutSetup.separateChannelEnabled ) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 30ab30653d..d23be11bf5 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -55,6 +55,14 @@ static void ps_pred_process( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, float qmf_mod_re[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t ch ); +#ifdef JBM_PARAMUPMIX +static void ps_pred_process_sf( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, DECODER_TC_BUFFER_HANDLE hTcBuffer, float qmf_mod_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float *param_interpol, const int16_t ch ); + +static void ivas_mc_paramupmix_dec_sf( Decoder_Struct *st_ivas, float *output_f[MAX_OUTPUT_CHANNELS] ); + +static void ivas_param_upmix_dec_decorr_subframes( Decoder_Struct *st_ivas ); +#endif + static void paramupmix_td_decorr_process( ivas_td_decorr_state_t *hTdDecorr[], float pcm_in[][L_FRAME48k], float **pp_out_pcm, const int16_t output_frame ); static int huff_read( Decoder_State *st, const int16_t ( *ht )[2] ); @@ -292,6 +300,99 @@ void ivas_mc_paramupmix_dec( } +#ifdef JBM_PARAMUPMIX +void ivas_mc_paramupmix_dec_digest_tc( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ +) +{ + MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; + hMCParamUpmix = st_ivas->hMCParamUpmix; + assert( hMCParamUpmix ); + + push_wmops( "ivas_mc_paramupmix_dec_digest_tc" ); + + ivas_param_upmix_dec_decorr_subframes( st_ivas ); + + /* adapt subframes */ + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); + + pop_wmops(); +} + + +void ivas_mc_paramupmix_dec_render( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ + float *input_f[], /* i: core-coder transport channels */ + float *output_f[] /* i/o: synthesized core-coder transport channels */ +) +{ + int16_t slots_to_render, first_sf, last_sf, subframe_idx; + uint16_t slot_size, ch; + float *output_f_local[MAX_OUTPUT_CHANNELS]; + MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; + hMCParamUpmix = st_ivas->hMCParamUpmix; + assert( hMCParamUpmix ); + + push_wmops( "ivas_mc_paramupmix_dec_render" ); + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + output_f_local[ch] = output_f[ch]; + } + + slot_size = NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ); + + /* loop for synthesis, assume we always have to render in multiples of 5ms subframes with spills */ + slots_to_render = min( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered, nSamplesAsked / slot_size ); + *nSamplesRendered = slots_to_render * slot_size; + first_sf = st_ivas->hTcBuffer->subframes_rendered; + last_sf = first_sf; + + for ( ch = 0; ch < MAX_TRANSPORT_CHANNELS; ch++ ) + { + mvr2r( input_f[ch], output_f_local[ch], *nSamplesRendered); + } + + while ( slots_to_render > 0 ) + { + slots_to_render -= st_ivas->hTcBuffer->subframe_nbslots[last_sf]; + last_sf++; + } +#ifdef DEBUGGING + assert( slots_to_render == 0 ); +#endif + + for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) + { + int16_t n_samples_sf = slot_size * st_ivas->hTcBuffer->subframe_nbslots[subframe_idx]; + + ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local ); + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + output_f_local[ch] += n_samples_sf; + } + } + + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + mvr2r( hMCParamUpmix->alphas[ch], hMCParamUpmix->alpha_prev[ch], IVAS_MAX_NUM_BANDS ); + mvr2r( hMCParamUpmix->betas[ch], hMCParamUpmix->beta_prev[ch], IVAS_MAX_NUM_BANDS ); + } + + *nSamplesAvailable = ( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered ) * slot_size; + + pop_wmops(); +} +#endif + + + /*------------------------------------------------------------------------- * ivas_mc_paramupmix_dec_open() * @@ -307,10 +408,6 @@ ivas_error ivas_mc_paramupmix_dec_open( int16_t nchan_transport; uint16_t i; ivas_error error; -#ifdef JBM_PARAMUPMIX - int16_t num_channels_internal; - num_channels_internal = MC_PARAMUPMIX_MAX_TRANSPORT_CHANS; -#endif error = IVAS_ERR_OK; @@ -385,6 +482,11 @@ ivas_error ivas_mc_paramupmix_dec_open( { return error; } + if ( ( hMCParamUpmix->param_interpolator = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for interpolator\n" ) ); + } + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } #endif @@ -421,6 +523,12 @@ void ivas_mc_paramupmix_dec_close( free( ( *hMCParamUpmix )->pcm_delay[i] ); } } +#ifdef JBM_PARAMUPMIX + if ( ( *hMCParamUpmix )->param_interpolator != NULL ) + { + free( ( *hMCParamUpmix )->param_interpolator ); + } +#endif free( *hMCParamUpmix ); *hMCParamUpmix = NULL; @@ -479,7 +587,7 @@ static void paramupmix_td_decorr_process_jbm( return; } -void ivas_param_upmix_dec_decorr_subframes( +static void ivas_param_upmix_dec_decorr_subframes( Decoder_Struct *st_ivas /* i/o: IVAS decoder handle */ ) { @@ -487,7 +595,6 @@ void ivas_param_upmix_dec_decorr_subframes( int16_t i, ch; int32_t output_Fs; int16_t output_frame; - float Pcm_decorr[MC_PARAMUPMIX_COMBINATIONS][L_FRAME48k]; /* decorrelated channels */ float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ float *p_tc[MC_PARAMUPMIX_MAX_INPUT_CHANS]; @@ -584,6 +691,184 @@ static void ps_pred_process( return; } +#ifdef JBM_PARAMUPMIX +static void ps_pred_process_sf( + MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, + DECODER_TC_BUFFER_HANDLE hTcBuffer, + float qmf_mod_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], /* in/out */ + float qmf_mod_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], /* in/out */ + float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float *param_interpol, + const int16_t ch ) +{ + float vmre, vmim, vsre, vsim; + int16_t iqmf, ipar, ismp, iismp; + float alpha_smp, beta_smp; + float *alpha1, *alpha2; + float *beta1, *beta2; + float *alpha_prev = hMCParamUpmix->alpha_prev[ch]; + float *beta_prev = hMCParamUpmix->beta_prev[ch]; + + const int16_t qmf_to_par_band[] = { + 0, 1, 2, 3, 4, 5, 5, 6, 6, 7, + 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, + 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 + }; + + for ( iqmf = 0; iqmf < CLDFB_NO_CHANNELS_MAX; iqmf++ ) + { + /* For changing no of parameter bands (ipar1 != ipar2), TIGGER_FRAMING assumed */ + ipar = qmf_to_par_band[iqmf]; + alpha1 = alpha_prev; + beta1 = beta_prev; + + ismp = 0; + alpha2 = hMCParamUpmix->alphas[ch]; + beta2 = hMCParamUpmix->betas[ch]; + alpha_smp = alpha1[ipar]; + beta_smp = beta1[ipar]; + + for ( iismp = 0; iismp < hTcBuffer->subframe_nbslots[hTcBuffer->subframes_rendered]; iismp++ ) + { + alpha_smp = alpha1[ipar] + ( alpha2[ipar] - alpha1[ipar] ) * param_interpol[iismp]; + beta_smp = beta1[ipar] + ( beta2[ipar] - beta1[ipar] ) * param_interpol[iismp]; + + vmre = qmf_mod_re[ismp][iqmf]; + vmim = qmf_mod_im[ismp][iqmf]; + vsre = qmf_side_re[ismp][iqmf]; + vsim = qmf_side_im[ismp][iqmf]; + + qmf_side_re[ismp][iqmf] = alpha_smp * vmre + beta_smp * vsre; + qmf_side_im[ismp][iqmf] = alpha_smp * vmim + beta_smp * vsim; + + ismp++; + } + } + + return; +} + + +static void ivas_mc_paramupmix_dec_sf( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + float *output_f[MAX_OUTPUT_CHANNELS] /* i/o: synthesized core-coder transport channels */ +) +{ + int16_t i, ch, slot_idx, k; + float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ + /*CLDFB*/ + float Cldfb_RealBuffer[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + int16_t noparamupmix_delay, n_samples_rendered; + MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; + hMCParamUpmix = st_ivas->hMCParamUpmix; + assert( hMCParamUpmix ); + + push_wmops( "ivas_mc_paramupmix_dec_sf" ); + + for ( i = 0; i < MC_PARAMUPMIX_COMBINATIONS; i++ ) + { + pPcm_temp[2 * i] = output_f[i + 4]; /* un-decorrelated */ + pPcm_temp[2 * i + 1] = output_f[i + 8]; /* decorrelated */ + } + + /* CLDFB Analysis*/ + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) + { + /* slot loop for gathering the input data */ + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + cldfbAnalysis_ts( &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), Cldfb_RealBuffer[ch][slot_idx], Cldfb_ImagBuffer[ch][slot_idx], hMCParamUpmix->num_freq_bands, st_ivas->cldfbAnaDec[ch] ); + } + } + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + ps_pred_process_sf( hMCParamUpmix, + st_ivas->hTcBuffer, + Cldfb_RealBuffer[2 * ch], /* in/out */ + Cldfb_ImagBuffer[2 * ch], + Cldfb_RealBuffer[2 * ch + 1], /* in/out decorr */ + Cldfb_ImagBuffer[2 * ch + 1], + &hMCParamUpmix->param_interpolator[st_ivas->hTcBuffer->slots_rendered], + ch ); + + /*-- m, s -> l, r ----------------------------*/ + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + for ( k = 0; k < CLDFB_NO_CHANNELS_MAX; k++ ) + { + float qlre = Cldfb_RealBuffer[2 * ch][slot_idx][k]; + float qlim = Cldfb_ImagBuffer[2 * ch][slot_idx][k]; + float qrre = Cldfb_RealBuffer[2 * ch + 1][slot_idx][k]; + float qrim = Cldfb_ImagBuffer[2 * ch + 1][slot_idx][k]; + + Cldfb_RealBuffer[2 * ch][slot_idx][k] = qlre + qrre; + Cldfb_ImagBuffer[2 * ch][slot_idx][k] = qlim + qrim; + Cldfb_RealBuffer[2 * ch + 1][slot_idx][k] = qlre - qrre; + Cldfb_ImagBuffer[2 * ch + 1][slot_idx][k] = qlim - qrim; + } + } + } + /* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */ + pPcm_temp[0] = output_f[4]; + pPcm_temp[1] = output_f[6]; + pPcm_temp[2] = output_f[5]; + pPcm_temp[3] = output_f[7]; + pPcm_temp[4] = output_f[8]; + pPcm_temp[5] = output_f[10]; + pPcm_temp[6] = output_f[9]; + pPcm_temp[7] = output_f[11]; + + /* CLDFB synthesis */ + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) + { + for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) + { + float *ptr_im[1], *ptr_re[1]; + ptr_re[0] = Cldfb_RealBuffer[ch][slot_idx]; + ptr_im[0] = Cldfb_ImagBuffer[ch][slot_idx]; + + cldfbSynthesis( ptr_re, ptr_im, &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ), + hMCParamUpmix->num_freq_bands, st_ivas->cldfbSynDec[ch] ); + } + } + /* adjust delay of other channels */ + noparamupmix_delay = NS2SA( st_ivas->hDecoderConfig->output_Fs, IVAS_FB_DEC_DELAY_NS ); + n_samples_rendered = st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered] * hMCParamUpmix->num_freq_bands; + if (n_samples_rendered > noparamupmix_delay) + { + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + float tmp_buf[L_SUBFRAME5MS_48k]; + mvr2r( &output_f[ch][n_samples_rendered - noparamupmix_delay], tmp_buf, noparamupmix_delay ); + mvr2r( output_f[ch], &output_f[ch][noparamupmix_delay], n_samples_rendered - noparamupmix_delay ); + mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], noparamupmix_delay ); + mvr2r( tmp_buf, hMCParamUpmix->pcm_delay[ch], noparamupmix_delay ); + } + } + else + { + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + float tmp_buf[L_SUBFRAME5MS_48k]; + mvr2r( &output_f[ch][0], tmp_buf, n_samples_rendered ); + mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], n_samples_rendered ); + mvr2r( &hMCParamUpmix->pcm_delay[ch][n_samples_rendered], &hMCParamUpmix->pcm_delay[ch][0], noparamupmix_delay - n_samples_rendered ); + mvr2r( tmp_buf, &hMCParamUpmix->pcm_delay[ch][noparamupmix_delay-n_samples_rendered], n_samples_rendered ); + } + } + + st_ivas->hTcBuffer->slots_rendered += st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; + st_ivas->hTcBuffer->subframes_rendered++; + + pop_wmops(); +} +#endif + static void paramupmix_td_decorr_process( ivas_td_decorr_state_t *hTdDecorr[], /* i/o: SPAR Covar. decoder handle */ diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 315cfbad66..697a577c87 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -802,6 +802,9 @@ typedef struct ivas_mc_paramupmix_dec_data_structure int32_t beta_quant[MC_PARAMUPMIX_COMBINATIONS][IVAS_MAX_NUM_BANDS]; int16_t first_frame; float *pcm_delay[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS]; +#ifdef JBM_PARAMUPMIX + float *param_interpolator; +#endif } MC_PARAMUPMIX_DEC_DATA, *MC_PARAMUPMIX_DEC_HANDLE; -- GitLab From 15281eaf66487c23e903fd0c30591db5f605609c Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Fri, 30 Jun 2023 09:32:42 +0200 Subject: [PATCH 05/19] Fix binaural crash --- lib_dec/ivas_jbm_dec.c | 3 +- lib_dec/ivas_mc_paramupmix_dec.c | 58 +++++++++++++++++++------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index c86739c323..9119451443 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -397,7 +397,6 @@ ivas_error ivas_jbm_dec_tc( ivas_ls_setup_conversion( st_ivas, audioCfg2channels( AUDIO_CONFIG_5_1_2 ), output_frame, p_output, p_output ); } } - } #endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) @@ -1494,7 +1493,7 @@ int16_t ivas_jbm_dec_get_num_tc_channels( #ifdef JBM_PARAMUPMIX else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) { - num_tc = MC_PARAMUPMIX_MAX_INPUT_CHANS; + num_tc = MC_PARAMUPMIX_MAX_INPUT_CHANS; } #endif else if ( st_ivas->mc_mode == MC_MODE_MCMASA ) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index d23be11bf5..421f008396 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -302,8 +302,8 @@ void ivas_mc_paramupmix_dec( #ifdef JBM_PARAMUPMIX void ivas_mc_paramupmix_dec_digest_tc( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ ) { MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; @@ -311,7 +311,7 @@ void ivas_mc_paramupmix_dec_digest_tc( assert( hMCParamUpmix ); push_wmops( "ivas_mc_paramupmix_dec_digest_tc" ); - + ivas_param_upmix_dec_decorr_subframes( st_ivas ); /* adapt subframes */ @@ -324,12 +324,12 @@ void ivas_mc_paramupmix_dec_digest_tc( void ivas_mc_paramupmix_dec_render( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ - uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ - uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ - float *input_f[], /* i: core-coder transport channels */ - float *output_f[] /* i/o: synthesized core-coder transport channels */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ + float *input_f[], /* i: core-coder transport channels */ + float *output_f[] /* i/o: synthesized core-coder transport channels */ ) { int16_t slots_to_render, first_sf, last_sf, subframe_idx; @@ -346,7 +346,7 @@ void ivas_mc_paramupmix_dec_render( output_f_local[ch] = output_f[ch]; } - slot_size = NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ); + slot_size = st_ivas->hTcBuffer->n_samples_granularity; /* loop for synthesis, assume we always have to render in multiples of 5ms subframes with spills */ slots_to_render = min( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered, nSamplesAsked / slot_size ); @@ -356,7 +356,7 @@ void ivas_mc_paramupmix_dec_render( for ( ch = 0; ch < MAX_TRANSPORT_CHANNELS; ch++ ) { - mvr2r( input_f[ch], output_f_local[ch], *nSamplesRendered); + mvr2r( input_f[ch], output_f_local[ch], *nSamplesRendered ); } while ( slots_to_render > 0 ) @@ -387,12 +387,19 @@ void ivas_mc_paramupmix_dec_render( *nSamplesAvailable = ( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered ) * slot_size; + /* Binaural renderer will do the slot rendering */ + if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) + { + st_ivas->hTcBuffer->slots_rendered = 0; + st_ivas->hTcBuffer->subframes_rendered = 0; + } + + pop_wmops(); } #endif - /*------------------------------------------------------------------------- * ivas_mc_paramupmix_dec_open() * @@ -478,9 +485,12 @@ ivas_error ivas_mc_paramupmix_dec_open( { nchan_to_allocate = 2 * BINAURAL_CHANNELS; } - if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) + if ( st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV && st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV_ROOM ) { - return error; + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) + { + return error; + } } if ( ( hMCParamUpmix->param_interpolator = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * sizeof( float ) ) ) == NULL ) { @@ -588,14 +598,14 @@ static void paramupmix_td_decorr_process_jbm( } static void ivas_param_upmix_dec_decorr_subframes( - Decoder_Struct *st_ivas /* i/o: IVAS decoder handle */ + Decoder_Struct *st_ivas /* i/o: IVAS decoder handle */ ) { MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; int16_t i, ch; int32_t output_Fs; int16_t output_frame; - float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ + float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ float *p_tc[MC_PARAMUPMIX_MAX_INPUT_CHANS]; hMCParamUpmix = st_ivas->hMCParamUpmix; @@ -613,7 +623,7 @@ static void ivas_param_upmix_dec_decorr_subframes( for ( i = 0; i < MC_PARAMUPMIX_COMBINATIONS; i++ ) { - pPcm_temp[i] = p_tc[8+i]; /* decorrelated */ + pPcm_temp[i] = p_tc[8 + i]; /* decorrelated */ } paramupmix_td_decorr_process_jbm( hMCParamUpmix->hTdDecorr, &( p_tc[4] ), pPcm_temp, output_frame ); @@ -753,13 +763,13 @@ static void ps_pred_process_sf( } -static void ivas_mc_paramupmix_dec_sf( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - float *output_f[MAX_OUTPUT_CHANNELS] /* i/o: synthesized core-coder transport channels */ +static void ivas_mc_paramupmix_dec_sf( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + float *output_f[MAX_OUTPUT_CHANNELS] /* i/o: synthesized core-coder transport channels */ ) { int16_t i, ch, slot_idx, k; - float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ + float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ /*CLDFB*/ float Cldfb_RealBuffer[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; @@ -772,7 +782,7 @@ static void ivas_mc_paramupmix_dec_sf( for ( i = 0; i < MC_PARAMUPMIX_COMBINATIONS; i++ ) { - pPcm_temp[2 * i] = output_f[i + 4]; /* un-decorrelated */ + pPcm_temp[2 * i] = output_f[i + 4]; /* un-decorrelated */ pPcm_temp[2 * i + 1] = output_f[i + 8]; /* decorrelated */ } @@ -839,7 +849,7 @@ static void ivas_mc_paramupmix_dec_sf( /* adjust delay of other channels */ noparamupmix_delay = NS2SA( st_ivas->hDecoderConfig->output_Fs, IVAS_FB_DEC_DELAY_NS ); n_samples_rendered = st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered] * hMCParamUpmix->num_freq_bands; - if (n_samples_rendered > noparamupmix_delay) + if ( n_samples_rendered > noparamupmix_delay ) { for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) { @@ -858,7 +868,7 @@ static void ivas_mc_paramupmix_dec_sf( mvr2r( &output_f[ch][0], tmp_buf, n_samples_rendered ); mvr2r( hMCParamUpmix->pcm_delay[ch], output_f[ch], n_samples_rendered ); mvr2r( &hMCParamUpmix->pcm_delay[ch][n_samples_rendered], &hMCParamUpmix->pcm_delay[ch][0], noparamupmix_delay - n_samples_rendered ); - mvr2r( tmp_buf, &hMCParamUpmix->pcm_delay[ch][noparamupmix_delay-n_samples_rendered], n_samples_rendered ); + mvr2r( tmp_buf, &hMCParamUpmix->pcm_delay[ch][noparamupmix_delay - n_samples_rendered], n_samples_rendered ); } } -- GitLab From d9deaa78ac41b63ce1ece98aaf41f833725a6e9a Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Fri, 30 Jun 2023 09:57:02 +0200 Subject: [PATCH 06/19] Adapt interpolators --- lib_dec/ivas_mc_paramupmix_dec.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 421f008396..d1e25c18e1 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -317,8 +317,15 @@ void ivas_mc_paramupmix_dec_digest_tc( /* adapt subframes */ ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); - + if (st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM) + { + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_SUBFRAMES_5MS, nCldfbSlots, hMCParamUpmix->param_interpolator ); + } + else + { + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); + } + pop_wmops(); } @@ -394,7 +401,6 @@ void ivas_mc_paramupmix_dec_render( st_ivas->hTcBuffer->subframes_rendered = 0; } - pop_wmops(); } #endif @@ -475,6 +481,11 @@ ivas_error ivas_mc_paramupmix_dec_open( buffer_mode = TC_BUFFER_MODE_RENDERER; nchan_tc = ivas_jbm_dec_get_num_tc_channels( st_ivas ); nchan_to_allocate = nchan_tc; + if ( ( hMCParamUpmix->param_interpolator = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for interpolator\n" ) ); + } + if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_STEREO || st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO ) { buffer_mode = TC_BUFFER_MODE_BUFFER; @@ -485,18 +496,18 @@ ivas_error ivas_mc_paramupmix_dec_open( { nchan_to_allocate = 2 * BINAURAL_CHANNELS; } - if ( st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV && st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV_ROOM ) + if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) + { + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_SUBFRAMES_5MS, DEFAULT_JBM_SUBFRAMES_5MS, hMCParamUpmix->param_interpolator ); + } + else { if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) { return error; } + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } - if ( ( hMCParamUpmix->param_interpolator = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * sizeof( float ) ) ) == NULL ) - { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for interpolator\n" ) ); - } - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } #endif -- GitLab From d63fd0684518506f6445a94d9a68b387b810edb7 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 3 Jul 2023 13:27:13 +0200 Subject: [PATCH 07/19] Change granularity of MIXER_CONV for PARAMUPMIX --- lib_com/ivas_prot.h | 6 +++++- lib_dec/ivas_init_dec.c | 11 +++++++++++ lib_dec/ivas_ism_dec.c | 5 ++++- lib_dec/ivas_jbm_dec.c | 31 ++++++++++++++++++++++++++++++- lib_dec/ivas_mc_paramupmix_dec.c | 14 +++++--------- lib_dec/ivas_mct_dec.c | 4 ++++ 6 files changed, 59 insertions(+), 12 deletions(-) diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 6014794be5..66eb5b6233 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -837,7 +837,11 @@ TC_BUFFER_MODE ivas_jbm_dec_get_tc_buffer_mode( /*! r: render granularity */ int16_t ivas_jbm_dec_get_render_granularity( const RENDERER_TYPE rendererType, /* i : renderer type */ - const int32_t output_Fs /* i : sampling rate */ +#ifdef JBM_PARAMUPMIX + const IVAS_FORMAT ivas_format, /* i : ivas format */ + const MC_MODE mc_mode, /* i : MC mode */ +#endif + const int32_t output_Fs /* i : sampling rate */ ); ivas_error ivas_jbm_dec_tc_buffer_open( diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 2294203053..0e63166be3 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1332,7 +1332,18 @@ ivas_error ivas_init_decoder( if ( st_ivas->hDecoderConfig->voip_active ) { +#ifdef JBM_PARAMUPMIX + if ( ( st_ivas->ivas_format == MC_FORMAT ) && ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) ) + { + granularity = NS2SA( output_Fs, CLDFB_SLOT_NS ); + } + else + { + granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); + } +#else granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); +#endif n_channels_transport_jbm = ivas_jbm_dec_get_num_tc_channels( st_ivas ); if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, n_channels_transport_jbm, n_channels_transport_jbm, n_channels_transport_jbm, granularity ) ) != IVAS_ERR_OK ) diff --git a/lib_dec/ivas_ism_dec.c b/lib_dec/ivas_ism_dec.c index c031ec8c4b..b3c5acc2e4 100644 --- a/lib_dec/ivas_ism_dec.c +++ b/lib_dec/ivas_ism_dec.c @@ -133,8 +133,11 @@ static ivas_error ivas_ism_bitrate_switching( /* JBM: when granularity goes down (e.g. Discrete ISM with TD Obj Renderer -> ParamISM with binaural fastconv render what still fits in the new granularity */ +#ifdef JBM_PARAMUPMIX + tc_granularity_new = ivas_jbm_dec_get_render_granularity( st_ivas->renderer_type, st_ivas->ivas_format, st_ivas->mc_mode, st_ivas->hDecoderConfig->output_Fs ); +#else tc_granularity_new = ivas_jbm_dec_get_render_granularity( st_ivas->renderer_type, st_ivas->hDecoderConfig->output_Fs ); - +#endif if ( tc_granularity_new < st_ivas->hTcBuffer->n_samples_granularity ) { if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, tc_granularity_new, renderer_type_old, intern_config_old, &hIntSetupOld, MC_MODE_NONE, last_ism_mode, nSamplesRendered, data ) ) != IVAS_ERR_OK ) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 9119451443..f77beed717 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -1579,7 +1579,36 @@ static void ivas_jbm_dec_copy_tc( * * *--------------------------------------------------------------------------*/ +#ifdef JBM_PARAMUPMIX +/*! r: render granularity */ +int16_t ivas_jbm_dec_get_render_granularity( + const RENDERER_TYPE rendererType, /* i : renderer type */ + const IVAS_FORMAT ivas_format, /* i : ivas format */ + const MC_MODE mc_mode, /* i : MC mode */ + const int32_t output_Fs /* i : sampling rate */ +) +{ + int16_t render_granularity; + + if ( rendererType == RENDERER_BINAURAL_OBJECTS_TD || rendererType == RENDERER_BINAURAL_MIXER_CONV || rendererType == RENDERER_BINAURAL_MIXER_CONV_ROOM ) + { + if ( ( ivas_format == MC_FORMAT ) && ( mc_mode == MC_MODE_PARAMUPMIX ) ) + { + render_granularity = NS2SA( output_Fs, CLDFB_SLOT_NS ); + } + else + { + render_granularity = NS2SA( output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); + } + } + else + { + render_granularity = NS2SA( output_Fs, CLDFB_SLOT_NS ); + } + return render_granularity; +} +#else /*! r: render granularity */ int16_t ivas_jbm_dec_get_render_granularity( const RENDERER_TYPE rendererType, /* i : renderer type */ @@ -1599,7 +1628,7 @@ int16_t ivas_jbm_dec_get_render_granularity( return render_granularity; } - +#endif /*--------------------------------------------------------------------------* * ivas_jbm_dec_tc_buffer_open() diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index d1e25c18e1..98385d9394 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -317,15 +317,15 @@ void ivas_mc_paramupmix_dec_digest_tc( /* adapt subframes */ ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); - if (st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM) + if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) { - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_SUBFRAMES_5MS, nCldfbSlots, hMCParamUpmix->param_interpolator ); + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); } else { ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); } - + pop_wmops(); } @@ -496,18 +496,14 @@ ivas_error ivas_mc_paramupmix_dec_open( { nchan_to_allocate = 2 * BINAURAL_CHANNELS; } - if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) - { - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_SUBFRAMES_5MS, DEFAULT_JBM_SUBFRAMES_5MS, hMCParamUpmix->param_interpolator ); - } - else + if ( st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV && st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV_ROOM ) { if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) { return error; } - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } #endif diff --git a/lib_dec/ivas_mct_dec.c b/lib_dec/ivas_mct_dec.c index 51d6856987..acdd30811b 100755 --- a/lib_dec/ivas_mct_dec.c +++ b/lib_dec/ivas_mct_dec.c @@ -748,7 +748,11 @@ static ivas_error ivas_mc_dec_reconfig( /* JBM: when granularity goes down (e.g. MCT with CREND -> ParamMC with binaural fastconv render what still fits in the new granularity */ +#ifdef JBM_PARAMUPMIX + tc_granularity_new = ivas_jbm_dec_get_render_granularity( st_ivas->renderer_type, st_ivas->ivas_format, st_ivas->mc_mode, st_ivas->hDecoderConfig->output_Fs ); +#else tc_granularity_new = ivas_jbm_dec_get_render_granularity( st_ivas->renderer_type, st_ivas->hDecoderConfig->output_Fs ); +#endif if ( tc_granularity_new < st_ivas->hTcBuffer->n_samples_granularity ) { if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, tc_granularity_new, renderer_type_old, intern_config_old, &hIntSetupOld, last_mc_mode, ISM_MODE_NONE, nSamplesRendered, data ) ) != IVAS_ERR_OK ) -- GitLab From 8295a205d3cd2c3778769754886fd334be186d96 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Tue, 4 Jul 2023 09:23:19 +0200 Subject: [PATCH 08/19] Fix memory error --- lib_dec/ivas_mc_paramupmix_dec.c | 4 +++- lib_dec/ivas_stat_dec.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 98385d9394..318f7f1705 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -472,6 +472,7 @@ ivas_error ivas_mc_paramupmix_dec_open( #ifdef JBM_PARAMUPMIX /* allocate transport channels*/ + hMCParamUpmix->free_param_interpolator = 0; if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) { int16_t nchan_to_allocate; @@ -502,6 +503,7 @@ ivas_error ivas_mc_paramupmix_dec_open( { return error; } + hMCParamUpmix->free_param_interpolator = 1; } ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } @@ -541,7 +543,7 @@ void ivas_mc_paramupmix_dec_close( } } #ifdef JBM_PARAMUPMIX - if ( ( *hMCParamUpmix )->param_interpolator != NULL ) + if ( ( ( *hMCParamUpmix )->param_interpolator != NULL ) && ( ( *hMCParamUpmix )->free_param_interpolator == 1 ) ) { free( ( *hMCParamUpmix )->param_interpolator ); } diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 697a577c87..152cf5091e 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -804,6 +804,7 @@ typedef struct ivas_mc_paramupmix_dec_data_structure float *pcm_delay[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS]; #ifdef JBM_PARAMUPMIX float *param_interpolator; + int16_t free_param_interpolator; #endif } MC_PARAMUPMIX_DEC_DATA, *MC_PARAMUPMIX_DEC_HANDLE; -- GitLab From 40008d54638d967cd14c988aa03a2863f3bbd797 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Tue, 4 Jul 2023 12:16:29 +0200 Subject: [PATCH 09/19] Decorrelation JBM split up --- lib_com/ivas_prot.h | 3 +- lib_dec/ivas_jbm_dec.c | 2 +- lib_dec/ivas_mc_paramupmix_dec.c | 48 +++++++++++++++++++------------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index a185318a7a..5f18f27668 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -3884,7 +3884,8 @@ void ivas_mc_paramupmix_dec_read_BS( #ifdef JBM_PARAMUPMIX void ivas_mc_paramupmix_dec_digest_tc( Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ + const uint8_t nCldfbSlots, /* i : number of CLFBS slots in the transport channels */ + const int16_t nSamplesForRendering /* i : number of samples provided */ ); void ivas_mc_paramupmix_dec_render( diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 9fffa4a0f5..3b26ef44e1 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -625,7 +625,7 @@ ivas_error ivas_jbm_dec_feed_tc_to_renderer( #ifdef JBM_PARAMUPMIX else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) { - ivas_mc_paramupmix_dec_digest_tc( st_ivas, (uint8_t) n_render_timeslots ); + ivas_mc_paramupmix_dec_digest_tc( st_ivas, (uint8_t) n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); } #endif else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 318f7f1705..d63a67302c 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -60,7 +60,7 @@ static void ps_pred_process_sf( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, DECODER_ static void ivas_mc_paramupmix_dec_sf( Decoder_Struct *st_ivas, float *output_f[MAX_OUTPUT_CHANNELS] ); -static void ivas_param_upmix_dec_decorr_subframes( Decoder_Struct *st_ivas ); +static void ivas_param_upmix_dec_decorr_subframes( Decoder_Struct *st_ivas, const int16_t nSamplesForRendering ); #endif static void paramupmix_td_decorr_process( ivas_td_decorr_state_t *hTdDecorr[], float pcm_in[][L_FRAME48k], float **pp_out_pcm, const int16_t output_frame ); @@ -302,8 +302,9 @@ void ivas_mc_paramupmix_dec( #ifdef JBM_PARAMUPMIX void ivas_mc_paramupmix_dec_digest_tc( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint8_t nCldfbSlots, /* i : number of CLFBS slots in the transport channels */ + const int16_t nSamplesForRendering /* i : number of samples provided */ ) { MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; @@ -312,7 +313,7 @@ void ivas_mc_paramupmix_dec_digest_tc( push_wmops( "ivas_mc_paramupmix_dec_digest_tc" ); - ivas_param_upmix_dec_decorr_subframes( st_ivas ); + ivas_param_upmix_dec_decorr_subframes( st_ivas, nSamplesForRendering ); /* adapt subframes */ ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); @@ -607,35 +608,44 @@ static void paramupmix_td_decorr_process_jbm( } static void ivas_param_upmix_dec_decorr_subframes( - Decoder_Struct *st_ivas /* i/o: IVAS decoder handle */ -) + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const int16_t nSamplesForRendering ) { MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; - int16_t i, ch; - int32_t output_Fs; - int16_t output_frame; - float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ - float *p_tc[MC_PARAMUPMIX_MAX_INPUT_CHANS]; + float *pPcm_tmp[MC_PARAMUPMIX_COMBINATIONS]; + float *p_tc[MC_PARAMUPMIX_COMBINATIONS]; + int16_t nchan_internal, ch; + int16_t nSamplesLeftForTD, default_frame; hMCParamUpmix = st_ivas->hMCParamUpmix; assert( hMCParamUpmix ); push_wmops( "ivas_param_upmix_dec_decorr_subframes" ); - output_Fs = st_ivas->hDecoderConfig->output_Fs; - output_frame = (int16_t) ( output_Fs / FRAMES_PER_SEC ); + /* TD decorrelator */ + default_frame = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); + nSamplesLeftForTD = nSamplesForRendering; + nchan_internal = MC_PARAMUPMIX_COMBINATIONS; - for ( ch = 0; ch < MC_PARAMUPMIX_MAX_INPUT_CHANS; ch++ ) + for ( ch = 0; ch < nchan_internal; ch++ ) { - p_tc[ch] = st_ivas->hTcBuffer->tc[ch]; + pPcm_tmp[ch] = st_ivas->hTcBuffer->tc[ch + 8]; + p_tc[ch] = st_ivas->hTcBuffer->tc[ch + 4]; } - for ( i = 0; i < MC_PARAMUPMIX_COMBINATIONS; i++ ) + while ( nSamplesLeftForTD ) { - pPcm_temp[i] = p_tc[8 + i]; /* decorrelated */ - } + int16_t nSamplesToDecorr = min( nSamplesLeftForTD, default_frame ); + + paramupmix_td_decorr_process_jbm( hMCParamUpmix->hTdDecorr, p_tc, pPcm_tmp, nSamplesToDecorr ); + + for ( ch = 0; ch < nchan_internal; ch++ ) + { + p_tc[ch] += nSamplesToDecorr; + } - paramupmix_td_decorr_process_jbm( hMCParamUpmix->hTdDecorr, &( p_tc[4] ), pPcm_temp, output_frame ); + nSamplesLeftForTD -= nSamplesToDecorr; + } pop_wmops(); -- GitLab From 60a5fe0352d3a63c692c9508d08d4d2527830082 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Tue, 4 Jul 2023 13:12:07 +0200 Subject: [PATCH 10/19] Msan fix --- lib_dec/ivas_mc_paramupmix_dec.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index d63a67302c..fdedb13e77 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -474,6 +474,7 @@ ivas_error ivas_mc_paramupmix_dec_open( #ifdef JBM_PARAMUPMIX /* allocate transport channels*/ hMCParamUpmix->free_param_interpolator = 0; + hMCParamUpmix->param_interpolator = NULL; if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) { int16_t nchan_to_allocate; @@ -544,9 +545,12 @@ void ivas_mc_paramupmix_dec_close( } } #ifdef JBM_PARAMUPMIX - if ( ( ( *hMCParamUpmix )->param_interpolator != NULL ) && ( ( *hMCParamUpmix )->free_param_interpolator == 1 ) ) + if ( ( *hMCParamUpmix )->param_interpolator != NULL ) { - free( ( *hMCParamUpmix )->param_interpolator ); + if ( ( *hMCParamUpmix )->free_param_interpolator == 1 ) + { + free( ( *hMCParamUpmix )->param_interpolator ); + } } #endif free( *hMCParamUpmix ); -- GitLab From 31f625e1791124d8dcd636f5dadc7bf0b8586e3e Mon Sep 17 00:00:00 2001 From: Michael Schug Date: Tue, 4 Jul 2023 17:12:36 +0200 Subject: [PATCH 11/19] for paramupmix jbm -vs 100 compared to normal operation should give identical output --- lib_dec/ivas_mc_paramupmix_dec.c | 42 ++++++++++++++++++++++++++------ lib_dec/ivas_stat_dec.h | 2 ++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index fdedb13e77..25538a2ae4 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -56,7 +56,7 @@ static void ps_pred_process( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, float qmf_mod_re[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t ch ); #ifdef JBM_PARAMUPMIX -static void ps_pred_process_sf( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, DECODER_TC_BUFFER_HANDLE hTcBuffer, float qmf_mod_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float *param_interpol, const int16_t ch ); +static void ps_pred_process_sf( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, DECODER_TC_BUFFER_HANDLE hTcBuffer, float qmf_mod_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float *param_interpol, const int16_t ch, const int16_t slots_rendered ); static void ivas_mc_paramupmix_dec_sf( Decoder_Struct *st_ivas, float *output_f[MAX_OUTPUT_CHANNELS] ); @@ -376,6 +376,12 @@ void ivas_mc_paramupmix_dec_render( assert( slots_to_render == 0 ); #endif + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + mvr2r( hMCParamUpmix->alpha_prev[ch], hMCParamUpmix->alpha_sf[ch], IVAS_MAX_NUM_BANDS ); + mvr2r( hMCParamUpmix->beta_prev[ch], hMCParamUpmix->beta_sf[ch], IVAS_MAX_NUM_BANDS ); + } + for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) { int16_t n_samples_sf = slot_size * st_ivas->hTcBuffer->subframe_nbslots[subframe_idx]; @@ -733,7 +739,8 @@ static void ps_pred_process_sf( float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], /* in/out */ float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float *param_interpol, - const int16_t ch ) + const int16_t ch, + const int16_t slots_rendered ) { float vmre, vmim, vsre, vsim; int16_t iqmf, ipar, ismp, iismp; @@ -742,6 +749,10 @@ static void ps_pred_process_sf( float *beta1, *beta2; float *alpha_prev = hMCParamUpmix->alpha_prev[ch]; float *beta_prev = hMCParamUpmix->beta_prev[ch]; + float *alpha_sf = hMCParamUpmix->alpha_sf[ch]; + float *beta_sf = hMCParamUpmix->beta_sf[ch]; + float dalpha, dbeta, ifac; + float alpha_start[IVAS_MAX_NUM_BANDS], beta_start[IVAS_MAX_NUM_BANDS]; const int16_t qmf_to_par_band[] = { 0, 1, 2, 3, 4, 5, 5, 6, 6, 7, @@ -752,9 +763,10 @@ static void ps_pred_process_sf( 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }; + mvr2r(alpha_sf, alpha_start, IVAS_MAX_NUM_BANDS); + mvr2r(beta_sf, beta_start, IVAS_MAX_NUM_BANDS); for ( iqmf = 0; iqmf < CLDFB_NO_CHANNELS_MAX; iqmf++ ) { - /* For changing no of parameter bands (ipar1 != ipar2), TIGGER_FRAMING assumed */ ipar = qmf_to_par_band[iqmf]; alpha1 = alpha_prev; beta1 = beta_prev; @@ -762,13 +774,24 @@ static void ps_pred_process_sf( ismp = 0; alpha2 = hMCParamUpmix->alphas[ch]; beta2 = hMCParamUpmix->betas[ch]; - alpha_smp = alpha1[ipar]; - beta_smp = beta1[ipar]; + alpha_smp = alpha_start[ipar]; + beta_smp = beta_start[ipar]; for ( iismp = 0; iismp < hTcBuffer->subframe_nbslots[hTcBuffer->subframes_rendered]; iismp++ ) { - alpha_smp = alpha1[ipar] + ( alpha2[ipar] - alpha1[ipar] ) * param_interpol[iismp]; - beta_smp = beta1[ipar] + ( beta2[ipar] - beta1[ipar] ) * param_interpol[iismp]; + if ((slots_rendered == 0) && (iismp == 0)) + { + ifac = param_interpol[iismp]; + } + else + { + ifac = param_interpol[iismp] - param_interpol[iismp-1]; + } + dalpha = (alpha2[ipar] - alpha1[ipar]) * ifac; + dbeta = (beta2[ipar] - beta1[ipar]) * ifac; + + alpha_smp += dalpha; + beta_smp += dbeta; vmre = qmf_mod_re[ismp][iqmf]; vmim = qmf_mod_im[ismp][iqmf]; @@ -780,6 +803,8 @@ static void ps_pred_process_sf( ismp++; } + alpha_sf[ipar] = alpha_smp; + beta_sf[ipar] = beta_smp; } return; @@ -827,7 +852,8 @@ static void ivas_mc_paramupmix_dec_sf( Cldfb_RealBuffer[2 * ch + 1], /* in/out decorr */ Cldfb_ImagBuffer[2 * ch + 1], &hMCParamUpmix->param_interpolator[st_ivas->hTcBuffer->slots_rendered], - ch ); + ch, + st_ivas->hTcBuffer->slots_rendered ); /*-- m, s -> l, r ----------------------------*/ for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ ) diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 152cf5091e..e13be18793 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -804,6 +804,8 @@ typedef struct ivas_mc_paramupmix_dec_data_structure float *pcm_delay[MC_PARAMUPMIX_MAX_TRANSPORT_CHANS]; #ifdef JBM_PARAMUPMIX float *param_interpolator; + float alpha_sf[MC_PARAMUPMIX_COMBINATIONS][IVAS_MAX_NUM_BANDS]; + float beta_sf[MC_PARAMUPMIX_COMBINATIONS][IVAS_MAX_NUM_BANDS]; int16_t free_param_interpolator; #endif -- GitLab From 67d82c2aa3298894c094c5cc3d645bf22e1230d9 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Wed, 5 Jul 2023 11:23:42 +0200 Subject: [PATCH 12/19] Clang and msan fix --- lib_dec/ivas_init_dec.c | 17 ++++++++++++++++- lib_dec/ivas_jbm_dec.c | 2 +- lib_dec/ivas_mc_paramupmix_dec.c | 18 +++++++++--------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 0e63166be3..3154f19461 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1336,20 +1336,35 @@ ivas_error ivas_init_decoder( if ( ( st_ivas->ivas_format == MC_FORMAT ) && ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) ) { granularity = NS2SA( output_Fs, CLDFB_SLOT_NS ); + + n_channels_transport_jbm = ivas_jbm_dec_get_num_tc_channels( st_ivas ); + + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, n_channels_transport_jbm, MC_PARAMUPMIX_MAX_INPUT_CHANS, MC_PARAMUPMIX_MAX_INPUT_CHANS, granularity ) ) != IVAS_ERR_OK ) + { + return error; + } } else { granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); + + n_channels_transport_jbm = ivas_jbm_dec_get_num_tc_channels( st_ivas ); + + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, n_channels_transport_jbm, n_channels_transport_jbm, n_channels_transport_jbm, granularity ) ) != IVAS_ERR_OK ) + { + return error; + } } #else granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); -#endif + n_channels_transport_jbm = ivas_jbm_dec_get_num_tc_channels( st_ivas ); if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, n_channels_transport_jbm, n_channels_transport_jbm, n_channels_transport_jbm, granularity ) ) != IVAS_ERR_OK ) { return error; } +#endif } } diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 3b26ef44e1..43ad5d59fe 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -1486,7 +1486,7 @@ int16_t ivas_jbm_dec_get_num_tc_channels( #ifdef JBM_PARAMUPMIX else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) { - num_tc = MC_PARAMUPMIX_MAX_INPUT_CHANS; + num_tc = MC_PARAMUPMIX_MAX_TRANSPORT_CHANS; } #endif else if ( st_ivas->mc_mode == MC_MODE_MCMASA ) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 25538a2ae4..c790d5b59d 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -489,7 +489,7 @@ ivas_error ivas_mc_paramupmix_dec_open( buffer_mode = TC_BUFFER_MODE_RENDERER; nchan_tc = ivas_jbm_dec_get_num_tc_channels( st_ivas ); - nchan_to_allocate = nchan_tc; + nchan_to_allocate = MC_PARAMUPMIX_MAX_INPUT_CHANS; if ( ( hMCParamUpmix->param_interpolator = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * sizeof( float ) ) ) == NULL ) { return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for interpolator\n" ) ); @@ -503,7 +503,7 @@ ivas_error ivas_mc_paramupmix_dec_open( } else if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { - nchan_to_allocate = 2 * BINAURAL_CHANNELS; + nchan_to_allocate = MC_PARAMUPMIX_MAX_INPUT_CHANS; } if ( st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV && st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV_ROOM ) { @@ -763,8 +763,8 @@ static void ps_pred_process_sf( 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }; - mvr2r(alpha_sf, alpha_start, IVAS_MAX_NUM_BANDS); - mvr2r(beta_sf, beta_start, IVAS_MAX_NUM_BANDS); + mvr2r( alpha_sf, alpha_start, IVAS_MAX_NUM_BANDS ); + mvr2r( beta_sf, beta_start, IVAS_MAX_NUM_BANDS ); for ( iqmf = 0; iqmf < CLDFB_NO_CHANNELS_MAX; iqmf++ ) { ipar = qmf_to_par_band[iqmf]; @@ -779,16 +779,16 @@ static void ps_pred_process_sf( for ( iismp = 0; iismp < hTcBuffer->subframe_nbslots[hTcBuffer->subframes_rendered]; iismp++ ) { - if ((slots_rendered == 0) && (iismp == 0)) + if ( ( slots_rendered == 0 ) && ( iismp == 0 ) ) { ifac = param_interpol[iismp]; } else { - ifac = param_interpol[iismp] - param_interpol[iismp-1]; + ifac = param_interpol[iismp] - param_interpol[iismp - 1]; } - dalpha = (alpha2[ipar] - alpha1[ipar]) * ifac; - dbeta = (beta2[ipar] - beta1[ipar]) * ifac; + dalpha = ( alpha2[ipar] - alpha1[ipar] ) * ifac; + dbeta = ( beta2[ipar] - beta1[ipar] ) * ifac; alpha_smp += dalpha; beta_smp += dbeta; @@ -852,7 +852,7 @@ static void ivas_mc_paramupmix_dec_sf( Cldfb_RealBuffer[2 * ch + 1], /* in/out decorr */ Cldfb_ImagBuffer[2 * ch + 1], &hMCParamUpmix->param_interpolator[st_ivas->hTcBuffer->slots_rendered], - ch, + ch, st_ivas->hTcBuffer->slots_rendered ); /*-- m, s -> l, r ----------------------------*/ -- GitLab From 6d7d0eff4a345e689c494bbb8a37aec3de9e4d82 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Wed, 5 Jul 2023 17:59:31 +0200 Subject: [PATCH 13/19] Fix binaural crash for vs!=100 --- lib_dec/ivas_mc_paramupmix_dec.c | 17 +++++++---------- lib_rend/ivas_crend.c | 6 +++++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index c790d5b59d..461072d79d 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -318,14 +318,7 @@ void ivas_mc_paramupmix_dec_digest_tc( /* adapt subframes */ ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); - if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) - { - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); - } - else - { - ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); - } + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hMCParamUpmix->param_interpolator ); pop_wmops(); } @@ -344,6 +337,8 @@ void ivas_mc_paramupmix_dec_render( uint16_t slot_size, ch; float *output_f_local[MAX_OUTPUT_CHANNELS]; MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; + int16_t saved_subframes_rendered; + int16_t saved_slots_rendered; hMCParamUpmix = st_ivas->hMCParamUpmix; assert( hMCParamUpmix ); @@ -359,6 +354,8 @@ void ivas_mc_paramupmix_dec_render( /* loop for synthesis, assume we always have to render in multiples of 5ms subframes with spills */ slots_to_render = min( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered, nSamplesAsked / slot_size ); *nSamplesRendered = slots_to_render * slot_size; + saved_subframes_rendered = st_ivas->hTcBuffer->subframes_rendered; + saved_slots_rendered = st_ivas->hTcBuffer->slots_rendered; first_sf = st_ivas->hTcBuffer->subframes_rendered; last_sf = first_sf; @@ -404,8 +401,8 @@ void ivas_mc_paramupmix_dec_render( /* Binaural renderer will do the slot rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) { - st_ivas->hTcBuffer->slots_rendered = 0; - st_ivas->hTcBuffer->subframes_rendered = 0; + st_ivas->hTcBuffer->slots_rendered = saved_slots_rendered; + st_ivas->hTcBuffer->subframes_rendered = saved_subframes_rendered; } pop_wmops(); diff --git a/lib_rend/ivas_crend.c b/lib_rend/ivas_crend.c index b2b4887499..6d536bd625 100644 --- a/lib_rend/ivas_crend.c +++ b/lib_rend/ivas_crend.c @@ -1514,8 +1514,9 @@ ivas_error ivas_rend_crendProcessSubframe( slots_to_render = min( hTcBuffer->num_slots - hTcBuffer->slots_rendered, n_samples_to_render / slot_size ); first_sf = hTcBuffer->subframes_rendered; last_sf = first_sf; +#ifndef JBM_PARAMUPMIX hTcBuffer->slots_rendered += slots_to_render; - +#endif while ( slots_to_render > 0 ) { slots_to_render -= hTcBuffer->subframe_nbslots[last_sf]; @@ -1569,6 +1570,9 @@ ivas_error ivas_rend_crendProcessSubframe( { p_pcm_tmp[ch] += subframe_len; } +#ifdef JBM_PARAMUPMIX + hTcBuffer->slots_rendered += hTcBuffer->subframe_nbslots[subframe_idx]; +#endif } else { -- GitLab From a5119a46c1feab2cb3292266f9930f7e7c89ea75 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 10 Jul 2023 13:53:29 +0200 Subject: [PATCH 14/19] Switch to Fastconv --- lib_dec/ivas_dec.c | 6 ++- lib_dec/ivas_jbm_dec.c | 9 +--- lib_dec/ivas_mc_paramupmix_dec.c | 79 ++++++++++++++++++-------------- lib_dec/ivas_output_config.c | 20 ++++++++ 4 files changed, 71 insertions(+), 43 deletions(-) diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index 439a90fb2c..2b88822d06 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -532,6 +532,10 @@ ivas_error ivas_dec( } /* Rendering */ +#ifdef JBM_PARAMUPMIX + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) + { +#else if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) { if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig, @@ -539,7 +543,7 @@ ivas_error ivas_dec( { return error; } - +#endif ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output ); } else if ( st_ivas->renderer_type == RENDERER_MC ) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 43ad5d59fe..ed672cac5c 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -896,15 +896,8 @@ ivas_error ivas_jbm_dec_render( } /* Rendering */ - if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { - if ( ( error = ivas_rend_crendProcessSubframe( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig, - st_ivas->hCombinedOrientationData, - &st_ivas->hIntSetup, st_ivas->hEFAPdata, st_ivas->hTcBuffer, p_output, p_output, *nSamplesRendered, output_Fs ) ) != IVAS_ERR_OK ) - { - return error; - } - ivas_binaural_add_LFE( st_ivas, *nSamplesRendered, p_output, p_output ); } else if ( st_ivas->renderer_type == RENDERER_MC ) diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 461072d79d..6ae1097e7e 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -181,7 +181,9 @@ void ivas_mc_paramupmix_dec( float *pPcm_temp[MC_PARAMUPMIX_COMBINATIONS * 2]; /* decorrelated and undecorrelated*/ int16_t noparamupmix_delay; AUDIO_CONFIG output_config; - +#ifdef JBM_PARAMUPMIX + float *p_output[MAX_OUTPUT_CHANNELS]; +#endif hMCParamUpmix = st_ivas->hMCParamUpmix; assert( hMCParamUpmix ); @@ -198,6 +200,19 @@ void ivas_mc_paramupmix_dec( { first_empty_channel = 8; /* Don't upmix */ } +#ifdef JBM_PARAMUPMIX + else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) + { + /* Implement binaural rendering */ + first_empty_channel = 4; /* don't erase LFE */ + + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS * 2; ch++ ) + { + p_output[ch] = output_f[ch]; + } + ivas_binaural_cldfb( st_ivas, p_output ); + } +#endif else { first_empty_channel = 12; @@ -337,8 +352,7 @@ void ivas_mc_paramupmix_dec_render( uint16_t slot_size, ch; float *output_f_local[MAX_OUTPUT_CHANNELS]; MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix; - int16_t saved_subframes_rendered; - int16_t saved_slots_rendered; + hMCParamUpmix = st_ivas->hMCParamUpmix; assert( hMCParamUpmix ); @@ -354,8 +368,6 @@ void ivas_mc_paramupmix_dec_render( /* loop for synthesis, assume we always have to render in multiples of 5ms subframes with spills */ slots_to_render = min( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered, nSamplesAsked / slot_size ); *nSamplesRendered = slots_to_render * slot_size; - saved_subframes_rendered = st_ivas->hTcBuffer->subframes_rendered; - saved_slots_rendered = st_ivas->hTcBuffer->slots_rendered; first_sf = st_ivas->hTcBuffer->subframes_rendered; last_sf = first_sf; @@ -372,39 +384,40 @@ void ivas_mc_paramupmix_dec_render( #ifdef DEBUGGING assert( slots_to_render == 0 ); #endif - - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + /* Implement binaural rendering */ + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { - mvr2r( hMCParamUpmix->alpha_prev[ch], hMCParamUpmix->alpha_sf[ch], IVAS_MAX_NUM_BANDS ); - mvr2r( hMCParamUpmix->beta_prev[ch], hMCParamUpmix->beta_sf[ch], IVAS_MAX_NUM_BANDS ); + ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, output_f_local ); } - - for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) + else { - int16_t n_samples_sf = slot_size * st_ivas->hTcBuffer->subframe_nbslots[subframe_idx]; - ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local ); - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) { - output_f_local[ch] += n_samples_sf; + mvr2r( hMCParamUpmix->alpha_prev[ch], hMCParamUpmix->alpha_sf[ch], IVAS_MAX_NUM_BANDS ); + mvr2r( hMCParamUpmix->beta_prev[ch], hMCParamUpmix->beta_sf[ch], IVAS_MAX_NUM_BANDS ); } - } - for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) - { - mvr2r( hMCParamUpmix->alphas[ch], hMCParamUpmix->alpha_prev[ch], IVAS_MAX_NUM_BANDS ); - mvr2r( hMCParamUpmix->betas[ch], hMCParamUpmix->beta_prev[ch], IVAS_MAX_NUM_BANDS ); - } + for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) + { + int16_t n_samples_sf = slot_size * st_ivas->hTcBuffer->subframe_nbslots[subframe_idx]; - *nSamplesAvailable = ( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered ) * slot_size; + ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local ); + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + output_f_local[ch] += n_samples_sf; + } + } - /* Binaural renderer will do the slot rendering */ - if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM ) - { - st_ivas->hTcBuffer->slots_rendered = saved_slots_rendered; - st_ivas->hTcBuffer->subframes_rendered = saved_subframes_rendered; + for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ ) + { + mvr2r( hMCParamUpmix->alphas[ch], hMCParamUpmix->alpha_prev[ch], IVAS_MAX_NUM_BANDS ); + mvr2r( hMCParamUpmix->betas[ch], hMCParamUpmix->beta_prev[ch], IVAS_MAX_NUM_BANDS ); + } } + *nSamplesAvailable = ( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered ) * slot_size; + pop_wmops(); } #endif @@ -498,18 +511,16 @@ ivas_error ivas_mc_paramupmix_dec_open( nchan_tc = st_ivas->hDecoderConfig->nchan_out; nchan_to_allocate = nchan_tc; } - else if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) + else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { nchan_to_allocate = MC_PARAMUPMIX_MAX_INPUT_CHANS; } - if ( st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV && st_ivas->renderer_type != RENDERER_BINAURAL_MIXER_CONV_ROOM ) + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) { - if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_tc, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) - { - return error; - } - hMCParamUpmix->free_param_interpolator = 1; + return error; } + hMCParamUpmix->free_param_interpolator = 1; + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, DEFAULT_JBM_CLDFB_TIMESLOTS, hMCParamUpmix->param_interpolator ); } #endif diff --git a/lib_dec/ivas_output_config.c b/lib_dec/ivas_output_config.c index 16787b63cd..a068e97b79 100644 --- a/lib_dec/ivas_output_config.c +++ b/lib_dec/ivas_output_config.c @@ -199,7 +199,11 @@ void ivas_renderer_select( } else { +#ifdef JBM_PARAMUPMIX + if ( st_ivas->mc_mode == MC_MODE_MCT ) +#else if ( ( st_ivas->mc_mode == MC_MODE_MCT ) || ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) ) +#endif { *renderer_type = RENDERER_BINAURAL_MIXER_CONV; } @@ -223,11 +227,21 @@ void ivas_renderer_select( /* force HOA3 domain for rotation*/ *internal_config = AUDIO_CONFIG_HOA3; } +#ifdef JBM_PARAMUPMIX + if ( ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) && ( *renderer_type == RENDERER_BINAURAL_FASTCONV ) ) + { + *internal_config = AUDIO_CONFIG_5_1_2; + } +#endif } } else { +#ifdef JBM_PARAMUPMIX + if ( st_ivas->mc_mode == MC_MODE_MCT ) +#else if ( st_ivas->mc_mode == MC_MODE_MCT || st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) +#endif { *renderer_type = RENDERER_BINAURAL_MIXER_CONV_ROOM; } @@ -244,6 +258,12 @@ void ivas_renderer_select( { *renderer_type = RENDERER_BINAURAL_FASTCONV_ROOM; } +#endif +#ifdef JBM_PARAMUPMIX + if ( ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) && ( *renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) ) + { + *internal_config = AUDIO_CONFIG_5_1_2; + } #endif } } -- GitLab From f23ebeb1908baf8cc07e7bc3c38b46ab53aae486 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 10 Jul 2023 14:55:04 +0200 Subject: [PATCH 15/19] Fix issue with non-100 speeds --- lib_com/ivas_prot.h | 3 +++ lib_dec/ivas_binRenderer_internal.c | 9 +++++++++ lib_dec/ivas_jbm_dec.c | 4 ++++ lib_dec/ivas_mc_paramupmix_dec.c | 2 +- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 5f18f27668..094ebb7194 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -5174,6 +5174,9 @@ void ivas_binaural_cldfb( void ivas_binaural_cldfb_sf( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const int16_t n_samples_to_render, /* i : output frame length per channel */ +#ifdef JBM_PARAMUPMIX + const int16_t slot_size, /* i : JBM slot size */ +#endif float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ ); diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index 5bd09236e0..4cf4c52b22 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -1027,6 +1027,9 @@ void ivas_binaural_cldfb( void ivas_binaural_cldfb_sf( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const int16_t n_samples_to_render, /* i : output frame length per channel */ +#ifdef JBM_PARAMUPMIX + const int16_t slot_size, /* i : JBM slot size */ +#endif float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ ) { @@ -1035,12 +1038,18 @@ void ivas_binaural_cldfb_sf( float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; int16_t slot_idx, subframeIdx, index_slot, idx_in, idx_lfe, maxBand, ch; +#ifdef JBM_PARAMUPMIX + int16_t slots_to_render, first_sf, last_sf; +#else int16_t slot_size, slots_to_render, first_sf, last_sf; +#endif int16_t slot_index_start, slot_index_start_cldfb; /* Implement a 5 msec loops */ maxBand = (int16_t) ( ( CLDFB_NO_CHANNELS_MAX * st_ivas->hDecoderConfig->output_Fs ) / 48000 ); +#ifndef JBM_PARAMUPMIX slot_size = st_ivas->hTcBuffer->nb_subframes; +#endif /* loop for synthesis, assume we always have to render in multiples of 5ms subframes with spills */ slots_to_render = min( st_ivas->hTcBuffer->num_slots - st_ivas->hTcBuffer->slots_rendered, n_samples_to_render / slot_size ); diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index ed672cac5c..29fd38e55a 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -797,7 +797,11 @@ ivas_error ivas_jbm_dec_render( #ifdef DEBUGGING else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { +#ifdef JBM_PARAMUPMIX + ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, st_ivas->hTcBuffer->nb_subframes, p_output ); +#else ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, p_output ); +#endif } #endif } diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 6ae1097e7e..3307356964 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -387,7 +387,7 @@ void ivas_mc_paramupmix_dec_render( /* Implement binaural rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) { - ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, output_f_local ); + ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, slot_size, output_f_local ); } else { -- GitLab From edbaf7a8a72cebff090d843a0773c494c8d2a317 Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 10 Jul 2023 15:33:57 +0200 Subject: [PATCH 16/19] Fix pipelines --- lib_com/ivas_prot.h | 19 +++++++++++++++---- lib_dec/ivas_binRenderer_internal.c | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 094ebb7194..5ef73792aa 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -5165,7 +5165,7 @@ void ivas_binRenderer_close( BINAURAL_RENDERER_HANDLE *hBinRenderer /* i/o: decoder binaural renderer handle */ ); -#ifdef DEBUGGING +#ifdef JBM_PARAMUPMIX void ivas_binaural_cldfb( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ @@ -5174,14 +5174,25 @@ void ivas_binaural_cldfb( void ivas_binaural_cldfb_sf( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const int16_t n_samples_to_render, /* i : output frame length per channel */ -#ifdef JBM_PARAMUPMIX const int16_t slot_size, /* i : JBM slot size */ -#endif float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ - +); +#else +#ifdef DEBUGGING +void ivas_binaural_cldfb( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ ); +void ivas_binaural_cldfb_sf( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + const int16_t n_samples_to_render, /* i : output frame length per channel */ + float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ + +); #endif +#endif + void ivas_binRenderer( BINAURAL_RENDERER_HANDLE hBinRenderer, /* i/o: fastconv binaural renderer handle */ COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined head and external orientation handle */ diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index 4cf4c52b22..dae1f9d037 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -1028,9 +1028,9 @@ void ivas_binaural_cldfb_sf( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const int16_t n_samples_to_render, /* i : output frame length per channel */ #ifdef JBM_PARAMUPMIX - const int16_t slot_size, /* i : JBM slot size */ + const int16_t slot_size, /* i : JBM slot size */ #endif - float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ + float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ ) { float Cldfb_RealBuffer[MAX_INTERN_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; -- GitLab From 86c53e8abea406b124e56210c5341bc85cfb6eaf Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 10 Jul 2023 15:50:07 +0200 Subject: [PATCH 17/19] More pipeline work --- lib_dec/ivas_binRenderer_internal.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index dae1f9d037..dd005934e0 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -942,7 +942,6 @@ void ivas_binaural_add_LFE( } -#ifdef DEBUGGING /*-------------------------------------------------------------------------* * ivas_binaural_cldfb() * @@ -1119,7 +1118,6 @@ void ivas_binaural_cldfb_sf( return; } -#endif /*------------------------------------------------------------------------- -- GitLab From cb9b71a0aacfb1ef838358277d636a043f6b640a Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Thu, 13 Jul 2023 08:29:23 +0200 Subject: [PATCH 18/19] Update switch --- lib_com/options.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib_com/options.h b/lib_com/options.h index fb40c0e4e9..fd193beb7c 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -166,6 +166,7 @@ #define FIX_580_PARAMMC_ENER_BURSTS /* FhG: issue 580: energy bursts due to ILD holding when energy relations change too much */ #define FIX_593_STL_INCLUDE /* FhG: Issue 593: correct include of stl.h in lib_enc/ivas_stereo_eclvq_enc.c */ #define FIX_583_CLANG_TRANS_DET /* FhG: Issue 583: clang left shift on ramp_up_flag in transient detector */ +#define JBM_PARAMUPMIX /* Dlb: Issue 471: Integrate the Multichannel Parametric Upmix into the JBM path */ /* ################## End BE DEVELOPMENT switches ######################### */ @@ -173,7 +174,6 @@ /* #################### Start NON-BE CR switches ########################## */ /* any switch which is non-be wrt operation points tested in selection */ /* all switches in this category should start with "CR_" */ -#define JBM_PARAMUPMIX /* ##################### End NON-BE CR switches ########################### */ -- GitLab From 11b4e9e69173a28230d1b4fe63dfd96061a30a3d Mon Sep 17 00:00:00 2001 From: Simon Plain Date: Mon, 17 Jul 2023 11:19:50 +0200 Subject: [PATCH 19/19] Cosmetic changes --- lib_com/cnst.h | 2 ++ lib_com/ivas_cnst.h | 8 ++++++++ lib_dec/ivas_mc_paramupmix_dec.c | 6 +++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib_com/cnst.h b/lib_com/cnst.h index e1362b0d75..a93a2a890e 100644 --- a/lib_com/cnst.h +++ b/lib_com/cnst.h @@ -2237,11 +2237,13 @@ enum VOIP_RTPDUMP }; +#ifndef JBM_PARAMUPMIX typedef enum _COV_SMOOTHING_TYPE { COV_SMOOTH_SPAR, COV_SMOOTH_MC } COV_SMOOTHING_TYPE; +#endif /* clang-format on */ #endif /* CNST_H */ diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 60807f62cb..aa75d3261e 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -1332,6 +1332,14 @@ typedef enum #define MC_PARAMUPMIX_NCH 2 /* number of channels to combine into 1 */ #define MC_PARAMUPMIX_MIN_CLDFB 8 +#ifdef JBM_PARAMUPMIX +typedef enum _COV_SMOOTHING_TYPE +{ + COV_SMOOTH_SPAR, + COV_SMOOTH_MC +} COV_SMOOTHING_TYPE; +#endif + typedef struct { const int32_t *value; const uint16_t *length; diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index 3307356964..9ae580ae3a 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -478,16 +478,16 @@ ivas_error ivas_mc_paramupmix_dec_open( ivas_td_decorr_dec_open( &( hMCParamUpmix->hTdDecorr[i] ), output_Fs, 2, 1 ); } +#ifdef JBM_PARAMUPMIX for ( i = 0; i < MC_PARAMUPMIX_MAX_TRANSPORT_CHANS; i++ ) { - if ( ( hMCParamUpmix->pcm_delay[i] = (float *) malloc( 240 * sizeof( float ) ) ) == NULL ) + if ( ( hMCParamUpmix->pcm_delay[i] = (float *) malloc( NS2SA( output_Fs, IVAS_FB_DEC_DELAY_NS ) * sizeof( float ) ) ) == NULL ) { return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for delay buffer\n" ) ); } - set_zero( hMCParamUpmix->pcm_delay[i], 240 ); + set_zero( hMCParamUpmix->pcm_delay[i], NS2SA( output_Fs, IVAS_FB_DEC_DELAY_NS ) ); } -#ifdef JBM_PARAMUPMIX /* allocate transport channels*/ hMCParamUpmix->free_param_interpolator = 0; hMCParamUpmix->param_interpolator = NULL; -- GitLab