From beac45cea6abdbea12b2fc9d9f36608767097796 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Mon, 24 Jul 2023 09:35:35 +0200 Subject: [PATCH 1/5] initial support for parambin renderer in split rendering in decoder. switch: SPLIT_REND_WITH_HEAD_ROT_PARAMBIN --- lib_com/options.h | 3 + lib_dec/ivas_dec.c | 19 + lib_dec/ivas_dirac_dec.c | 79 +- lib_dec/ivas_init_dec.c | 46 +- lib_dec/ivas_ism_dec.c | 9 +- lib_dec/ivas_ism_param_dec.c | 13 + lib_dec/ivas_masa_dec.c | 39 + lib_dec/ivas_mct_dec.c | 8 + lib_dec/ivas_sba_dec.c | 21 + lib_dec/ivas_stat_dec.h | 4 + lib_rend/ivas_dirac_dec_binaural_functions.c | 1003 ++++++++++++++--- lib_rend/lib_rend.c | 22 +- tests/split_rendering/test_split_rendering.py | 8 +- 13 files changed, 1120 insertions(+), 154 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index b35772ecc1..e8a5a8b4d1 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -211,6 +211,9 @@ #define FIX_622_SILENCE_USAN_WARNING /* FhG: silenceusan warning in ifft code */ #define FIX_615_UBSAN_SPAR_TO_DIRAC /*Dlb : Fix for UBSAN issue 615*/ +#define SPLIT_REND_WITH_HEAD_ROT_PARAMBIN /* Nokia: Issue 623: Split rendering support for parambin renderer */ +#define SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT /* Nokia: Issue 623: Split rendering support for parambin renderer, computational optimization */ + /* ################## End BE DEVELOPMENT switches ######################### */ diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index 5ca878a00e..e945f04525 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -124,10 +124,20 @@ ivas_error ivas_dec( if ( ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN assert( ( st_ivas->ivas_format == SBA_FORMAT || + st_ivas->ivas_format == MASA_FORMAT || + st_ivas->ivas_format == ISM_FORMAT || + st_ivas->ivas_format == MC_FORMAT ) && + ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA, MASA, discrete and parametric ISM, or MC formats and 48 kHz sampling rate only" ); +#else + assert( ( st_ivas->ivas_format == SBA_FORMAT || + st_ivas->ivas_format == MASA_FORMAT || + st_ivas->ivas_format == ISM_FORMAT || ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode != ISM_MODE_PARAM ) || ( st_ivas->ivas_format == MC_FORMAT && st_ivas->mc_mode != MC_MODE_MCMASA ) ) && ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA, discrete ISM, or MCT-MC formats and 48 kHz sampling rate only" ); +#endif ivas_set_split_rend_setup( &st_ivas->splitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, hSplitRendBits ); } #endif @@ -845,6 +855,11 @@ ivas_error ivas_dec( IVAS_DEC_SPLIT_REND_WRAPPER *hSplitBinRend; int16_t max_band; int16_t pcm_out; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t td_input; + + td_input = st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC; +#endif hSplitBinRend = &st_ivas->splitBinRend; max_band = (int16_t) ( ( BINAURAL_MAXBANDS * output_Fs ) / 48000 ); pcm_out = ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0; @@ -857,7 +872,11 @@ ivas_error ivas_dec( hSplitBinRend->hMultiBinCldfbData->Cldfb_RealBuffer_Binaural, hSplitBinRend->hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural, max_band, output, 1, +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + td_input, +#else st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV, +#endif pcm_out ); free( st_ivas->splitBinRend.hMultiBinCldfbData ); diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index 766f4ff08f..3e9987e5da 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -826,7 +826,11 @@ ivas_error ivas_dirac_dec_config( if ( !need_parambin ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif } need_dirac_rend = 0; @@ -870,13 +874,48 @@ ivas_error ivas_dirac_dec_config( { if ( st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + /* populate pMultiBinPoseData with the additional poses determined from the DoF */ + ivas_renderSplitGetMultiBinPoseData( + &st_ivas->hRenderConfig->split_rend_config, + &st_ivas->splitBinRend.splitrend.multiBinPoseData, + st_ivas->hHeadTrackData->sr_pose_pred_axis ); + } +#endif if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) { return error; } } - +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + /* TODO: can we keep isRenderingInTd fixed? */ + ivas_split_rend_choose_default_codec( &st_ivas->hRenderConfig->split_rend_config.codec, + 0, /* isRenderingInTd */ + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); /* pcm_out */ + + error = ivas_split_renderer_open( &st_ivas->splitBinRend.splitrend, + &st_ivas->hRenderConfig->split_rend_config, + st_ivas->hDecoderConfig->output_Fs, + 1, /* is_cldfb_in */ + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); /* is_pcm_out */ + + if ( error != IVAS_ERR_OK ) + { + return error; + } + } +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] == NULL ) +#else if ( st_ivas->hDiracDecBin == NULL ) +#endif { if ( ( error = ivas_dirac_dec_init_binaural_data( st_ivas, st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) { @@ -886,24 +925,55 @@ ivas_error ivas_dirac_dec_config( else { /* This is required to keep BE in rate switching. This probably means that 1TC and 2TC MASA perform differently. */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params != NULL && !( st_ivas->ivas_format == MASA_FORMAT && st_ivas->nSCE > 0 ) ) +#else if ( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params != NULL && !( st_ivas->ivas_format == MASA_FORMAT && st_ivas->nSCE > 0 ) ) +#endif { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_decorr_close( &st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params, &st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_state ); +#else ivas_dirac_dec_decorr_close( &st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params, &st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_state ); +#endif } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin[0]->hTdDecorr ), &( st_ivas->hDiracDecBin[0]->useTdDecorr ) ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin->hTdDecorr ), &( st_ivas->hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) +#endif { return error; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + /* copy td-decorr flag to split renderer side rendereres */ + for ( int16_t pos_idx = 1; pos_idx < st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses; pos_idx++ ) + { + st_ivas->hDiracDecBin[pos_idx]->useTdDecorr = st_ivas->hDiracDecBin[0]->useTdDecorr; + } + + if ( !st_ivas->hDiracDecBin[0]->useTdDecorr ) +#else if ( !st_ivas->hDiracDecBin->useTdDecorr ) +#endif { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params == NULL ) +#else if ( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params == NULL ) +#endif { float frequency_axis[CLDFB_NO_CHANNELS_MAX]; ivas_dirac_dec_get_frequency_axis( frequency_axis, st_ivas->hDecoderConfig->output_Fs, st_ivas->hSpatParamRendCom->num_freq_bands ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( error = ivas_dirac_dec_decorr_open( &( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params ), + &( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_state ), +#else if ( ( error = ivas_dirac_dec_decorr_open( &( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params ), &( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_state ), +#endif st_ivas->hSpatParamRendCom->num_freq_bands, BINAURAL_CHANNELS, BINAURAL_CHANNELS, @@ -917,7 +987,14 @@ ivas_error ivas_dirac_dec_config( } } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( int16_t pos_idx = 0; pos_idx < st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses; pos_idx++ ) + { + st_ivas->hDiracDecBin[pos_idx]->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); + } +#else st_ivas->hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); +#endif } } } diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 616069f25f..2087e7c9e4 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1319,6 +1319,16 @@ ivas_error ivas_init_decoder( /* ParamISM is handled separately from other common config */ else if ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM && ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + ivas_renderSplitGetMultiBinPoseData( + &st_ivas->hRenderConfig->split_rend_config, + &st_ivas->splitBinRend.splitrend.multiBinPoseData, + st_ivas->hHeadTrackData->sr_pose_pred_axis ); + } +#endif if ( st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) @@ -1331,6 +1341,26 @@ ivas_error ivas_init_decoder( { return error; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { +#ifdef FIX_SPLIT_REND_OPEN_ERROR_HANDLING + error = +#endif + ivas_split_renderer_open( &st_ivas->splitBinRend.splitrend, + &st_ivas->hRenderConfig->split_rend_config, + hDecoderConfig->output_Fs, + 1, ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); + +#ifdef FIX_SPLIT_REND_OPEN_ERROR_HANDLING + if ( error != IVAS_ERR_OK ) + { + return error; + } +#endif + } +#endif } else if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) { @@ -1820,7 +1850,14 @@ void ivas_initialize_handles_dec( st_ivas->splitBinRend.hSplitRendBits = NULL; ivas_init_split_rend_handles( &st_ivas->splitBinRend.splitrend ); #endif +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) + { + st_ivas->hDiracDecBin[i] = NULL; + } +#else st_ivas->hDiracDecBin = NULL; +#endif st_ivas->hDirACRend = NULL; st_ivas->hSpatParamRendCom = NULL; st_ivas->hLsSetUpConversion = NULL; @@ -1984,7 +2021,11 @@ void ivas_destroy_dec( #endif /* Parametric binaural renderer handle */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif /* Crend handle */ ivas_rend_closeCrend( &( st_ivas->hCrendWrapper ) @@ -2103,8 +2144,11 @@ void ivas_init_dec_get_num_cldfb_instances( { *numCldfbAnalyses = st_ivas->nchan_transport + 1; } - +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->useTdDecorr ) +#else if ( st_ivas->hDiracDecBin->useTdDecorr ) +#endif { *numCldfbAnalyses += 2; } diff --git a/lib_dec/ivas_ism_dec.c b/lib_dec/ivas_ism_dec.c index fa5783d90a..95cf886cb3 100644 --- a/lib_dec/ivas_ism_dec.c +++ b/lib_dec/ivas_ism_dec.c @@ -178,8 +178,11 @@ static ivas_error ivas_ism_bitrate_switching( #endif { /* close the parametric binaural renderer */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); - +#endif /* Open the TD Binaural renderer */ if ( st_ivas->hHrtfTD == NULL || st_ivas->hBinRendererTd == NULL ) { @@ -221,7 +224,11 @@ static ivas_error ivas_ism_bitrate_switching( #endif { /* close the parametric binaural renderer */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif /* Open Crend Binaural renderer */ if ( ( error = ivas_rend_openCrend( &( st_ivas->hCrendWrapper ), st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hRenderConfig, st_ivas->hSetOfHRTF, st_ivas->hDecoderConfig->output_Fs diff --git a/lib_dec/ivas_ism_param_dec.c b/lib_dec/ivas_ism_param_dec.c index 4662304a4a..1e8b2f3083 100644 --- a/lib_dec/ivas_ism_param_dec.c +++ b/lib_dec/ivas_ism_param_dec.c @@ -386,7 +386,11 @@ static ivas_error ivas_param_ism_rendering_init( return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for interpolator\n" ) ); } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM) ) +#else if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) ) +#endif { /* computation of proto matrix */ ivas_ism_get_proto_matrix( hOutSetup, nchan_transport, hParamIsmRendering->proto_matrix ); @@ -540,8 +544,13 @@ ivas_error ivas_param_ism_dec_open( } } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || + output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) +#else if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO ) ) +#endif { /* Initialize efap handle */ if ( ( error = efap_init_data( &( st_ivas->hEFAPdata ), hOutSetup.ls_azimuth, hOutSetup.ls_elevation, hOutSetup.nchan_out_woLFE, EFAP_MODE_EFAP ) ) != IVAS_ERR_OK ) @@ -559,7 +568,11 @@ ivas_error ivas_param_ism_dec_open( hSpatParamRendCom->dirac_read_idx = 0; hDirAC->spar_to_dirac_write_idx = 0; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) +#else if ( output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) +#endif { if ( ( error = ivas_dirac_allocate_parameters( hSpatParamRendCom, 1 ) ) != IVAS_ERR_OK ) { diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index 4a6053ccd5..ca48e4bdf2 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -971,6 +971,9 @@ ivas_error ivas_masa_dec_reconfigure( int32_t ivas_total_brate, last_ivas_total_brate; int16_t numCldfbAnalyses_old, numCldfbSyntheses_old; ivas_error error; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; +#endif error = IVAS_ERR_OK; @@ -982,7 +985,11 @@ ivas_error ivas_masa_dec_reconfigure( /* renderer might have changed, reselect */ ivas_renderer_select( st_ivas ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend == NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] == NULL ) ) +#else if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend == NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin == NULL ) ) +#endif { /* init a new DirAC dec */ if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_OPEN ) ) != IVAS_ERR_OK ) @@ -995,7 +1002,11 @@ ivas_error ivas_masa_dec_reconfigure( if ( st_ivas->hDirAC != NULL ) { /* close all unnecessary parametric decoding and rendering */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif ivas_dirac_rend_close( &( st_ivas->hDirACRend ) ); ivas_spat_hSpatParamRendCom_close( &( st_ivas->hSpatParamRendCom ) ); ivas_dirac_dec_close( &( st_ivas->hDirAC ) ); @@ -1019,7 +1030,11 @@ ivas_error ivas_masa_dec_reconfigure( sts[0]->bit_stream = bit_stream + num_bits; num_bits += (int16_t) ( st_ivas->hSCE[sce_id]->element_brate / FRAMES_PER_SEC ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] != NULL ) +#else if ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin != NULL ) +#endif { if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_RECONFIGURE ) ) != IVAS_ERR_OK ) { @@ -1046,7 +1061,11 @@ ivas_error ivas_masa_dec_reconfigure( { st_ivas->hCPE[cpe_id]->nchan_out = 1; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] != NULL ) ) +#else if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin != NULL ) ) +#endif { if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_RECONFIGURE ) ) != IVAS_ERR_OK ) { @@ -1058,7 +1077,11 @@ ivas_error ivas_masa_dec_reconfigure( { st_ivas->hCPE[cpe_id]->nchan_out = CPE_CHANNELS; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] != NULL ) ) +#else if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin != NULL ) ) +#endif { if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_RECONFIGURE ) ) != IVAS_ERR_OK ) { @@ -1068,10 +1091,22 @@ ivas_error ivas_masa_dec_reconfigure( } } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) + { + if ( st_ivas->hDiracDecBin[pos_idx] != NULL ) + { +#else if ( st_ivas->hDiracDecBin != NULL ) { +#endif /* regularization factor is bitrate-dependent */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + st_ivas->hDiracDecBin[pos_idx]->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); + } +#else st_ivas->hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); +#endif } /*-----------------------------------------------------------------* @@ -1149,7 +1184,11 @@ void ivas_spar_param_to_masa_param_mapping( hDirAC = st_ivas->hDirAC; hSpatParamRendCom = st_ivas->hSpatParamRendCom; hSpatParamRendCom->numSimultaneousDirections = 1; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + hDiffuseDist = st_ivas->hDiracDecBin[0]->hDiffuseDist; /* TODO: this may need to use different pose indices */ +#else hDiffuseDist = st_ivas->hDiracDecBin->hDiffuseDist; +#endif nchan_transport = st_ivas->nchan_transport; band_grouping = hDirAC->band_grouping; hSpar = st_ivas->hSpar; diff --git a/lib_dec/ivas_mct_dec.c b/lib_dec/ivas_mct_dec.c index e99cf63fa6..f31aff2327 100755 --- a/lib_dec/ivas_mct_dec.c +++ b/lib_dec/ivas_mct_dec.c @@ -1161,11 +1161,19 @@ static ivas_error ivas_mc_dec_reconfig( ivas_td_binaural_close( &st_ivas->hBinRendererTd ); } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] != NULL ) +#else if ( st_ivas->hDiracDecBin != NULL ) +#endif { if ( st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif } } diff --git a/lib_dec/ivas_sba_dec.c b/lib_dec/ivas_sba_dec.c index 763a1b222f..5659a96b24 100755 --- a/lib_dec/ivas_sba_dec.c +++ b/lib_dec/ivas_sba_dec.c @@ -357,12 +357,21 @@ ivas_error ivas_sba_dec_reconfigure( * TD Decorrelator *-----------------------------------------------------------------*/ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] != NULL ) + { + if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin[0]->hTdDecorr ), &( st_ivas->hDiracDecBin[0]->useTdDecorr ) ) ) != IVAS_ERR_OK ) + { + return error; + } +#else if ( st_ivas->hDiracDecBin != NULL ) { if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin->hTdDecorr ), &( st_ivas->hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) { return error; } +#endif } /*-----------------------------------------------------------------* @@ -450,7 +459,11 @@ ivas_error ivas_sba_dec_digest_tc( ivas_spar_dec_digest_tc( st_ivas, st_ivas->nchan_transport, nCldfbSlots, nSamplesForRendering ); } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] != NULL && ( st_ivas->hDiracDecBin[0]->useTdDecorr ) ) +#else if ( st_ivas->hDiracDecBin != NULL && ( st_ivas->hDiracDecBin->useTdDecorr ) ) +#endif { int16_t nSamplesLeftForTD, default_frame; float *decorr_signal[BINAURAL_CHANNELS]; @@ -468,9 +481,17 @@ ivas_error ivas_sba_dec_digest_tc( while ( nSamplesLeftForTD ) { int16_t nSamplesToDecorr = min( nSamplesLeftForTD, default_frame ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->hTdDecorr ) +#else if ( st_ivas->hDiracDecBin->hTdDecorr ) +#endif { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_td_decorr_process( st_ivas->hDiracDecBin[0]->hTdDecorr, p_tc, decorr_signal, nSamplesToDecorr ); +#else ivas_td_decorr_process( st_ivas->hDiracDecBin->hTdDecorr, p_tc, decorr_signal, nSamplesToDecorr ); +#endif } for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) { diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 5c70409ac2..003a45056c 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -997,7 +997,11 @@ typedef struct Decoder_Struct BINAURAL_RENDERER_HANDLE hBinRenderer; /* fastconv binaural renderer handle */ BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd; /* Time domain binaural object renderer handle */ TDREND_HRFILT_FiltSet_t *hHrtfTD; /* pointer to HRTF data for TD renderer */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + DIRAC_DEC_BIN_HANDLE hDiracDecBin[MAX_HEAD_ROT_POSES]; /* parametric binaural renderer handle */ +#else DIRAC_DEC_BIN_HANDLE hDiracDecBin; /* parametric binaural renderer handle */ +#endif LSSETUP_CONVERSION_HANDLE hLsSetUpConversion; /* MC LS configuration convertion handle */ EFAP_HANDLE hEFAPdata; /* EFAP structure */ VBAP_HANDLE hVBAPdata; /* VBAP structure */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 8a3f33b80b..258a8bf7d8 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -95,11 +95,26 @@ static void ivas_dirac_dec_binaural_internal( Decoder_Struct *st_ivas, COMBINED_ static void ivas_dirac_dec_decorrelate_slot( DIRAC_DEC_BIN_HANDLE hDiracDecBin, const int16_t num_freq_bands, const int16_t slot, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float decRe[][CLDFB_NO_CHANNELS_MAX], float decIm[][CLDFB_NO_CHANNELS_MAX] ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT +static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t subframe, const int16_t isHeadtracked, float *subFrameTotalEne, float *IIReneLimiter ); +static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked, const float *subFrameTotalEne, const float *IIReneLimiter ); +#else static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked ); +#endif static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t isHeadtracked ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN +static void ivas_dirac_dec_binaural_process_output( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, HANDLE_CLDFB_FILTER_BANK cldfbSynDec[MAX_OUTPUT_CHANNELS], + float *output_f[], float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const int16_t max_band_decorr, const int16_t numInChannels, const int16_t processReverb, const int16_t subframe, + float outRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float outIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float reverbRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float reverbIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float decorrRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float decorrIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const uint8_t recompute ); +#else static void ivas_dirac_dec_binaural_process_output( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, HANDLE_CLDFB_FILTER_BANK cldfbSynDec[MAX_OUTPUT_CHANNELS], float *output_f[], float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const int16_t numInChannels, const int16_t processReverb, const int16_t subframe ); +#endif static void adaptTransportSignalsHeadtracked( COMBINED_ORIENTATION_HANDLE hHeadTrackData, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t nBins, const int16_t nSlots, float Rmat[3][3] ); @@ -138,143 +153,169 @@ ivas_error ivas_dirac_dec_init_binaural_data( float binCenterFreq, tmpFloat; ivas_error error; float frequency_axis[CLDFB_NO_CHANNELS_MAX]; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; + for ( pos_idx = 0; pos_idx < st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses; pos_idx++ ) + { + hDiracDecBin = st_ivas->hDiracDecBin[pos_idx]; +#else hDiracDecBin = st_ivas->hDiracDecBin; +#endif - if ( hDiracDecBin == NULL ) - { - if ( ( hDiracDecBin = (DIRAC_DEC_BIN_HANDLE) malloc( sizeof( DIRAC_DEC_BIN_DATA ) ) ) == NULL ) + if ( hDiracDecBin == NULL ) { - return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC binaural handle " ); + if ( ( hDiracDecBin = (DIRAC_DEC_BIN_HANDLE) malloc( sizeof( DIRAC_DEC_BIN_DATA ) ) ) == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC binaural handle " ); + } + + hDiracDecBin->hTdDecorr = NULL; + hDiracDecBin->hReverb = NULL; + hDiracDecBin->h_freq_domain_decorr_ap_params = NULL; + hDiracDecBin->h_freq_domain_decorr_ap_state = NULL; } - hDiracDecBin->hTdDecorr = NULL; - hDiracDecBin->hReverb = NULL; - hDiracDecBin->h_freq_domain_decorr_ap_params = NULL; - hDiracDecBin->h_freq_domain_decorr_ap_state = NULL; - } + output_Fs = st_ivas->hDecoderConfig->output_Fs; + nBins = st_ivas->hSpatParamRendCom->num_freq_bands; + renderer_type = st_ivas->renderer_type; - output_Fs = st_ivas->hDecoderConfig->output_Fs; - nBins = st_ivas->hSpatParamRendCom->num_freq_bands; - renderer_type = st_ivas->renderer_type; + for ( j = 0; j < BINAURAL_CHANNELS; j++ ) + { + for ( k = 0; k < BINAURAL_CHANNELS + 1; k++ ) + { + set_zero( hDiracDecBin->processMtxRe[j][k], nBins ); + set_zero( hDiracDecBin->processMtxIm[j][k], nBins ); + } - for ( j = 0; j < BINAURAL_CHANNELS; j++ ) - { - for ( k = 0; k < BINAURAL_CHANNELS + 1; k++ ) + for ( k = 0; k < BINAURAL_CHANNELS; k++ ) + { + set_zero( hDiracDecBin->processMtxDecRe[j][k], nBins ); + set_zero( hDiracDecBin->processMtxDecIm[j][k], nBins ); + } + set_zero( hDiracDecBin->ChEnePrev[j], nBins ); + set_zero( hDiracDecBin->ChEneOutPrev[j], nBins ); + } + set_zero( hDiracDecBin->ChCrossRePrev, nBins ); + set_zero( hDiracDecBin->ChCrossImPrev, nBins ); + set_zero( hDiracDecBin->ChCrossReOutPrev, nBins ); + set_zero( hDiracDecBin->ChCrossImOutPrev, nBins ); + hDiracDecBin->renderStereoOutputInsteadOfBinaural = 0; + + + for ( bin = 0; bin < nBins; bin++ ) { - set_zero( hDiracDecBin->processMtxRe[j][k], nBins ); - set_zero( hDiracDecBin->processMtxIm[j][k], nBins ); + binCenterFreq = ( (float) bin + CLDFB_HALF_BIN_FREQUENCY_OFFSET ) / (float) nBins * ( (float) output_Fs / 2.0f ); + /* These formulas and values are from Christian Borss's publication for binaural diffuse field coherence */ + tmpFloat = max( 0.0f, 1.0f - binCenterFreq / 2700.0f ); + hDiracDecBin->diffuseFieldCoherence[bin] = tmpFloat * sinf( binCenterFreq * EVS_PI / 550.0f ) / ( binCenterFreq * EVS_PI / 550.0f ); } - for ( k = 0; k < BINAURAL_CHANNELS; k++ ) + for ( bin = 0; bin < BINAURAL_COHERENCE_DIFFERENCE_BINS; bin++ ) { - set_zero( hDiracDecBin->processMtxDecRe[j][k], nBins ); - set_zero( hDiracDecBin->processMtxDecIm[j][k], nBins ); + hDiracDecBin->diffuseFieldCoherenceX[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceX[bin]; + hDiracDecBin->diffuseFieldCoherenceY[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceY[bin]; + hDiracDecBin->diffuseFieldCoherenceZ[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceZ[bin]; } - set_zero( hDiracDecBin->ChEnePrev[j], nBins ); - set_zero( hDiracDecBin->ChEneOutPrev[j], nBins ); - } - set_zero( hDiracDecBin->ChCrossRePrev, nBins ); - set_zero( hDiracDecBin->ChCrossImPrev, nBins ); - set_zero( hDiracDecBin->ChCrossReOutPrev, nBins ); - set_zero( hDiracDecBin->ChCrossImOutPrev, nBins ); - hDiracDecBin->renderStereoOutputInsteadOfBinaural = 0; + if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC ) /* Indication of binaural rendering without room effect */ + { + set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); + hDiracDecBin->hReverb = NULL; + } + else if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) /* Indication of binaural rendering with room effect */ + { + mvr2r( hHrtfParambin->parametricEarlyPartEneCorrection, hDiracDecBin->earlyPartEneCorrection, nBins ); - for ( bin = 0; bin < nBins; bin++ ) - { - binCenterFreq = ( (float) bin + CLDFB_HALF_BIN_FREQUENCY_OFFSET ) / (float) nBins * ( (float) output_Fs / 2.0f ); - /* These formulas and values are from Christian Borss's publication for binaural diffuse field coherence */ - tmpFloat = max( 0.0f, 1.0f - binCenterFreq / 2700.0f ); - hDiracDecBin->diffuseFieldCoherence[bin] = tmpFloat * sinf( binCenterFreq * EVS_PI / 550.0f ) / ( binCenterFreq * EVS_PI / 550.0f ); - } + /* reconfiguration needed when Reverb. parameters are changed -> close and open the handle again */ + if ( hDiracDecBin->hReverb != NULL && ( ( hDiracDecBin->hReverb->numBins != nBins ) || + ( hDiracDecBin->hReverb->blockSize != CLDFB_SLOTS_PER_SUBFRAME ) ) ) + { + ivas_binaural_reverb_close( &( hDiracDecBin->hReverb ) ); + } - for ( bin = 0; bin < BINAURAL_COHERENCE_DIFFERENCE_BINS; bin++ ) - { - hDiracDecBin->diffuseFieldCoherenceX[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceX[bin]; - hDiracDecBin->diffuseFieldCoherenceY[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceY[bin]; - hDiracDecBin->diffuseFieldCoherenceZ[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceZ[bin]; - } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( hDiracDecBin->hReverb == NULL && pos_idx == 0 ) /* open reverb only for the main direction */ +#else + if ( hDiracDecBin->hReverb == NULL ) +#endif + { + if ( ( error = ivas_binaural_reverb_open( &hDiracDecBin->hReverb, + nBins, + CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, NULL, + st_ivas->hIntSetup.output_config, + output_Fs, + RENDERER_BINAURAL_PARAMETRIC_ROOM, + st_ivas->hHrtfFastConv, + st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) + { + return error; + } + } + } + else if ( renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); + hDiracDecBin->hReverb = NULL; + hDiracDecBin->renderStereoOutputInsteadOfBinaural = 1; + } + else /* Not valid renderer type for this renderer */ + { + assert( false ); + } - if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC ) /* Indication of binaural rendering without room effect */ - { - set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); - hDiracDecBin->hReverb = NULL; - } - else if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) /* Indication of binaural rendering with room effect */ - { - mvr2r( hHrtfParambin->parametricEarlyPartEneCorrection, hDiracDecBin->earlyPartEneCorrection, nBins ); + hDiracDecBin->hDiffuseDist = NULL; /* Memory is allocated from stack during runtime when needed */ + + if ( hDiracDecBin->hTdDecorr == NULL ) + { + hDiracDecBin->useTdDecorr = 0; + } - /* reconfiguration needed when Reverb. parameters are changed -> close and open the handle again */ - if ( hDiracDecBin->hReverb != NULL && ( ( hDiracDecBin->hReverb->numBins != nBins ) || - ( hDiracDecBin->hReverb->blockSize != CLDFB_SLOTS_PER_SUBFRAME ) ) ) + if ( hDiracDecBin->h_freq_domain_decorr_ap_params != NULL ) { - ivas_binaural_reverb_close( &( hDiracDecBin->hReverb ) ); + ivas_dirac_dec_decorr_close( &hDiracDecBin->h_freq_domain_decorr_ap_params, &hDiracDecBin->h_freq_domain_decorr_ap_state ); } - if ( hDiracDecBin->hReverb == NULL ) +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( pos_idx == 0) /* open decorrelator only for the main direction */ { - if ( ( error = ivas_binaural_reverb_open( &hDiracDecBin->hReverb, - nBins, - CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, NULL, - st_ivas->hIntSetup.output_config, - output_Fs, - RENDERER_BINAURAL_PARAMETRIC_ROOM, - st_ivas->hHrtfFastConv, - st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) +#endif + if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( hDiracDecBin->hTdDecorr ), &( hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) { return error; } + if ( !hDiracDecBin->useTdDecorr && !( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM ) ) + { + ivas_dirac_dec_get_frequency_axis( frequency_axis, output_Fs, nBins ); + if ( ( error = ivas_dirac_dec_decorr_open( &( hDiracDecBin->h_freq_domain_decorr_ap_params ), + &( hDiracDecBin->h_freq_domain_decorr_ap_state ), + nBins, + BINAURAL_CHANNELS, + BINAURAL_CHANNELS, + DIRAC_SYNTHESIS_PSD_LS, + frequency_axis, + BINAURAL_CHANNELS, + output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN } - } - else if ( renderer_type == RENDERER_STEREO_PARAMETRIC ) - { - set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); - hDiracDecBin->hReverb = NULL; - hDiracDecBin->renderStereoOutputInsteadOfBinaural = 1; - } - else /* Not valid renderer type for this renderer */ - { - assert( false ); - } - - hDiracDecBin->hDiffuseDist = NULL; /* Memory is allocated from stack during runtime when needed */ - - if ( hDiracDecBin->hTdDecorr == NULL ) - { - hDiracDecBin->useTdDecorr = 0; - } - - if ( hDiracDecBin->h_freq_domain_decorr_ap_params != NULL ) - { - ivas_dirac_dec_decorr_close( &hDiracDecBin->h_freq_domain_decorr_ap_params, &hDiracDecBin->h_freq_domain_decorr_ap_state ); - } - - if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( hDiracDecBin->hTdDecorr ), &( hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) - { - return error; - } - - if ( !hDiracDecBin->useTdDecorr && !( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM ) ) - { - ivas_dirac_dec_get_frequency_axis( frequency_axis, output_Fs, nBins ); - if ( ( error = ivas_dirac_dec_decorr_open( &( hDiracDecBin->h_freq_domain_decorr_ap_params ), - &( hDiracDecBin->h_freq_domain_decorr_ap_state ), - nBins, - BINAURAL_CHANNELS, - BINAURAL_CHANNELS, - DIRAC_SYNTHESIS_PSD_LS, - frequency_axis, - BINAURAL_CHANNELS, - output_Fs ) ) != IVAS_ERR_OK ) + else { - return error; + hDiracDecBin->useTdDecorr = st_ivas->hDiracDecBin[0]->useTdDecorr; /* copy the flag, but the implementation re-uses the decorrelated signal */ } - } +#endif - hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); + hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + st_ivas->hDiracDecBin[pos_idx] = hDiracDecBin; + } +#else st_ivas->hDiracDecBin = hDiracDecBin; +#endif /* allocate transport channels*/ if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) @@ -302,11 +343,31 @@ void ivas_dirac_dec_close_binaural_data( DIRAC_DEC_BIN_HANDLE *hBinaural /* i/o: decoder DirAC binaural data handle */ ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; +#endif if ( hBinaural == NULL || *hBinaural == NULL ) { return; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) + { + if ( hBinaural[pos_idx] != NULL ) + { + if ( hBinaural[pos_idx]->hReverb != NULL ) + { + ivas_binaural_reverb_close( &( hBinaural[pos_idx]->hReverb ) ); + } + + ivas_td_decorr_dec_close( &( hBinaural[pos_idx]->hTdDecorr ) ); + + free( hBinaural[pos_idx] ); + hBinaural[pos_idx] = NULL; + } + } +#else if ( ( *hBinaural )->hReverb != NULL ) { ivas_binaural_reverb_close( &( ( *hBinaural )->hReverb ) ); @@ -320,6 +381,7 @@ void ivas_dirac_dec_close_binaural_data( free( *hBinaural ); *hBinaural = NULL; +#endif return; } @@ -423,6 +485,7 @@ void ivas_dirac_dec_binaural_render( { int16_t n_samples_sf = slot_size * hSpatParamRendCom->subframe_nbslots[subframe_idx]; ivas_dirac_dec_binaural_internal( st_ivas, st_ivas->hCombinedOrientationData, output_f_local, nchan_transport, subframe_idx ); + for ( ch = 0; ch < nchan_out; ch++ ) { output_f_local[ch] += n_samples_sf; @@ -519,8 +582,11 @@ void ivas_dirac_dec_binaural( { ivas_spar_dec_set_render_map( st_ivas, DEFAULT_JBM_CLDFB_TIMESLOTS ); } - +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->useTdDecorr ) +#else if ( st_ivas->hDiracDecBin->useTdDecorr ) +#endif { float *decorr_signal[BINAURAL_CHANNELS]; int16_t output_frame; @@ -532,7 +598,11 @@ void ivas_dirac_dec_binaural( } output_frame = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_td_decorr_process( st_ivas->hDiracDecBin[0]->hTdDecorr, p_output, decorr_signal, output_frame ); +#else ivas_td_decorr_process( st_ivas->hDiracDecBin->hTdDecorr, p_output, decorr_signal, output_frame ); +#endif } if ( nchan_transport == 1 && st_ivas->nchan_transport != 2 && st_ivas->hSCE[0]->hCoreCoder[0] != NULL && st_ivas->hSCE[0]->hCoreCoder[0]->cng_sba_flag ) @@ -588,13 +658,31 @@ static void ivas_dirac_dec_binaural_internal( int16_t nBins, offsetSamples; int16_t i, j; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; + MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + float tmp_Cldfb_out_re[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float tmp_Cldfb_out_im[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + /* these allow re-using the reverb and freq-domain decorrelator signals from ivas_dirac_dec_binaural_process_output() in split rendering for the side renderings */ + float reverbRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float decorrRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float decorrIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT + float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; + float IIReneLimiter[CLDFB_NO_CHANNELS_MAX]; +#endif + + hDiracDecBin = st_ivas->hDiracDecBin[0]; +#else hDiracDecBin = st_ivas->hDiracDecBin; +#endif assert( hDiracDecBin ); hSpatParamRendCom = st_ivas->hSpatParamRendCom; nBins = hSpatParamRendCom->num_freq_bands; offsetSamples = hSpatParamRendCom->slots_rendered * nBins; - /* Setuo internal config */ + /* Setup internal config */ config_data.separateCenterChannelRendering = st_ivas->hOutSetup.separateChannelEnabled; config_data.ivas_format = st_ivas->ivas_format; config_data.mc_mode = st_ivas->mc_mode; @@ -726,7 +814,6 @@ static void ivas_dirac_dec_binaural_internal( hDiracDecBin->hDiffuseDist = &diffuseDistData; ivas_spar_param_to_masa_param_mapping( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe ); - ivas_sba_prototype_renderer( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe ); } @@ -742,14 +829,19 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + /* in case of split rendering, determine the prototype rotation based on the main direction and use the same prototypes for the offset directions */ +#endif adaptTransportSignalsHeadtracked( hCombinedOrientationData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, hSpatParamRendCom->subframe_nbslots[subframe], Rmat ); ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( hCombinedOrientationData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, hSpatParamRendCom->subframe_nbslots[subframe], Rmat ); } } +#ifndef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); +#endif if ( config_data.ivas_format == ISM_FORMAT ) { @@ -764,13 +856,121 @@ static void ivas_dirac_dec_binaural_internal( max_band_decorr = hDiracDecBin->h_freq_domain_decorr_ap_params->max_band_decorr; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT + ivas_dirac_dec_binaural_formulate_input_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter ); + ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter ); +#else + ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); +#endif +#endif ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); - ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, max_band_decorr, numInChannels, config_data.processReverb, subframe ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + pMultiBinPoseData = &st_ivas->splitBinRend.splitrend.multiBinPoseData; + + if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) + { + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, + reverbRe, reverbIm, decorrRe, decorrIm, 1 ); + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) + { + mvr2r(tmp_Cldfb_out_re[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + mvr2r(tmp_Cldfb_out_im[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + } + } + } + else + { + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, NULL, NULL, + reverbRe, reverbIm, decorrRe, decorrIm, 1 ); + } +#endif hDiracDecBin->hDiffuseDist = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) + { + /* quaternion-based rotation from ivas_binRenderer_internal.c:ivas_binRenderer(), but using absolute rotation instead of delta rotations */ + IVAS_QUATERNION Quaternions_rot, Quaternions_abs, *Quaternions_ref; + float Rmat_local[3][3]; + + if ( hCombinedOrientationData ) + { + Quaternions_ref = &hCombinedOrientationData->Quaternions[0]; + Quaternions_rot.w = -3.0f; /* signal to use Euler */ + Quaternions_abs.w = -3.0f; /* signal to use Euler */ + Quat2EulerDegree( *Quaternions_ref, &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + for ( pos_idx = 1; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + { + Quaternions_rot.x = Quaternions_abs.x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternions_rot.y = Quaternions_abs.y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternions_rot.z = Quaternions_abs.z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; + + QuatToRotMat( Quaternions_rot, Rmat_local ); + + hDiracDecBin = st_ivas->hDiracDecBin[pos_idx]; + assert(hDiracDecBin != NULL && "No DiracDecBin handle for this position"); + if ( config_data.ivas_format == SBA_FORMAT ) + { + hDiracDecBin->hDiffuseDist = &diffuseDistData; + } + +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT + /* re-use input covariance for the side renderings */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + mvr2r(st_ivas->hDiracDecBin[0]->ChEne[ch], hDiracDecBin->ChEne[ch], hSpatParamRendCom->num_freq_bands); + } + mvr2r(st_ivas->hDiracDecBin[0]->ChCrossRe, hDiracDecBin->ChCrossRe, hSpatParamRendCom->num_freq_bands); + mvr2r(st_ivas->hDiracDecBin[0]->ChCrossIm, hDiracDecBin->ChCrossIm, hSpatParamRendCom->num_freq_bands); + + ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat_local, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter ); + +#else + ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat_local, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); +#endif + + ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); + + /* re-use reverb and decorr from main direction for the sides */ + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, + reverbRe, reverbIm, + decorrRe, decorrIm, 0); + + /* copy from temporary buffer to the main split rendering buffer */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) + { + mvr2r(tmp_Cldfb_out_re[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + mvr2r(tmp_Cldfb_out_im[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + } + } + + hDiracDecBin->hDiffuseDist = NULL; + } + } + } + /* update this counter only after the last rendering of split directions */ +#endif hSpatParamRendCom->slots_rendered += hSpatParamRendCom->subframe_nbslots[subframe]; hSpatParamRendCom->subframes_rendered++; @@ -832,36 +1032,29 @@ static void ivas_dirac_dec_decorrelate_slot( return; } - -static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT +static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], - float Rmat[3][3], const int16_t subframe, - const int16_t isHeadtracked ) + const int16_t isHeadtracked, + float *subFrameTotalEne, + float *IIReneLimiter ) { int16_t ch, slot, bin; - int16_t separateCenterChannelRendering; - int16_t nBins, idx; - float frameMeanDiffusenessEneWeight[CLDFB_NO_CHANNELS_MAX]; + int16_t nBins; float IIReneLimiterFactor; float qualityBasedSmFactor; float lowBitRateEQ[CLDFB_NO_CHANNELS_MAX]; uint8_t applyLowBitRateEQ; - int16_t dirac_read_idx; - float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; - PARAMBIN_HRTF_GAIN_CACHE gainCache[MAX_GAIN_CACHE_SIZE]; IVAS_FORMAT ivas_format; - MC_MODE mc_mode; int32_t ivas_total_brate; int16_t nchan_transport; - separateCenterChannelRendering = hConfig->separateCenterChannelRendering; ivas_format = hConfig->ivas_format; - mc_mode = hConfig->mc_mode; ivas_total_brate = hConfig->ivas_total_brate; nchan_transport = hConfig->nchan_transport; qualityBasedSmFactor = hConfig->qualityBasedSmFactor; @@ -870,20 +1063,9 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric set_zero( hDiracDecBin->ChCrossRe, nBins ); set_zero( hDiracDecBin->ChCrossIm, nBins ); - set_zero( hDiracDecBin->ChCrossReOut, nBins ); - set_zero( hDiracDecBin->ChCrossImOut, nBins ); for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { set_zero( hDiracDecBin->ChEne[ch], nBins ); - set_zero( hDiracDecBin->ChEneOut[ch], nBins ); - } - set_zero( hDiracDecBin->frameMeanDiffuseness, nBins ); - - set_zero( frameMeanDiffusenessEneWeight, CLDFB_NO_CHANNELS_MAX ); - - for ( idx = 0; idx < MAX_GAIN_CACHE_SIZE; idx++ ) - { - gainCache[idx].azi = -1000; /* Use -1000 as value for uninitialized cache. */ } /* Determine EQ for low bit rates (13.2 and 16.4 kbps) */ @@ -909,7 +1091,6 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric /* Formulate input and target covariance matrices for this subframe */ set_zero( subFrameTotalEne, CLDFB_NO_CHANNELS_MAX ); - dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; /* Calculate input covariance matrix */ for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) @@ -964,6 +1145,96 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric } } + /* Temporal IIR-type smoothing of covariance matrices. Also apply encoding quality based smoothing factor. */ + if ( ivas_format == MASA_FORMAT && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) + { + IIReneLimiterFactor = 16.0f + ( 1.0f - qualityBasedSmFactor ); + } + else + { + IIReneLimiterFactor = 8.0f + ( 1.0f - qualityBasedSmFactor ); + } + for ( bin = 0; bin < nBins; bin++ ) + { + float eneRatio; + + /* Temporally smooth cov mtx estimates for resulting mixing matrix stability. The design principle is that + * the energy history (IIR) must not be more than double of the current frame energy. This provides more + * robust performance at energy offsets when compared to typical IIR averaging. */ + eneRatio = ( hDiracDecBin->ChEne[0][bin] + hDiracDecBin->ChEne[1][bin] ) / fmaxf( 1e-12f, ( hDiracDecBin->ChEnePrev[0][bin] + hDiracDecBin->ChEnePrev[1][bin] ) ); + IIReneLimiter[bin] = fminf( 1.0f, eneRatio * IIReneLimiterFactor ); + + hDiracDecBin->ChCrossRe[bin] *= qualityBasedSmFactor; + hDiracDecBin->ChCrossIm[bin] *= qualityBasedSmFactor; + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEne[ch][bin] *= qualityBasedSmFactor; + } + + hDiracDecBin->ChCrossRe[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossRePrev[bin]; + hDiracDecBin->ChCrossIm[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossImPrev[bin]; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEne[ch][bin] += IIReneLimiter[bin] * hDiracDecBin->ChEnePrev[ch][bin]; + } + + /* Store energy values and coefficients for next round */ + hDiracDecBin->ChCrossRePrev[bin] = hDiracDecBin->ChCrossRe[bin]; + hDiracDecBin->ChCrossImPrev[bin] = hDiracDecBin->ChCrossIm[bin]; + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEnePrev[ch][bin] = hDiracDecBin->ChEne[ch][bin]; + } + } + + return; +} + +static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( + DIRAC_DEC_BIN_HANDLE hDiracDecBin, + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, + PARAMBIN_REND_CONFIG_HANDLE hConfig, + float Rmat[3][3], + const int16_t subframe, + const int16_t isHeadtracked, + const float *subFrameTotalEne, + const float *IIReneLimiter ) +{ + int16_t ch, bin; + int16_t separateCenterChannelRendering; + int16_t nBins, idx; + float frameMeanDiffusenessEneWeight[CLDFB_NO_CHANNELS_MAX]; + float qualityBasedSmFactor; + int16_t dirac_read_idx; + PARAMBIN_HRTF_GAIN_CACHE gainCache[MAX_GAIN_CACHE_SIZE]; + IVAS_FORMAT ivas_format; + MC_MODE mc_mode; + + separateCenterChannelRendering = hConfig->separateCenterChannelRendering; + ivas_format = hConfig->ivas_format; + mc_mode = hConfig->mc_mode; + qualityBasedSmFactor = hConfig->qualityBasedSmFactor; + qualityBasedSmFactor *= qualityBasedSmFactor; + nBins = hSpatParamRendCom->num_freq_bands; /* Actually bins */ + + set_zero( hDiracDecBin->ChCrossReOut, nBins ); + set_zero( hDiracDecBin->ChCrossImOut, nBins ); + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + set_zero( hDiracDecBin->ChEneOut[ch], nBins ); + } + set_zero( hDiracDecBin->frameMeanDiffuseness, nBins ); + + set_zero( frameMeanDiffusenessEneWeight, CLDFB_NO_CHANNELS_MAX ); + + for ( idx = 0; idx < MAX_GAIN_CACHE_SIZE; idx++ ) + { + gainCache[idx].azi = -1000; /* Use -1000 as value for uninitialized cache. */ + } + + dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; + /* Determine target covariance matrix containing target binaural properties */ for ( bin = 0; bin < nBins; bin++ ) { @@ -1187,14 +1458,400 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric hDiracDecBin->frameMeanDiffuseness[bin] /= fmaxf( 1e-12f, frameMeanDiffusenessEneWeight[bin] ); } - /* Temporal IIR-type smoothing of covariance matrices. Also apply encoding quality based smoothing factor. */ - if ( ivas_format == MASA_FORMAT && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) - { - IIReneLimiterFactor = 16.0f + ( 1.0f - qualityBasedSmFactor ); - } - else + for ( bin = 0; bin < nBins; bin++ ) { - IIReneLimiterFactor = 8.0f + ( 1.0f - qualityBasedSmFactor ); + hDiracDecBin->ChCrossReOut[bin] *= qualityBasedSmFactor; + hDiracDecBin->ChCrossImOut[bin] *= qualityBasedSmFactor; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEneOut[ch][bin] *= qualityBasedSmFactor; + } + + hDiracDecBin->ChCrossReOut[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossReOutPrev[bin]; + hDiracDecBin->ChCrossImOut[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossImOutPrev[bin]; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEneOut[ch][bin] += IIReneLimiter[bin] * hDiracDecBin->ChEneOutPrev[ch][bin]; + } + + /* Store energy values and coefficients for next round */ + hDiracDecBin->ChCrossReOutPrev[bin] = hDiracDecBin->ChCrossReOut[bin]; + hDiracDecBin->ChCrossImOutPrev[bin] = hDiracDecBin->ChCrossImOut[bin]; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEneOutPrev[ch][bin] = hDiracDecBin->ChEneOut[ch][bin]; + } + } + + return; +} + +#else +static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( + DIRAC_DEC_BIN_HANDLE hDiracDecBin, + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, + PARAMBIN_REND_CONFIG_HANDLE hConfig, + float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float Rmat[3][3], + const int16_t subframe, + const int16_t isHeadtracked ) +{ + int16_t ch, slot, bin; + int16_t separateCenterChannelRendering; + int16_t nBins, idx; + float frameMeanDiffusenessEneWeight[CLDFB_NO_CHANNELS_MAX]; + float IIReneLimiterFactor; + float qualityBasedSmFactor; + float lowBitRateEQ[CLDFB_NO_CHANNELS_MAX]; + uint8_t applyLowBitRateEQ; + int16_t dirac_read_idx; + float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; + PARAMBIN_HRTF_GAIN_CACHE gainCache[MAX_GAIN_CACHE_SIZE]; + IVAS_FORMAT ivas_format; + MC_MODE mc_mode; + int32_t ivas_total_brate; + int16_t nchan_transport; + + separateCenterChannelRendering = hConfig->separateCenterChannelRendering; + ivas_format = hConfig->ivas_format; + mc_mode = hConfig->mc_mode; + ivas_total_brate = hConfig->ivas_total_brate; + nchan_transport = hConfig->nchan_transport; + qualityBasedSmFactor = hConfig->qualityBasedSmFactor; + qualityBasedSmFactor *= qualityBasedSmFactor; + nBins = hSpatParamRendCom->num_freq_bands; /* Actually bins */ + + set_zero( hDiracDecBin->ChCrossRe, nBins ); + set_zero( hDiracDecBin->ChCrossIm, nBins ); + set_zero( hDiracDecBin->ChCrossReOut, nBins ); + set_zero( hDiracDecBin->ChCrossImOut, nBins ); + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + set_zero( hDiracDecBin->ChEne[ch], nBins ); + set_zero( hDiracDecBin->ChEneOut[ch], nBins ); + } + set_zero( hDiracDecBin->frameMeanDiffuseness, nBins ); + + set_zero( frameMeanDiffusenessEneWeight, CLDFB_NO_CHANNELS_MAX ); + + for ( idx = 0; idx < MAX_GAIN_CACHE_SIZE; idx++ ) + { + gainCache[idx].azi = -1000; /* Use -1000 as value for uninitialized cache. */ + } + + /* Determine EQ for low bit rates (13.2 and 16.4 kbps) */ + applyLowBitRateEQ = 0; + if ( ( ivas_format == MASA_FORMAT || ivas_format == MC_FORMAT ) && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) + { + applyLowBitRateEQ = 1; + if ( ivas_total_brate == IVAS_16k4 ) + { + for ( bin = 0; bin < LOW_BIT_RATE_BINAURAL_EQ_BINS; bin++ ) + { + lowBitRateEQ[bin + LOW_BIT_RATE_BINAURAL_EQ_OFFSET] = lowBitRateBinauralEQ[bin] * 0.5f + 0.5f; + } + } + else + { + for ( bin = 0; bin < LOW_BIT_RATE_BINAURAL_EQ_BINS; bin++ ) + { + lowBitRateEQ[bin + LOW_BIT_RATE_BINAURAL_EQ_OFFSET] = lowBitRateBinauralEQ[bin]; + } + } + } + + /* Formulate input and target covariance matrices for this subframe */ + set_zero( subFrameTotalEne, CLDFB_NO_CHANNELS_MAX ); + dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; + + /* Calculate input covariance matrix */ + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + float instEne; + + instEne = ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ); + instEne += ( inIm[ch][slot][bin] * inIm[ch][slot][bin] ); + hDiracDecBin->ChEne[ch][bin] += instEne; + subFrameTotalEne[bin] += instEne; + } + hDiracDecBin->ChCrossRe[bin] += inRe[0][slot][bin] * inRe[1][slot][bin]; + hDiracDecBin->ChCrossRe[bin] += inIm[0][slot][bin] * inIm[1][slot][bin]; + hDiracDecBin->ChCrossIm[bin] += inRe[0][slot][bin] * inIm[1][slot][bin]; + hDiracDecBin->ChCrossIm[bin] -= inIm[0][slot][bin] * inRe[1][slot][bin]; + } + } + + /* Apply EQ at low bit rates */ + if ( applyLowBitRateEQ ) + { + int16_t lastEqBin = LOW_BIT_RATE_BINAURAL_EQ_OFFSET + LOW_BIT_RATE_BINAURAL_EQ_BINS - 1; + + for ( bin = LOW_BIT_RATE_BINAURAL_EQ_OFFSET; bin < lastEqBin; bin++ ) + { + subFrameTotalEne[bin] *= lowBitRateEQ[bin]; + } + for ( ; bin < nBins; bin++ ) + { + subFrameTotalEne[bin] *= lowBitRateEQ[lastEqBin]; + } + } + + if ( ivas_format == SBA_FORMAT && nchan_transport == 2 ) + { + float tempRe, tempIm; + + set_zero( subFrameTotalEne, CLDFB_NO_CHANNELS_MAX ); + + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + tempRe = inRe[0][slot][bin] + inRe[1][slot][bin]; + tempIm = inIm[0][slot][bin] + inIm[1][slot][bin]; + subFrameTotalEne[bin] += tempRe * tempRe + tempIm * tempIm; + } + } + } + + /* Determine target covariance matrix containing target binaural properties */ + for ( bin = 0; bin < nBins; bin++ ) + { + float diffuseness = 1.0f; /* ratio1 and ratio2 are subtracted from diffuseness further below */ + float surCoh = 0.0f, spreadCoh = 0.0f; /* Default values if spreadSurroundCoherenceApplied == false */ + float diffEne, dirEne, meanEnePerCh; + int16_t dirIndex; + + /* When BINAURAL_ROOM is not indicated, hBinaural->earlyPartEneCorrection[bin] values are all 1.0f. + * When BINAURAL_ROOM is indicated, the binaural audio output is based on combined use of the + * HRTF data set and a BRIR-based data set. The HRTF data set is spectrally corrected to match + * the early spectrum of the BRIR data, using the spectral correction data in + * hBinaural->earlyPartEneCorrection[bin], based on the BRIR set. */ + meanEnePerCh = hDiracDecBin->earlyPartEneCorrection[bin] * subFrameTotalEne[bin] / 2.0f; + + /* Determine direct part target covariance matrix (for 1 or 2 directions) */ + for ( dirIndex = 0; dirIndex < hSpatParamRendCom->numSimultaneousDirections; dirIndex++ ) + { + int16_t aziDeg, eleDeg; + float lRealp, lImagp, rRealp, rImagp; + float lRealpTmp, lImagpTmp, rRealpTmp, rImagpTmp; + float hrtfEne[BINAURAL_CHANNELS], hrtfCrossRe, hrtfCrossIm, ratio; + + if ( dirIndex == 0 ) /* For first of the two simultaneous directions */ + { + aziDeg = hSpatParamRendCom->azimuth[dirac_read_idx][bin]; + eleDeg = hSpatParamRendCom->elevation[dirac_read_idx][bin]; + ratio = hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin]; + spreadCoh = hSpatParamRendCom->spreadCoherence[dirac_read_idx][bin]; + } + else /* For second of the two simultaneous directions */ + { + if ( ( ratio = hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin] ) < 0.001 ) + { + /* This touches only MASA path where second direction always has smaller ratio and + * for non-2dir it is zero. As the whole direction contribution is multiplied with + * the ratio, a very small ratio does not contribute any energy to output. Thus, + * it is better to save complexity. */ + continue; + } + aziDeg = hSpatParamRendCom->azimuth2[dirac_read_idx][bin]; + eleDeg = hSpatParamRendCom->elevation2[dirac_read_idx][bin]; + spreadCoh = hSpatParamRendCom->spreadCoherence2[dirac_read_idx][bin]; + } + diffuseness -= ratio; /* diffuseness = 1 - ratio1 - ratio2 */ + + if ( separateCenterChannelRendering ) + { + /* In masa + mono rendering mode, the center directions originate from phantom sources, so the + * spread coherence is increased */ + float aziRad, eleRad, doaVectorX, spatialAngleDeg, altSpreadCoh; + + aziRad = (float) aziDeg * PI_OVER_180; + eleRad = (float) eleDeg * PI_OVER_180; + doaVectorX = cosf( aziRad ) * cosf( eleRad ); + spatialAngleDeg = acosf( doaVectorX ) * _180_OVER_PI; + altSpreadCoh = 1.0f - ( spatialAngleDeg / 30.0f ); + spreadCoh = max( spreadCoh, altSpreadCoh ); + } + + getDirectPartGains( bin, aziDeg, eleDeg, &lRealp, &lImagp, &rRealp, &rImagp, hDiracDecBin->renderStereoOutputInsteadOfBinaural, Rmat, &gainCache[( dirIndex * 3 )], isHeadtracked ); + + if ( hDiracDecBin->renderStereoOutputInsteadOfBinaural ) + { + /* Synthesizing spread coherence is not needed for stereo loudspeaker output, + * as directional sound is reproduced with two loudspeakers in any case */ + spreadCoh = 0.0f; + } + + if ( spreadCoh > 0.0f ) + { + float centerMul, sidesMul; + float hrtfEneCenter, hrtfEneSides, hrtfEneRealized, eneCorrectionFactor; + float w1, w2, w3, eq; + + hrtfEneCenter = ( lRealp * lRealp ) + ( lImagp * lImagp ) + ( rRealp * rRealp ) + ( rImagp * rImagp ); + + /* Spread coherence is synthesized as coherent sources at 30 degree horizontal spacing. + * The following formulas determine the gains for these sources. + * spreadCoh = 0: Only panning + * spreadCoh = 0.5: Three sources coherent panning (e.g. 30 0 -30 deg azi) + * spreadCoh = 1.0: Two sources coherent panning with gap (as above, but center is silent) */ + if ( spreadCoh < 0.5f ) + { + /* 0.0f < spreadCoh < 0.5f */ + sidesMul = 0.5774f * spreadCoh * 2.0f; /* sqrt(1/3) = 0.5774f */ + centerMul = 1.0f - ( spreadCoh * 2.0f ) + sidesMul; + } + else + { + /* 0.5f <= spreadCoh < 1.0f */ + centerMul = 2.0f - ( 2.0f * spreadCoh ); + sidesMul = inv_sqrt( centerMul + 2.0f ); + centerMul *= sidesMul; + } + + /* Apply the gain for the center source of the three coherent sources */ + lRealp *= centerMul; + lImagp *= centerMul; + rRealp *= centerMul; + rImagp *= centerMul; + + /* Apply the gain for the left source of the three coherent sources */ + getDirectPartGains( bin, aziDeg + 30, eleDeg, &lRealpTmp, &lImagpTmp, &rRealpTmp, &rImagpTmp, hDiracDecBin->renderStereoOutputInsteadOfBinaural, Rmat, &gainCache[( dirIndex * 3 + 1 )], isHeadtracked ); + + hrtfEneSides = ( lRealpTmp * lRealpTmp ) + ( lImagpTmp * lImagpTmp ) + ( rRealpTmp * rRealpTmp ) + ( rImagpTmp * rImagpTmp ); + lRealp += sidesMul * lRealpTmp; + lImagp += sidesMul * lImagpTmp; + rRealp += sidesMul * rRealpTmp; + rImagp += sidesMul * rImagpTmp; + + /* Apply the gain for the right source of the three coherent sources. + * -30 degrees to 330 wrapping due to internal functions. */ + getDirectPartGains( bin, aziDeg + 330, eleDeg, &lRealpTmp, &lImagpTmp, &rRealpTmp, &rImagpTmp, hDiracDecBin->renderStereoOutputInsteadOfBinaural, Rmat, &gainCache[( dirIndex * 3 + 2 )], isHeadtracked ); + + hrtfEneSides += ( lRealpTmp * lRealpTmp ) + ( lImagpTmp * lImagpTmp ) + ( rRealpTmp * rRealpTmp ) + ( rImagpTmp * rImagpTmp ); + lRealp += sidesMul * lRealpTmp; + lImagp += sidesMul * lImagpTmp; + rRealp += sidesMul * rRealpTmp; + rImagp += sidesMul * rImagpTmp; + + /* Formulate an eneCorrectionFactor that compensates for the coherent summation of the HRTFs */ + hrtfEneRealized = ( lRealp * lRealp ) + ( lImagp * lImagp ) + ( rRealp * rRealp ) + ( rImagp * rImagp ); + eneCorrectionFactor = ( ( hrtfEneSides * sidesMul * sidesMul ) + + ( hrtfEneCenter * centerMul * centerMul ) ) / + max( 1e-12f, hrtfEneRealized ); + + /* Weighting factors to determine appropriate target spectrum for spread coherent sound */ + if ( spreadCoh < 0.5 ) + { + w1 = 1.0f - 2.0f * spreadCoh; + w2 = 2.0f * spreadCoh; + w3 = 0.0f; + } + else + { + w1 = 0.0f; + w2 = 2.0f - 2.0f * spreadCoh; + w3 = 2.0f * spreadCoh - 1.0f; + } + + if ( ( ivas_format == MC_FORMAT && mc_mode == MC_MODE_MCMASA ) ) + { + idx = min( bin, MASA_NUM_DEFINED_SUR_SPR_COH_ENE_BINS - 1 ); + + /* Apply the target spectrum to the eneCorrectionFactor */ + if ( separateCenterChannelRendering ) /* spreadCoh mostly originates from phantom sources in separate channel rendering mode */ + { + eneCorrectionFactor *= w1 * 1.0f + ( w2 + w3 ) * spreadCohEne1[idx]; + } + else + { + eneCorrectionFactor *= w1 * 1.0f + w2 * spreadCohEne05[idx] + w3 * spreadCohEne1[idx]; + } + } + + /* Equalize the spread coherent combined HRTFs */ + eq = min( 4.0f, sqrtf( eneCorrectionFactor ) ); + lRealp *= eq; + lImagp *= eq; + rRealp *= eq; + rImagp *= eq; + } + + hrtfEne[0] = ( lRealp * lRealp ) + ( lImagp * lImagp ); + hrtfEne[1] = ( rRealp * rRealp ) + ( rImagp * rImagp ); + hrtfCrossRe = ( lRealp * rRealp ) + ( lImagp * rImagp ); + hrtfCrossIm = ( -lImagp * rRealp ) + ( lRealp * rImagp ); + + /* Add direct part (1 or 2) covariance matrix */ + dirEne = ratio * meanEnePerCh; + hDiracDecBin->ChEneOut[0][bin] += dirEne * hrtfEne[0]; /* Dir ene part*/ + hDiracDecBin->ChEneOut[1][bin] += dirEne * hrtfEne[1]; + hDiracDecBin->ChCrossReOut[bin] += dirEne * hrtfCrossRe; /* Dir cross re */ + hDiracDecBin->ChCrossImOut[bin] += dirEne * hrtfCrossIm; /* Dir cross im */ + } + + /* Add diffuse / ambient part covariance matrix */ + diffuseness = max( 0.0f, diffuseness ); + diffEne = diffuseness * meanEnePerCh; + surCoh = hSpatParamRendCom->surroundingCoherence[dirac_read_idx][bin]; + if ( ( ivas_format == MC_FORMAT && mc_mode == MC_MODE_MCMASA ) ) + { + if ( !hDiracDecBin->renderStereoOutputInsteadOfBinaural ) + { + idx = min( bin, MASA_NUM_DEFINED_SUR_SPR_COH_ENE_BINS - 1 ); + /* Apply target spectrum that emphasizes low frequencies when the sound is surround coherent */ + diffEne *= ( 1.0f - surCoh ) + surCoh * surCohEne[idx]; + } + } + hDiracDecBin->ChEneOut[0][bin] += diffEne; /* Diff ene part*/ + hDiracDecBin->ChEneOut[1][bin] += diffEne; + + if ( hDiracDecBin->renderStereoOutputInsteadOfBinaural ) + { + /* When rendering stereo, ambience (except for surround coherent sound) has zero ICC. */ + hDiracDecBin->ChCrossReOut[bin] += surCoh * diffEne; + } + else /* When rendering binaural, ambience has frequency dependent ICC. */ + { + if ( ivas_format == SBA_FORMAT && bin < BINAURAL_COHERENCE_DIFFERENCE_BINS ) + { + float diffuseFieldCoherence; + diffuseFieldCoherence = hDiracDecBin->hDiffuseDist->diffuseRatioX[bin] * hDiracDecBin->diffuseFieldCoherenceX[bin] + hDiracDecBin->hDiffuseDist->diffuseRatioY[bin] * hDiracDecBin->diffuseFieldCoherenceY[bin] + hDiracDecBin->hDiffuseDist->diffuseRatioZ[bin] * hDiracDecBin->diffuseFieldCoherenceZ[bin]; + hDiracDecBin->ChCrossReOut[bin] += ( ( 1.0f - surCoh ) * diffuseFieldCoherence + surCoh ) * diffEne; + } + else + { + hDiracDecBin->ChCrossReOut[bin] += ( ( 1.0f - surCoh ) * hDiracDecBin->diffuseFieldCoherence[bin] + surCoh ) * diffEne; + } + } + + /* Store parameters for formulating average diffuseness over frame */ + hDiracDecBin->frameMeanDiffuseness[bin] += diffEne; + frameMeanDiffusenessEneWeight[bin] += meanEnePerCh; + } + + /* Formulate average diffuseness over frame */ + for ( bin = 0; bin < nBins; bin++ ) + { + hDiracDecBin->frameMeanDiffuseness[bin] /= fmaxf( 1e-12f, frameMeanDiffusenessEneWeight[bin] ); + } + + /* Temporal IIR-type smoothing of covariance matrices. Also apply encoding quality based smoothing factor. */ + if ( ivas_format == MASA_FORMAT && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) + { + IIReneLimiterFactor = 16.0f + ( 1.0f - qualityBasedSmFactor ); + } + else + { + IIReneLimiterFactor = 8.0f + ( 1.0f - qualityBasedSmFactor ); } for ( bin = 0; bin < nBins; bin++ ) { @@ -1243,7 +1900,7 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric return; } - +#endif static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, @@ -1436,14 +2093,27 @@ static void ivas_dirac_dec_binaural_process_output( const int16_t max_band_decorr, const int16_t numInChannels, const int16_t processReverb, - const int16_t subframe ) + const int16_t subframe +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + , + float outRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float outIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float reverbRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float reverbIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float decorrRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float decorrIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const uint8_t recompute +#endif + ) { int16_t slot, bin, chA, chB; int16_t nBins; float outSlotRe[CLDFB_NO_CHANNELS_MAX], outSlotIm[CLDFB_NO_CHANNELS_MAX]; float decSlotRe[BINAURAL_CHANNELS][CLDFB_NO_CHANNELS_MAX], decSlotIm[BINAURAL_CHANNELS][CLDFB_NO_CHANNELS_MAX]; +#ifndef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN float reverbRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; +#endif float interpVal; float *decSlotRePointer; float *decSlotImPointer; @@ -1457,7 +2127,14 @@ static void ivas_dirac_dec_binaural_process_output( if ( processReverb ) { /* Process second / room effect part of binaural output when needed */ - ivas_binaural_reverb_processSubframe( hDiracDecBin->hReverb, numInChannels, nSlots, inRe, inIm, reverbRe, reverbIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( recompute == 1 ) + { +#endif + ivas_binaural_reverb_processSubframe( hDiracDecBin->hReverb, numInChannels, nSlots, inRe, inIm, reverbRe, reverbIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + } +#endif } interpVal = 0.0f; @@ -1466,7 +2143,27 @@ static void ivas_dirac_dec_binaural_process_output( interpVal += 1.0f / (float) nSlots; if ( !hDiracDecBin->useTdDecorr && max_band_decorr > 0 ) { - ivas_dirac_dec_decorrelate_slot( hDiracDecBin, nBins, slot, inRe, inIm, decSlotRe, decSlotIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( recompute == 1 ) + { +#endif + ivas_dirac_dec_decorrelate_slot( hDiracDecBin, nBins, slot, inRe, inIm, decSlotRe, decSlotIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( chA = 0; chA < BINAURAL_CHANNELS; chA++ ) + { + mvr2r(decSlotRe[chA], decorrRe[chA][slot], CLDFB_NO_CHANNELS_MAX); + mvr2r(decSlotIm[chA], decorrIm[chA][slot], CLDFB_NO_CHANNELS_MAX); + } + } + else + { + for ( chA = 0; chA < BINAURAL_CHANNELS; chA++ ) + { + mvr2r(decorrRe[chA][slot], decSlotRe[chA], CLDFB_NO_CHANNELS_MAX); + mvr2r(decorrIm[chA][slot], decSlotIm[chA], CLDFB_NO_CHANNELS_MAX); + } + } +#endif } for ( chA = 0; chA < BINAURAL_CHANNELS; chA++ ) @@ -1531,8 +2228,22 @@ static void ivas_dirac_dec_binaural_process_output( outSlotRePr = &( outSlotRe[0] ); outSlotImPr = &( outSlotIm[0] ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( outRe != NULL && outIm != NULL ) + { + /* provide the data outside in CLDFB domain => mainly for split rendering */ + mvr2r(outSlotRePr, outRe[chA][slot], CLDFB_NO_CHANNELS_MAX); + mvr2r(outSlotImPr, outIm[chA][slot], CLDFB_NO_CHANNELS_MAX); + } + if ( recompute == 1 ) + { + /* Inverse filter bank */ + cldfbSynthesis( &outSlotRePr, &outSlotImPr, &( output_f[chA][nBins * slot + offsetSamples] ), nBins, cldfbSynDec[chA] ); + } +#else /* Inverse filter bank */ cldfbSynthesis( &outSlotRePr, &outSlotImPr, &( output_f[chA][nBins * slot + offsetSamples] ), nBins, cldfbSynDec[chA] ); +#endif } } diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index c8c4cabc3b..473ea1c24e 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -3244,7 +3244,14 @@ static DecoderDummy *initDecoderDummy( decDummy->hoa_dec_mtx = NULL; decDummy->hVBAPdata = NULL; // note: not used at the moment decDummy->hMasa = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) + { + decDummy->hDiracDecBin[i] = NULL; + } +#else decDummy->hDiracDecBin = NULL; +#endif decDummy->hDirACRend = NULL; decDummy->hSpatParamRendCom = NULL; decDummy->hQMetaData = NULL; @@ -3370,7 +3377,9 @@ static ivas_error setRendInputActiveMasa( { inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, numInChannels, outConfig, 0 ); inputMasa->metadataHasBeenFed = false; - +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + inputMasa->decDummy->hRenderConfig = hRendCfg; +#endif if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { return error; @@ -3455,7 +3464,11 @@ static void freeDecoderDummy( free( pDecDummy->hHrtfParambin ); /* Parametric binaural renderer handle */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( pDecDummy->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &pDecDummy->hDiracDecBin ); +#endif /* TC buffer */ ivas_jbm_dec_tc_buffer_close( &pDecDummy->hTcBuffer ); @@ -8101,6 +8114,13 @@ static ivas_error renderInputMasa( //#endif // renderMasaToBinauralRoom( masaInput, outConfig, outAudio ); // break; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + case IVAS_REND_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: + case IVAS_REND_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + /* TODO: implement */ + renderMasaToBinaural( masaInput, outAudio ); + break; +#endif default: return IVAS_ERR_INVALID_OUTPUT_FORMAT; } diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 76f114b689..3d93ee980a 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -37,8 +37,8 @@ from tests.split_rendering.utils import * @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajectory): - if in_fmt != "HOA3": - pytest.xfail("Split Rendering currently only supported with HOA3") + if in_fmt in ("MONO", "STEREO",): + pytest.xfail("Split Rendering currently not supported with MONO or STEREO") post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") @@ -56,8 +56,8 @@ def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajector @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): - if in_fmt != "HOA3": - pytest.xfail("Split Rendering currently only supported with HOA3") + if in_fmt in ("MONO", "STEREO",): + pytest.xfail("Split Rendering currently not supported with MONO or STEREO") post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") -- GitLab From c4a81ef0fa284dc56991e40d1061e61ec1370b4e Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Mon, 24 Jul 2023 09:38:45 +0200 Subject: [PATCH 2/5] remove switch SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT --- lib_com/options.h | 1 - lib_rend/ivas_dirac_dec_binaural_functions.c | 429 ------------------- 2 files changed, 430 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index e8a5a8b4d1..1566e2d7b9 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -212,7 +212,6 @@ #define FIX_615_UBSAN_SPAR_TO_DIRAC /*Dlb : Fix for UBSAN issue 615*/ #define SPLIT_REND_WITH_HEAD_ROT_PARAMBIN /* Nokia: Issue 623: Split rendering support for parambin renderer */ -#define SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT /* Nokia: Issue 623: Split rendering support for parambin renderer, computational optimization */ /* ################## End BE DEVELOPMENT switches ######################### */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 258a8bf7d8..0d659479f6 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -95,12 +95,8 @@ static void ivas_dirac_dec_binaural_internal( Decoder_Struct *st_ivas, COMBINED_ static void ivas_dirac_dec_decorrelate_slot( DIRAC_DEC_BIN_HANDLE hDiracDecBin, const int16_t num_freq_bands, const int16_t slot, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float decRe[][CLDFB_NO_CHANNELS_MAX], float decIm[][CLDFB_NO_CHANNELS_MAX] ); -#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t subframe, const int16_t isHeadtracked, float *subFrameTotalEne, float *IIReneLimiter ); static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked, const float *subFrameTotalEne, const float *IIReneLimiter ); -#else -static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked ); -#endif static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t isHeadtracked ); @@ -668,10 +664,8 @@ static void ivas_dirac_dec_binaural_internal( float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float decorrRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float decorrIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; -#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; float IIReneLimiter[CLDFB_NO_CHANNELS_MAX]; -#endif hDiracDecBin = st_ivas->hDiracDecBin[0]; #else @@ -857,17 +851,12 @@ static void ivas_dirac_dec_binaural_internal( } #ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN -#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT ivas_dirac_dec_binaural_formulate_input_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, subFrameTotalEne, IIReneLimiter ); ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, subFrameTotalEne, IIReneLimiter ); -#else - ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat, subframe, - hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); -#endif #endif ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); @@ -927,7 +916,6 @@ static void ivas_dirac_dec_binaural_internal( hDiracDecBin->hDiffuseDist = &diffuseDistData; } -#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT /* re-use input covariance for the side renderings */ for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { @@ -940,10 +928,6 @@ static void ivas_dirac_dec_binaural_internal( hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, subFrameTotalEne, IIReneLimiter ); -#else - ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat_local, subframe, - hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); -#endif ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); @@ -1032,7 +1016,6 @@ static void ivas_dirac_dec_decorrelate_slot( return; } -#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN_OPT static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, @@ -1489,418 +1472,6 @@ static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( return; } -#else -static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( - DIRAC_DEC_BIN_HANDLE hDiracDecBin, - SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, - PARAMBIN_REND_CONFIG_HANDLE hConfig, - float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], - float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], - float Rmat[3][3], - const int16_t subframe, - const int16_t isHeadtracked ) -{ - int16_t ch, slot, bin; - int16_t separateCenterChannelRendering; - int16_t nBins, idx; - float frameMeanDiffusenessEneWeight[CLDFB_NO_CHANNELS_MAX]; - float IIReneLimiterFactor; - float qualityBasedSmFactor; - float lowBitRateEQ[CLDFB_NO_CHANNELS_MAX]; - uint8_t applyLowBitRateEQ; - int16_t dirac_read_idx; - float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; - PARAMBIN_HRTF_GAIN_CACHE gainCache[MAX_GAIN_CACHE_SIZE]; - IVAS_FORMAT ivas_format; - MC_MODE mc_mode; - int32_t ivas_total_brate; - int16_t nchan_transport; - - separateCenterChannelRendering = hConfig->separateCenterChannelRendering; - ivas_format = hConfig->ivas_format; - mc_mode = hConfig->mc_mode; - ivas_total_brate = hConfig->ivas_total_brate; - nchan_transport = hConfig->nchan_transport; - qualityBasedSmFactor = hConfig->qualityBasedSmFactor; - qualityBasedSmFactor *= qualityBasedSmFactor; - nBins = hSpatParamRendCom->num_freq_bands; /* Actually bins */ - - set_zero( hDiracDecBin->ChCrossRe, nBins ); - set_zero( hDiracDecBin->ChCrossIm, nBins ); - set_zero( hDiracDecBin->ChCrossReOut, nBins ); - set_zero( hDiracDecBin->ChCrossImOut, nBins ); - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - set_zero( hDiracDecBin->ChEne[ch], nBins ); - set_zero( hDiracDecBin->ChEneOut[ch], nBins ); - } - set_zero( hDiracDecBin->frameMeanDiffuseness, nBins ); - - set_zero( frameMeanDiffusenessEneWeight, CLDFB_NO_CHANNELS_MAX ); - - for ( idx = 0; idx < MAX_GAIN_CACHE_SIZE; idx++ ) - { - gainCache[idx].azi = -1000; /* Use -1000 as value for uninitialized cache. */ - } - - /* Determine EQ for low bit rates (13.2 and 16.4 kbps) */ - applyLowBitRateEQ = 0; - if ( ( ivas_format == MASA_FORMAT || ivas_format == MC_FORMAT ) && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) - { - applyLowBitRateEQ = 1; - if ( ivas_total_brate == IVAS_16k4 ) - { - for ( bin = 0; bin < LOW_BIT_RATE_BINAURAL_EQ_BINS; bin++ ) - { - lowBitRateEQ[bin + LOW_BIT_RATE_BINAURAL_EQ_OFFSET] = lowBitRateBinauralEQ[bin] * 0.5f + 0.5f; - } - } - else - { - for ( bin = 0; bin < LOW_BIT_RATE_BINAURAL_EQ_BINS; bin++ ) - { - lowBitRateEQ[bin + LOW_BIT_RATE_BINAURAL_EQ_OFFSET] = lowBitRateBinauralEQ[bin]; - } - } - } - - /* Formulate input and target covariance matrices for this subframe */ - set_zero( subFrameTotalEne, CLDFB_NO_CHANNELS_MAX ); - dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; - - /* Calculate input covariance matrix */ - for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) - { - for ( bin = 0; bin < nBins; bin++ ) - { - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - float instEne; - - instEne = ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ); - instEne += ( inIm[ch][slot][bin] * inIm[ch][slot][bin] ); - hDiracDecBin->ChEne[ch][bin] += instEne; - subFrameTotalEne[bin] += instEne; - } - hDiracDecBin->ChCrossRe[bin] += inRe[0][slot][bin] * inRe[1][slot][bin]; - hDiracDecBin->ChCrossRe[bin] += inIm[0][slot][bin] * inIm[1][slot][bin]; - hDiracDecBin->ChCrossIm[bin] += inRe[0][slot][bin] * inIm[1][slot][bin]; - hDiracDecBin->ChCrossIm[bin] -= inIm[0][slot][bin] * inRe[1][slot][bin]; - } - } - - /* Apply EQ at low bit rates */ - if ( applyLowBitRateEQ ) - { - int16_t lastEqBin = LOW_BIT_RATE_BINAURAL_EQ_OFFSET + LOW_BIT_RATE_BINAURAL_EQ_BINS - 1; - - for ( bin = LOW_BIT_RATE_BINAURAL_EQ_OFFSET; bin < lastEqBin; bin++ ) - { - subFrameTotalEne[bin] *= lowBitRateEQ[bin]; - } - for ( ; bin < nBins; bin++ ) - { - subFrameTotalEne[bin] *= lowBitRateEQ[lastEqBin]; - } - } - - if ( ivas_format == SBA_FORMAT && nchan_transport == 2 ) - { - float tempRe, tempIm; - - set_zero( subFrameTotalEne, CLDFB_NO_CHANNELS_MAX ); - - for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) - { - for ( bin = 0; bin < nBins; bin++ ) - { - tempRe = inRe[0][slot][bin] + inRe[1][slot][bin]; - tempIm = inIm[0][slot][bin] + inIm[1][slot][bin]; - subFrameTotalEne[bin] += tempRe * tempRe + tempIm * tempIm; - } - } - } - - /* Determine target covariance matrix containing target binaural properties */ - for ( bin = 0; bin < nBins; bin++ ) - { - float diffuseness = 1.0f; /* ratio1 and ratio2 are subtracted from diffuseness further below */ - float surCoh = 0.0f, spreadCoh = 0.0f; /* Default values if spreadSurroundCoherenceApplied == false */ - float diffEne, dirEne, meanEnePerCh; - int16_t dirIndex; - - /* When BINAURAL_ROOM is not indicated, hBinaural->earlyPartEneCorrection[bin] values are all 1.0f. - * When BINAURAL_ROOM is indicated, the binaural audio output is based on combined use of the - * HRTF data set and a BRIR-based data set. The HRTF data set is spectrally corrected to match - * the early spectrum of the BRIR data, using the spectral correction data in - * hBinaural->earlyPartEneCorrection[bin], based on the BRIR set. */ - meanEnePerCh = hDiracDecBin->earlyPartEneCorrection[bin] * subFrameTotalEne[bin] / 2.0f; - - /* Determine direct part target covariance matrix (for 1 or 2 directions) */ - for ( dirIndex = 0; dirIndex < hSpatParamRendCom->numSimultaneousDirections; dirIndex++ ) - { - int16_t aziDeg, eleDeg; - float lRealp, lImagp, rRealp, rImagp; - float lRealpTmp, lImagpTmp, rRealpTmp, rImagpTmp; - float hrtfEne[BINAURAL_CHANNELS], hrtfCrossRe, hrtfCrossIm, ratio; - - if ( dirIndex == 0 ) /* For first of the two simultaneous directions */ - { - aziDeg = hSpatParamRendCom->azimuth[dirac_read_idx][bin]; - eleDeg = hSpatParamRendCom->elevation[dirac_read_idx][bin]; - ratio = hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin]; - spreadCoh = hSpatParamRendCom->spreadCoherence[dirac_read_idx][bin]; - } - else /* For second of the two simultaneous directions */ - { - if ( ( ratio = hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin] ) < 0.001 ) - { - /* This touches only MASA path where second direction always has smaller ratio and - * for non-2dir it is zero. As the whole direction contribution is multiplied with - * the ratio, a very small ratio does not contribute any energy to output. Thus, - * it is better to save complexity. */ - continue; - } - aziDeg = hSpatParamRendCom->azimuth2[dirac_read_idx][bin]; - eleDeg = hSpatParamRendCom->elevation2[dirac_read_idx][bin]; - spreadCoh = hSpatParamRendCom->spreadCoherence2[dirac_read_idx][bin]; - } - diffuseness -= ratio; /* diffuseness = 1 - ratio1 - ratio2 */ - - if ( separateCenterChannelRendering ) - { - /* In masa + mono rendering mode, the center directions originate from phantom sources, so the - * spread coherence is increased */ - float aziRad, eleRad, doaVectorX, spatialAngleDeg, altSpreadCoh; - - aziRad = (float) aziDeg * PI_OVER_180; - eleRad = (float) eleDeg * PI_OVER_180; - doaVectorX = cosf( aziRad ) * cosf( eleRad ); - spatialAngleDeg = acosf( doaVectorX ) * _180_OVER_PI; - altSpreadCoh = 1.0f - ( spatialAngleDeg / 30.0f ); - spreadCoh = max( spreadCoh, altSpreadCoh ); - } - - getDirectPartGains( bin, aziDeg, eleDeg, &lRealp, &lImagp, &rRealp, &rImagp, hDiracDecBin->renderStereoOutputInsteadOfBinaural, Rmat, &gainCache[( dirIndex * 3 )], isHeadtracked ); - - if ( hDiracDecBin->renderStereoOutputInsteadOfBinaural ) - { - /* Synthesizing spread coherence is not needed for stereo loudspeaker output, - * as directional sound is reproduced with two loudspeakers in any case */ - spreadCoh = 0.0f; - } - - if ( spreadCoh > 0.0f ) - { - float centerMul, sidesMul; - float hrtfEneCenter, hrtfEneSides, hrtfEneRealized, eneCorrectionFactor; - float w1, w2, w3, eq; - - hrtfEneCenter = ( lRealp * lRealp ) + ( lImagp * lImagp ) + ( rRealp * rRealp ) + ( rImagp * rImagp ); - - /* Spread coherence is synthesized as coherent sources at 30 degree horizontal spacing. - * The following formulas determine the gains for these sources. - * spreadCoh = 0: Only panning - * spreadCoh = 0.5: Three sources coherent panning (e.g. 30 0 -30 deg azi) - * spreadCoh = 1.0: Two sources coherent panning with gap (as above, but center is silent) */ - if ( spreadCoh < 0.5f ) - { - /* 0.0f < spreadCoh < 0.5f */ - sidesMul = 0.5774f * spreadCoh * 2.0f; /* sqrt(1/3) = 0.5774f */ - centerMul = 1.0f - ( spreadCoh * 2.0f ) + sidesMul; - } - else - { - /* 0.5f <= spreadCoh < 1.0f */ - centerMul = 2.0f - ( 2.0f * spreadCoh ); - sidesMul = inv_sqrt( centerMul + 2.0f ); - centerMul *= sidesMul; - } - - /* Apply the gain for the center source of the three coherent sources */ - lRealp *= centerMul; - lImagp *= centerMul; - rRealp *= centerMul; - rImagp *= centerMul; - - /* Apply the gain for the left source of the three coherent sources */ - getDirectPartGains( bin, aziDeg + 30, eleDeg, &lRealpTmp, &lImagpTmp, &rRealpTmp, &rImagpTmp, hDiracDecBin->renderStereoOutputInsteadOfBinaural, Rmat, &gainCache[( dirIndex * 3 + 1 )], isHeadtracked ); - - hrtfEneSides = ( lRealpTmp * lRealpTmp ) + ( lImagpTmp * lImagpTmp ) + ( rRealpTmp * rRealpTmp ) + ( rImagpTmp * rImagpTmp ); - lRealp += sidesMul * lRealpTmp; - lImagp += sidesMul * lImagpTmp; - rRealp += sidesMul * rRealpTmp; - rImagp += sidesMul * rImagpTmp; - - /* Apply the gain for the right source of the three coherent sources. - * -30 degrees to 330 wrapping due to internal functions. */ - getDirectPartGains( bin, aziDeg + 330, eleDeg, &lRealpTmp, &lImagpTmp, &rRealpTmp, &rImagpTmp, hDiracDecBin->renderStereoOutputInsteadOfBinaural, Rmat, &gainCache[( dirIndex * 3 + 2 )], isHeadtracked ); - - hrtfEneSides += ( lRealpTmp * lRealpTmp ) + ( lImagpTmp * lImagpTmp ) + ( rRealpTmp * rRealpTmp ) + ( rImagpTmp * rImagpTmp ); - lRealp += sidesMul * lRealpTmp; - lImagp += sidesMul * lImagpTmp; - rRealp += sidesMul * rRealpTmp; - rImagp += sidesMul * rImagpTmp; - - /* Formulate an eneCorrectionFactor that compensates for the coherent summation of the HRTFs */ - hrtfEneRealized = ( lRealp * lRealp ) + ( lImagp * lImagp ) + ( rRealp * rRealp ) + ( rImagp * rImagp ); - eneCorrectionFactor = ( ( hrtfEneSides * sidesMul * sidesMul ) + - ( hrtfEneCenter * centerMul * centerMul ) ) / - max( 1e-12f, hrtfEneRealized ); - - /* Weighting factors to determine appropriate target spectrum for spread coherent sound */ - if ( spreadCoh < 0.5 ) - { - w1 = 1.0f - 2.0f * spreadCoh; - w2 = 2.0f * spreadCoh; - w3 = 0.0f; - } - else - { - w1 = 0.0f; - w2 = 2.0f - 2.0f * spreadCoh; - w3 = 2.0f * spreadCoh - 1.0f; - } - - if ( ( ivas_format == MC_FORMAT && mc_mode == MC_MODE_MCMASA ) ) - { - idx = min( bin, MASA_NUM_DEFINED_SUR_SPR_COH_ENE_BINS - 1 ); - - /* Apply the target spectrum to the eneCorrectionFactor */ - if ( separateCenterChannelRendering ) /* spreadCoh mostly originates from phantom sources in separate channel rendering mode */ - { - eneCorrectionFactor *= w1 * 1.0f + ( w2 + w3 ) * spreadCohEne1[idx]; - } - else - { - eneCorrectionFactor *= w1 * 1.0f + w2 * spreadCohEne05[idx] + w3 * spreadCohEne1[idx]; - } - } - - /* Equalize the spread coherent combined HRTFs */ - eq = min( 4.0f, sqrtf( eneCorrectionFactor ) ); - lRealp *= eq; - lImagp *= eq; - rRealp *= eq; - rImagp *= eq; - } - - hrtfEne[0] = ( lRealp * lRealp ) + ( lImagp * lImagp ); - hrtfEne[1] = ( rRealp * rRealp ) + ( rImagp * rImagp ); - hrtfCrossRe = ( lRealp * rRealp ) + ( lImagp * rImagp ); - hrtfCrossIm = ( -lImagp * rRealp ) + ( lRealp * rImagp ); - - /* Add direct part (1 or 2) covariance matrix */ - dirEne = ratio * meanEnePerCh; - hDiracDecBin->ChEneOut[0][bin] += dirEne * hrtfEne[0]; /* Dir ene part*/ - hDiracDecBin->ChEneOut[1][bin] += dirEne * hrtfEne[1]; - hDiracDecBin->ChCrossReOut[bin] += dirEne * hrtfCrossRe; /* Dir cross re */ - hDiracDecBin->ChCrossImOut[bin] += dirEne * hrtfCrossIm; /* Dir cross im */ - } - - /* Add diffuse / ambient part covariance matrix */ - diffuseness = max( 0.0f, diffuseness ); - diffEne = diffuseness * meanEnePerCh; - surCoh = hSpatParamRendCom->surroundingCoherence[dirac_read_idx][bin]; - if ( ( ivas_format == MC_FORMAT && mc_mode == MC_MODE_MCMASA ) ) - { - if ( !hDiracDecBin->renderStereoOutputInsteadOfBinaural ) - { - idx = min( bin, MASA_NUM_DEFINED_SUR_SPR_COH_ENE_BINS - 1 ); - /* Apply target spectrum that emphasizes low frequencies when the sound is surround coherent */ - diffEne *= ( 1.0f - surCoh ) + surCoh * surCohEne[idx]; - } - } - hDiracDecBin->ChEneOut[0][bin] += diffEne; /* Diff ene part*/ - hDiracDecBin->ChEneOut[1][bin] += diffEne; - - if ( hDiracDecBin->renderStereoOutputInsteadOfBinaural ) - { - /* When rendering stereo, ambience (except for surround coherent sound) has zero ICC. */ - hDiracDecBin->ChCrossReOut[bin] += surCoh * diffEne; - } - else /* When rendering binaural, ambience has frequency dependent ICC. */ - { - if ( ivas_format == SBA_FORMAT && bin < BINAURAL_COHERENCE_DIFFERENCE_BINS ) - { - float diffuseFieldCoherence; - diffuseFieldCoherence = hDiracDecBin->hDiffuseDist->diffuseRatioX[bin] * hDiracDecBin->diffuseFieldCoherenceX[bin] + hDiracDecBin->hDiffuseDist->diffuseRatioY[bin] * hDiracDecBin->diffuseFieldCoherenceY[bin] + hDiracDecBin->hDiffuseDist->diffuseRatioZ[bin] * hDiracDecBin->diffuseFieldCoherenceZ[bin]; - hDiracDecBin->ChCrossReOut[bin] += ( ( 1.0f - surCoh ) * diffuseFieldCoherence + surCoh ) * diffEne; - } - else - { - hDiracDecBin->ChCrossReOut[bin] += ( ( 1.0f - surCoh ) * hDiracDecBin->diffuseFieldCoherence[bin] + surCoh ) * diffEne; - } - } - - /* Store parameters for formulating average diffuseness over frame */ - hDiracDecBin->frameMeanDiffuseness[bin] += diffEne; - frameMeanDiffusenessEneWeight[bin] += meanEnePerCh; - } - - /* Formulate average diffuseness over frame */ - for ( bin = 0; bin < nBins; bin++ ) - { - hDiracDecBin->frameMeanDiffuseness[bin] /= fmaxf( 1e-12f, frameMeanDiffusenessEneWeight[bin] ); - } - - /* Temporal IIR-type smoothing of covariance matrices. Also apply encoding quality based smoothing factor. */ - if ( ivas_format == MASA_FORMAT && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) - { - IIReneLimiterFactor = 16.0f + ( 1.0f - qualityBasedSmFactor ); - } - else - { - IIReneLimiterFactor = 8.0f + ( 1.0f - qualityBasedSmFactor ); - } - for ( bin = 0; bin < nBins; bin++ ) - { - float eneRatio, IIReneLimiter; - - /* Temporally smooth cov mtx estimates for resulting mixing matrix stability. The design principle is that - * the energy history (IIR) must not be more than double of the current frame energy. This provides more - * robust performance at energy offsets when compared to typical IIR averaging. */ - eneRatio = ( hDiracDecBin->ChEne[0][bin] + hDiracDecBin->ChEne[1][bin] ) / fmaxf( 1e-12f, ( hDiracDecBin->ChEnePrev[0][bin] + hDiracDecBin->ChEnePrev[1][bin] ) ); - IIReneLimiter = fminf( 1.0f, eneRatio * IIReneLimiterFactor ); - - hDiracDecBin->ChCrossRe[bin] *= qualityBasedSmFactor; - hDiracDecBin->ChCrossIm[bin] *= qualityBasedSmFactor; - hDiracDecBin->ChCrossReOut[bin] *= qualityBasedSmFactor; - hDiracDecBin->ChCrossImOut[bin] *= qualityBasedSmFactor; - - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - hDiracDecBin->ChEne[ch][bin] *= qualityBasedSmFactor; - hDiracDecBin->ChEneOut[ch][bin] *= qualityBasedSmFactor; - } - - hDiracDecBin->ChCrossRe[bin] += IIReneLimiter * hDiracDecBin->ChCrossRePrev[bin]; - hDiracDecBin->ChCrossIm[bin] += IIReneLimiter * hDiracDecBin->ChCrossImPrev[bin]; - hDiracDecBin->ChCrossReOut[bin] += IIReneLimiter * hDiracDecBin->ChCrossReOutPrev[bin]; - hDiracDecBin->ChCrossImOut[bin] += IIReneLimiter * hDiracDecBin->ChCrossImOutPrev[bin]; - - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - hDiracDecBin->ChEne[ch][bin] += IIReneLimiter * hDiracDecBin->ChEnePrev[ch][bin]; - hDiracDecBin->ChEneOut[ch][bin] += IIReneLimiter * hDiracDecBin->ChEneOutPrev[ch][bin]; - } - - /* Store energy values and coefficients for next round */ - hDiracDecBin->ChCrossRePrev[bin] = hDiracDecBin->ChCrossRe[bin]; - hDiracDecBin->ChCrossImPrev[bin] = hDiracDecBin->ChCrossIm[bin]; - hDiracDecBin->ChCrossReOutPrev[bin] = hDiracDecBin->ChCrossReOut[bin]; - hDiracDecBin->ChCrossImOutPrev[bin] = hDiracDecBin->ChCrossImOut[bin]; - - for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) - { - hDiracDecBin->ChEnePrev[ch][bin] = hDiracDecBin->ChEne[ch][bin]; - hDiracDecBin->ChEneOutPrev[ch][bin] = hDiracDecBin->ChEneOut[ch][bin]; - } - } - - return; -} -#endif static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, -- GitLab From 63d92d3df743fadd67dacf00f271d830b6466cc5 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Mon, 24 Jul 2023 10:11:19 +0200 Subject: [PATCH 3/5] remove unused parameter --- lib_rend/ivas_dirac_dec_binaural_functions.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 0d659479f6..e77fabf73c 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -95,7 +95,7 @@ static void ivas_dirac_dec_binaural_internal( Decoder_Struct *st_ivas, COMBINED_ static void ivas_dirac_dec_decorrelate_slot( DIRAC_DEC_BIN_HANDLE hDiracDecBin, const int16_t num_freq_bands, const int16_t slot, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float decRe[][CLDFB_NO_CHANNELS_MAX], float decIm[][CLDFB_NO_CHANNELS_MAX] ); -static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t subframe, const int16_t isHeadtracked, float *subFrameTotalEne, float *IIReneLimiter ); +static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t subframe, float *subFrameTotalEne, float *IIReneLimiter ); static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked, const float *subFrameTotalEne, const float *IIReneLimiter ); static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t isHeadtracked ); @@ -852,7 +852,6 @@ static void ivas_dirac_dec_binaural_internal( #ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN ivas_dirac_dec_binaural_formulate_input_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe, - hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, subFrameTotalEne, IIReneLimiter ); ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, @@ -1023,7 +1022,6 @@ static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t subframe, - const int16_t isHeadtracked, float *subFrameTotalEne, float *IIReneLimiter ) { -- GitLab From 3381064cace5933c4058dcaf5481c369f7c492d2 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Mon, 24 Jul 2023 10:33:04 +0200 Subject: [PATCH 4/5] fix memory leak --- lib_rend/ivas_dirac_dec_binaural_functions.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index e77fabf73c..a408af8d4f 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -359,6 +359,11 @@ void ivas_dirac_dec_close_binaural_data( ivas_td_decorr_dec_close( &( hBinaural[pos_idx]->hTdDecorr ) ); + if ( hBinaural[pos_idx]->h_freq_domain_decorr_ap_params != NULL ) + { + ivas_dirac_dec_decorr_close( &(hBinaural[pos_idx]->h_freq_domain_decorr_ap_params), &(hBinaural[pos_idx]->h_freq_domain_decorr_ap_state) ); + } + free( hBinaural[pos_idx] ); hBinaural[pos_idx] = NULL; } -- GitLab From 38e787590386f0de9d04ca6dedc75a10c32a1ec2 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Mon, 24 Jul 2023 10:39:03 +0200 Subject: [PATCH 5/5] fix double-free in renderer --- lib_rend/lib_rend.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 473ea1c24e..c60a2cdfe6 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -3377,9 +3377,7 @@ static ivas_error setRendInputActiveMasa( { inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, numInChannels, outConfig, 0 ); inputMasa->metadataHasBeenFed = false; -#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN - inputMasa->decDummy->hRenderConfig = hRendCfg; -#endif + if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { return error; -- GitLab