diff --git a/lib_com/options.h b/lib_com/options.h old mode 100755 new mode 100644 index c2bc0b866b05b09d53460339bca577ae01a9052d..004d198b9ebef35a8598c4ce771cae268ed2dee7 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -207,6 +207,7 @@ #define FIX_622_SILENCE_USAN_WARNING /* FhG: silenceusan warning in ifft code */ #define FIX_615_UBSAN_SPAR_TO_DIRAC /*Dlb : Fix for UBSAN issue 615*/ +#define SPLIT_REND_WITH_HEAD_ROT_PARAMBIN /* Nokia: Issue 623: Split rendering support for parambin renderer */ #define FIX_626_VARIABLE_TYPE_MDCT_CONC /* FhG: trivial fix to fix USAN error */ #define FIX_616_DIV_ZERO_MCT /*FhG : Fix UBSAN division by zero error of issue 616*/ diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index 5ca878a00ec58382864b493b2665782dbb4f20a9..e945f045256471f29963e46b680a0d6b839ca1c0 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -124,10 +124,20 @@ ivas_error ivas_dec( if ( ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN assert( ( st_ivas->ivas_format == SBA_FORMAT || + st_ivas->ivas_format == MASA_FORMAT || + st_ivas->ivas_format == ISM_FORMAT || + st_ivas->ivas_format == MC_FORMAT ) && + ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA, MASA, discrete and parametric ISM, or MC formats and 48 kHz sampling rate only" ); +#else + assert( ( st_ivas->ivas_format == SBA_FORMAT || + st_ivas->ivas_format == MASA_FORMAT || + st_ivas->ivas_format == ISM_FORMAT || ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode != ISM_MODE_PARAM ) || ( st_ivas->ivas_format == MC_FORMAT && st_ivas->mc_mode != MC_MODE_MCMASA ) ) && ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA, discrete ISM, or MCT-MC formats and 48 kHz sampling rate only" ); +#endif ivas_set_split_rend_setup( &st_ivas->splitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, hSplitRendBits ); } #endif @@ -845,6 +855,11 @@ ivas_error ivas_dec( IVAS_DEC_SPLIT_REND_WRAPPER *hSplitBinRend; int16_t max_band; int16_t pcm_out; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t td_input; + + td_input = st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC; +#endif hSplitBinRend = &st_ivas->splitBinRend; max_band = (int16_t) ( ( BINAURAL_MAXBANDS * output_Fs ) / 48000 ); pcm_out = ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0; @@ -857,7 +872,11 @@ ivas_error ivas_dec( hSplitBinRend->hMultiBinCldfbData->Cldfb_RealBuffer_Binaural, hSplitBinRend->hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural, max_band, output, 1, +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + td_input, +#else st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV, +#endif pcm_out ); free( st_ivas->splitBinRend.hMultiBinCldfbData ); diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index 766f4ff08fe05ea389e53845c393a6b5fda5f734..3e9987e5daf515755373aeb0ace53582fde53808 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -826,7 +826,11 @@ ivas_error ivas_dirac_dec_config( if ( !need_parambin ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif } need_dirac_rend = 0; @@ -870,13 +874,48 @@ ivas_error ivas_dirac_dec_config( { if ( st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + /* populate pMultiBinPoseData with the additional poses determined from the DoF */ + ivas_renderSplitGetMultiBinPoseData( + &st_ivas->hRenderConfig->split_rend_config, + &st_ivas->splitBinRend.splitrend.multiBinPoseData, + st_ivas->hHeadTrackData->sr_pose_pred_axis ); + } +#endif if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) { return error; } } - +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + /* TODO: can we keep isRenderingInTd fixed? */ + ivas_split_rend_choose_default_codec( &st_ivas->hRenderConfig->split_rend_config.codec, + 0, /* isRenderingInTd */ + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); /* pcm_out */ + + error = ivas_split_renderer_open( &st_ivas->splitBinRend.splitrend, + &st_ivas->hRenderConfig->split_rend_config, + st_ivas->hDecoderConfig->output_Fs, + 1, /* is_cldfb_in */ + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); /* is_pcm_out */ + + if ( error != IVAS_ERR_OK ) + { + return error; + } + } +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] == NULL ) +#else if ( st_ivas->hDiracDecBin == NULL ) +#endif { if ( ( error = ivas_dirac_dec_init_binaural_data( st_ivas, st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) { @@ -886,24 +925,55 @@ ivas_error ivas_dirac_dec_config( else { /* This is required to keep BE in rate switching. This probably means that 1TC and 2TC MASA perform differently. */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params != NULL && !( st_ivas->ivas_format == MASA_FORMAT && st_ivas->nSCE > 0 ) ) +#else if ( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params != NULL && !( st_ivas->ivas_format == MASA_FORMAT && st_ivas->nSCE > 0 ) ) +#endif { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_decorr_close( &st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params, &st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_state ); +#else ivas_dirac_dec_decorr_close( &st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params, &st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_state ); +#endif } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin[0]->hTdDecorr ), &( st_ivas->hDiracDecBin[0]->useTdDecorr ) ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin->hTdDecorr ), &( st_ivas->hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) +#endif { return error; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + /* copy td-decorr flag to split renderer side rendereres */ + for ( int16_t pos_idx = 1; pos_idx < st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses; pos_idx++ ) + { + st_ivas->hDiracDecBin[pos_idx]->useTdDecorr = st_ivas->hDiracDecBin[0]->useTdDecorr; + } + + if ( !st_ivas->hDiracDecBin[0]->useTdDecorr ) +#else if ( !st_ivas->hDiracDecBin->useTdDecorr ) +#endif { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params == NULL ) +#else if ( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params == NULL ) +#endif { float frequency_axis[CLDFB_NO_CHANNELS_MAX]; ivas_dirac_dec_get_frequency_axis( frequency_axis, st_ivas->hDecoderConfig->output_Fs, st_ivas->hSpatParamRendCom->num_freq_bands ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( error = ivas_dirac_dec_decorr_open( &( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_params ), + &( st_ivas->hDiracDecBin[0]->h_freq_domain_decorr_ap_state ), +#else if ( ( error = ivas_dirac_dec_decorr_open( &( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_params ), &( st_ivas->hDiracDecBin->h_freq_domain_decorr_ap_state ), +#endif st_ivas->hSpatParamRendCom->num_freq_bands, BINAURAL_CHANNELS, BINAURAL_CHANNELS, @@ -917,7 +987,14 @@ ivas_error ivas_dirac_dec_config( } } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( int16_t pos_idx = 0; pos_idx < st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses; pos_idx++ ) + { + st_ivas->hDiracDecBin[pos_idx]->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); + } +#else st_ivas->hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); +#endif } } } diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 616069f25fbe1000500cd49ff6179189ab351bcd..2087e7c9e4ee8060ae8e46d3f80f7da6e4d6fcd0 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1319,6 +1319,16 @@ ivas_error ivas_init_decoder( /* ParamISM is handled separately from other common config */ else if ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM && ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + ivas_renderSplitGetMultiBinPoseData( + &st_ivas->hRenderConfig->split_rend_config, + &st_ivas->splitBinRend.splitrend.multiBinPoseData, + st_ivas->hHeadTrackData->sr_pose_pred_axis ); + } +#endif if ( st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) @@ -1331,6 +1341,26 @@ ivas_error ivas_init_decoder( { return error; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { +#ifdef FIX_SPLIT_REND_OPEN_ERROR_HANDLING + error = +#endif + ivas_split_renderer_open( &st_ivas->splitBinRend.splitrend, + &st_ivas->hRenderConfig->split_rend_config, + hDecoderConfig->output_Fs, + 1, ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); + +#ifdef FIX_SPLIT_REND_OPEN_ERROR_HANDLING + if ( error != IVAS_ERR_OK ) + { + return error; + } +#endif + } +#endif } else if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD ) { @@ -1820,7 +1850,14 @@ void ivas_initialize_handles_dec( st_ivas->splitBinRend.hSplitRendBits = NULL; ivas_init_split_rend_handles( &st_ivas->splitBinRend.splitrend ); #endif +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) + { + st_ivas->hDiracDecBin[i] = NULL; + } +#else st_ivas->hDiracDecBin = NULL; +#endif st_ivas->hDirACRend = NULL; st_ivas->hSpatParamRendCom = NULL; st_ivas->hLsSetUpConversion = NULL; @@ -1984,7 +2021,11 @@ void ivas_destroy_dec( #endif /* Parametric binaural renderer handle */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif /* Crend handle */ ivas_rend_closeCrend( &( st_ivas->hCrendWrapper ) @@ -2103,8 +2144,11 @@ void ivas_init_dec_get_num_cldfb_instances( { *numCldfbAnalyses = st_ivas->nchan_transport + 1; } - +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->useTdDecorr ) +#else if ( st_ivas->hDiracDecBin->useTdDecorr ) +#endif { *numCldfbAnalyses += 2; } diff --git a/lib_dec/ivas_ism_dec.c b/lib_dec/ivas_ism_dec.c index 63464def7f1251008048659757c2a6f21b6228e3..485511c23d2052dbbb26334a67a4649cd01d9337 100644 --- a/lib_dec/ivas_ism_dec.c +++ b/lib_dec/ivas_ism_dec.c @@ -174,8 +174,11 @@ static ivas_error ivas_ism_bitrate_switching( if ( st_ivas->hOutSetup.output_config == AUDIO_CONFIG_BINAURAL || st_ivas->hOutSetup.output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) { /* close the parametric binaural renderer */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); - +#endif /* Open the TD Binaural renderer */ if ( st_ivas->hHrtfTD == NULL || st_ivas->hBinRendererTd == NULL ) { @@ -213,7 +216,11 @@ static ivas_error ivas_ism_bitrate_switching( if ( st_ivas->hOutSetup.output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR ) { /* close the parametric binaural renderer */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif /* Open Crend Binaural renderer */ if ( ( error = ivas_rend_openCrend( &( st_ivas->hCrendWrapper ), st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hRenderConfig, st_ivas->hSetOfHRTF, st_ivas->hDecoderConfig->output_Fs diff --git a/lib_dec/ivas_ism_param_dec.c b/lib_dec/ivas_ism_param_dec.c index 8cea5347b448516a5968080d4d7cccd9d79274e0..e536d8f25bb0f65912a3a6264f0fc0a0b74267a6 100644 --- a/lib_dec/ivas_ism_param_dec.c +++ b/lib_dec/ivas_ism_param_dec.c @@ -386,7 +386,11 @@ static ivas_error ivas_param_ism_rendering_init( return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for interpolator\n" ) ); } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM) ) +#else if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) ) +#endif { /* computation of proto matrix */ ivas_ism_get_proto_matrix( hOutSetup, nchan_transport, hParamIsmRendering->proto_matrix ); @@ -540,8 +544,13 @@ ivas_error ivas_param_ism_dec_open( } } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || + output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) +#else if ( !( output_config == AUDIO_CONFIG_EXTERNAL || output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO ) ) +#endif { /* Initialize efap handle */ if ( ( error = efap_init_data( &( st_ivas->hEFAPdata ), hOutSetup.ls_azimuth, hOutSetup.ls_elevation, hOutSetup.nchan_out_woLFE, EFAP_MODE_EFAP ) ) != IVAS_ERR_OK ) @@ -559,7 +568,11 @@ ivas_error ivas_param_ism_dec_open( hSpatParamRendCom->dirac_read_idx = 0; hDirAC->spar_to_dirac_write_idx = 0; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) +#else if ( output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_ROOM_IR || output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) +#endif { if ( ( error = ivas_dirac_allocate_parameters( hSpatParamRendCom, 1 ) ) != IVAS_ERR_OK ) { diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index 4a6053ccd5b44aa3ccf191ecb847ed553d4eea41..ca48e4bdf2360c8b7f3d77e1ab1815decdbaaf04 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -971,6 +971,9 @@ ivas_error ivas_masa_dec_reconfigure( int32_t ivas_total_brate, last_ivas_total_brate; int16_t numCldfbAnalyses_old, numCldfbSyntheses_old; ivas_error error; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; +#endif error = IVAS_ERR_OK; @@ -982,7 +985,11 @@ ivas_error ivas_masa_dec_reconfigure( /* renderer might have changed, reselect */ ivas_renderer_select( st_ivas ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend == NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] == NULL ) ) +#else if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend == NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin == NULL ) ) +#endif { /* init a new DirAC dec */ if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_OPEN ) ) != IVAS_ERR_OK ) @@ -995,7 +1002,11 @@ ivas_error ivas_masa_dec_reconfigure( if ( st_ivas->hDirAC != NULL ) { /* close all unnecessary parametric decoding and rendering */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif ivas_dirac_rend_close( &( st_ivas->hDirACRend ) ); ivas_spat_hSpatParamRendCom_close( &( st_ivas->hSpatParamRendCom ) ); ivas_dirac_dec_close( &( st_ivas->hDirAC ) ); @@ -1019,7 +1030,11 @@ ivas_error ivas_masa_dec_reconfigure( sts[0]->bit_stream = bit_stream + num_bits; num_bits += (int16_t) ( st_ivas->hSCE[sce_id]->element_brate / FRAMES_PER_SEC ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] != NULL ) +#else if ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin != NULL ) +#endif { if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_RECONFIGURE ) ) != IVAS_ERR_OK ) { @@ -1046,7 +1061,11 @@ ivas_error ivas_masa_dec_reconfigure( { st_ivas->hCPE[cpe_id]->nchan_out = 1; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] != NULL ) ) +#else if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin != NULL ) ) +#endif { if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_RECONFIGURE ) ) != IVAS_ERR_OK ) { @@ -1058,7 +1077,11 @@ ivas_error ivas_masa_dec_reconfigure( { st_ivas->hCPE[cpe_id]->nchan_out = CPE_CHANNELS; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin[0] != NULL ) ) +#else if ( ( st_ivas->renderer_type == RENDERER_DIRAC && st_ivas->hDirACRend != NULL ) || ( ( st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) && st_ivas->hDiracDecBin != NULL ) ) +#endif { if ( ( error = ivas_dirac_dec_config( st_ivas, DIRAC_RECONFIGURE ) ) != IVAS_ERR_OK ) { @@ -1068,10 +1091,22 @@ ivas_error ivas_masa_dec_reconfigure( } } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) + { + if ( st_ivas->hDiracDecBin[pos_idx] != NULL ) + { +#else if ( st_ivas->hDiracDecBin != NULL ) { +#endif /* regularization factor is bitrate-dependent */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + st_ivas->hDiracDecBin[pos_idx]->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); + } +#else st_ivas->hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); +#endif } /*-----------------------------------------------------------------* @@ -1149,7 +1184,11 @@ void ivas_spar_param_to_masa_param_mapping( hDirAC = st_ivas->hDirAC; hSpatParamRendCom = st_ivas->hSpatParamRendCom; hSpatParamRendCom->numSimultaneousDirections = 1; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + hDiffuseDist = st_ivas->hDiracDecBin[0]->hDiffuseDist; /* TODO: this may need to use different pose indices */ +#else hDiffuseDist = st_ivas->hDiracDecBin->hDiffuseDist; +#endif nchan_transport = st_ivas->nchan_transport; band_grouping = hDirAC->band_grouping; hSpar = st_ivas->hSpar; diff --git a/lib_dec/ivas_mct_dec.c b/lib_dec/ivas_mct_dec.c index e99cf63fa64376f9ab08878e7d95380cc2f284d8..f31aff232745bc531890483d288e5888a51af428 100755 --- a/lib_dec/ivas_mct_dec.c +++ b/lib_dec/ivas_mct_dec.c @@ -1161,11 +1161,19 @@ static ivas_error ivas_mc_dec_reconfig( ivas_td_binaural_close( &st_ivas->hBinRendererTd ); } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] != NULL ) +#else if ( st_ivas->hDiracDecBin != NULL ) +#endif { if ( st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &st_ivas->hDiracDecBin ); +#endif } } diff --git a/lib_dec/ivas_sba_dec.c b/lib_dec/ivas_sba_dec.c index 763a1b222fde2f2c596e3f0239a9ea812788475c..5659a96b2422deeac53e84692b656b48db281fff 100755 --- a/lib_dec/ivas_sba_dec.c +++ b/lib_dec/ivas_sba_dec.c @@ -357,12 +357,21 @@ ivas_error ivas_sba_dec_reconfigure( * TD Decorrelator *-----------------------------------------------------------------*/ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] != NULL ) + { + if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin[0]->hTdDecorr ), &( st_ivas->hDiracDecBin[0]->useTdDecorr ) ) ) != IVAS_ERR_OK ) + { + return error; + } +#else if ( st_ivas->hDiracDecBin != NULL ) { if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( st_ivas->hDiracDecBin->hTdDecorr ), &( st_ivas->hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) { return error; } +#endif } /*-----------------------------------------------------------------* @@ -450,7 +459,11 @@ ivas_error ivas_sba_dec_digest_tc( ivas_spar_dec_digest_tc( st_ivas, st_ivas->nchan_transport, nCldfbSlots, nSamplesForRendering ); } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0] != NULL && ( st_ivas->hDiracDecBin[0]->useTdDecorr ) ) +#else if ( st_ivas->hDiracDecBin != NULL && ( st_ivas->hDiracDecBin->useTdDecorr ) ) +#endif { int16_t nSamplesLeftForTD, default_frame; float *decorr_signal[BINAURAL_CHANNELS]; @@ -468,9 +481,17 @@ ivas_error ivas_sba_dec_digest_tc( while ( nSamplesLeftForTD ) { int16_t nSamplesToDecorr = min( nSamplesLeftForTD, default_frame ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->hTdDecorr ) +#else if ( st_ivas->hDiracDecBin->hTdDecorr ) +#endif { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_td_decorr_process( st_ivas->hDiracDecBin[0]->hTdDecorr, p_tc, decorr_signal, nSamplesToDecorr ); +#else ivas_td_decorr_process( st_ivas->hDiracDecBin->hTdDecorr, p_tc, decorr_signal, nSamplesToDecorr ); +#endif } for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) { diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index 5c70409ac2892ff9da3cd11cb968662475c17784..003a45056cb0a11c720e227460d687a1c301d134 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -997,7 +997,11 @@ typedef struct Decoder_Struct BINAURAL_RENDERER_HANDLE hBinRenderer; /* fastconv binaural renderer handle */ BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd; /* Time domain binaural object renderer handle */ TDREND_HRFILT_FiltSet_t *hHrtfTD; /* pointer to HRTF data for TD renderer */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + DIRAC_DEC_BIN_HANDLE hDiracDecBin[MAX_HEAD_ROT_POSES]; /* parametric binaural renderer handle */ +#else DIRAC_DEC_BIN_HANDLE hDiracDecBin; /* parametric binaural renderer handle */ +#endif LSSETUP_CONVERSION_HANDLE hLsSetUpConversion; /* MC LS configuration convertion handle */ EFAP_HANDLE hEFAPdata; /* EFAP structure */ VBAP_HANDLE hVBAPdata; /* VBAP structure */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 8a3f33b80b1949416653dfb99e889414c3e22b6d..a408af8d4ffce9398d738d66c529215dd8d8c4b9 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -95,11 +95,22 @@ static void ivas_dirac_dec_binaural_internal( Decoder_Struct *st_ivas, COMBINED_ static void ivas_dirac_dec_decorrelate_slot( DIRAC_DEC_BIN_HANDLE hDiracDecBin, const int16_t num_freq_bands, const int16_t slot, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float decRe[][CLDFB_NO_CHANNELS_MAX], float decIm[][CLDFB_NO_CHANNELS_MAX] ); -static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked ); +static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t subframe, float *subFrameTotalEne, float *IIReneLimiter ); +static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked, const float *subFrameTotalEne, const float *IIReneLimiter ); static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t isHeadtracked ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN +static void ivas_dirac_dec_binaural_process_output( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, HANDLE_CLDFB_FILTER_BANK cldfbSynDec[MAX_OUTPUT_CHANNELS], + float *output_f[], float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const int16_t max_band_decorr, const int16_t numInChannels, const int16_t processReverb, const int16_t subframe, + float outRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float outIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float reverbRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float reverbIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float decorrRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float decorrIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const uint8_t recompute ); +#else static void ivas_dirac_dec_binaural_process_output( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, HANDLE_CLDFB_FILTER_BANK cldfbSynDec[MAX_OUTPUT_CHANNELS], float *output_f[], float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const int16_t numInChannels, const int16_t processReverb, const int16_t subframe ); +#endif static void adaptTransportSignalsHeadtracked( COMBINED_ORIENTATION_HANDLE hHeadTrackData, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], const int16_t nBins, const int16_t nSlots, float Rmat[3][3] ); @@ -138,143 +149,169 @@ ivas_error ivas_dirac_dec_init_binaural_data( float binCenterFreq, tmpFloat; ivas_error error; float frequency_axis[CLDFB_NO_CHANNELS_MAX]; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; + for ( pos_idx = 0; pos_idx < st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses; pos_idx++ ) + { + hDiracDecBin = st_ivas->hDiracDecBin[pos_idx]; +#else hDiracDecBin = st_ivas->hDiracDecBin; +#endif - if ( hDiracDecBin == NULL ) - { - if ( ( hDiracDecBin = (DIRAC_DEC_BIN_HANDLE) malloc( sizeof( DIRAC_DEC_BIN_DATA ) ) ) == NULL ) + if ( hDiracDecBin == NULL ) { - return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC binaural handle " ); + if ( ( hDiracDecBin = (DIRAC_DEC_BIN_HANDLE) malloc( sizeof( DIRAC_DEC_BIN_DATA ) ) ) == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC binaural handle " ); + } + + hDiracDecBin->hTdDecorr = NULL; + hDiracDecBin->hReverb = NULL; + hDiracDecBin->h_freq_domain_decorr_ap_params = NULL; + hDiracDecBin->h_freq_domain_decorr_ap_state = NULL; } - hDiracDecBin->hTdDecorr = NULL; - hDiracDecBin->hReverb = NULL; - hDiracDecBin->h_freq_domain_decorr_ap_params = NULL; - hDiracDecBin->h_freq_domain_decorr_ap_state = NULL; - } + output_Fs = st_ivas->hDecoderConfig->output_Fs; + nBins = st_ivas->hSpatParamRendCom->num_freq_bands; + renderer_type = st_ivas->renderer_type; + + for ( j = 0; j < BINAURAL_CHANNELS; j++ ) + { + for ( k = 0; k < BINAURAL_CHANNELS + 1; k++ ) + { + set_zero( hDiracDecBin->processMtxRe[j][k], nBins ); + set_zero( hDiracDecBin->processMtxIm[j][k], nBins ); + } - output_Fs = st_ivas->hDecoderConfig->output_Fs; - nBins = st_ivas->hSpatParamRendCom->num_freq_bands; - renderer_type = st_ivas->renderer_type; + for ( k = 0; k < BINAURAL_CHANNELS; k++ ) + { + set_zero( hDiracDecBin->processMtxDecRe[j][k], nBins ); + set_zero( hDiracDecBin->processMtxDecIm[j][k], nBins ); + } + set_zero( hDiracDecBin->ChEnePrev[j], nBins ); + set_zero( hDiracDecBin->ChEneOutPrev[j], nBins ); + } + set_zero( hDiracDecBin->ChCrossRePrev, nBins ); + set_zero( hDiracDecBin->ChCrossImPrev, nBins ); + set_zero( hDiracDecBin->ChCrossReOutPrev, nBins ); + set_zero( hDiracDecBin->ChCrossImOutPrev, nBins ); + hDiracDecBin->renderStereoOutputInsteadOfBinaural = 0; - for ( j = 0; j < BINAURAL_CHANNELS; j++ ) - { - for ( k = 0; k < BINAURAL_CHANNELS + 1; k++ ) + + for ( bin = 0; bin < nBins; bin++ ) { - set_zero( hDiracDecBin->processMtxRe[j][k], nBins ); - set_zero( hDiracDecBin->processMtxIm[j][k], nBins ); + binCenterFreq = ( (float) bin + CLDFB_HALF_BIN_FREQUENCY_OFFSET ) / (float) nBins * ( (float) output_Fs / 2.0f ); + /* These formulas and values are from Christian Borss's publication for binaural diffuse field coherence */ + tmpFloat = max( 0.0f, 1.0f - binCenterFreq / 2700.0f ); + hDiracDecBin->diffuseFieldCoherence[bin] = tmpFloat * sinf( binCenterFreq * EVS_PI / 550.0f ) / ( binCenterFreq * EVS_PI / 550.0f ); } - for ( k = 0; k < BINAURAL_CHANNELS; k++ ) + for ( bin = 0; bin < BINAURAL_COHERENCE_DIFFERENCE_BINS; bin++ ) { - set_zero( hDiracDecBin->processMtxDecRe[j][k], nBins ); - set_zero( hDiracDecBin->processMtxDecIm[j][k], nBins ); + hDiracDecBin->diffuseFieldCoherenceX[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceX[bin]; + hDiracDecBin->diffuseFieldCoherenceY[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceY[bin]; + hDiracDecBin->diffuseFieldCoherenceZ[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceZ[bin]; } - set_zero( hDiracDecBin->ChEnePrev[j], nBins ); - set_zero( hDiracDecBin->ChEneOutPrev[j], nBins ); - } - set_zero( hDiracDecBin->ChCrossRePrev, nBins ); - set_zero( hDiracDecBin->ChCrossImPrev, nBins ); - set_zero( hDiracDecBin->ChCrossReOutPrev, nBins ); - set_zero( hDiracDecBin->ChCrossImOutPrev, nBins ); - hDiracDecBin->renderStereoOutputInsteadOfBinaural = 0; + if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC ) /* Indication of binaural rendering without room effect */ + { + set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); + hDiracDecBin->hReverb = NULL; + } + else if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) /* Indication of binaural rendering with room effect */ + { + mvr2r( hHrtfParambin->parametricEarlyPartEneCorrection, hDiracDecBin->earlyPartEneCorrection, nBins ); - for ( bin = 0; bin < nBins; bin++ ) - { - binCenterFreq = ( (float) bin + CLDFB_HALF_BIN_FREQUENCY_OFFSET ) / (float) nBins * ( (float) output_Fs / 2.0f ); - /* These formulas and values are from Christian Borss's publication for binaural diffuse field coherence */ - tmpFloat = max( 0.0f, 1.0f - binCenterFreq / 2700.0f ); - hDiracDecBin->diffuseFieldCoherence[bin] = tmpFloat * sinf( binCenterFreq * EVS_PI / 550.0f ) / ( binCenterFreq * EVS_PI / 550.0f ); - } + /* reconfiguration needed when Reverb. parameters are changed -> close and open the handle again */ + if ( hDiracDecBin->hReverb != NULL && ( ( hDiracDecBin->hReverb->numBins != nBins ) || + ( hDiracDecBin->hReverb->blockSize != CLDFB_SLOTS_PER_SUBFRAME ) ) ) + { + ivas_binaural_reverb_close( &( hDiracDecBin->hReverb ) ); + } - for ( bin = 0; bin < BINAURAL_COHERENCE_DIFFERENCE_BINS; bin++ ) - { - hDiracDecBin->diffuseFieldCoherenceX[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceX[bin]; - hDiracDecBin->diffuseFieldCoherenceY[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceY[bin]; - hDiracDecBin->diffuseFieldCoherenceZ[bin] = hDiracDecBin->diffuseFieldCoherence[bin] + diffuseFieldCoherenceDifferenceZ[bin]; - } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( hDiracDecBin->hReverb == NULL && pos_idx == 0 ) /* open reverb only for the main direction */ +#else + if ( hDiracDecBin->hReverb == NULL ) +#endif + { + if ( ( error = ivas_binaural_reverb_open( &hDiracDecBin->hReverb, + nBins, + CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, NULL, + st_ivas->hIntSetup.output_config, + output_Fs, + RENDERER_BINAURAL_PARAMETRIC_ROOM, + st_ivas->hHrtfFastConv, + st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) + { + return error; + } + } + } + else if ( renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); + hDiracDecBin->hReverb = NULL; + hDiracDecBin->renderStereoOutputInsteadOfBinaural = 1; + } + else /* Not valid renderer type for this renderer */ + { + assert( false ); + } - if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC ) /* Indication of binaural rendering without room effect */ - { - set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); - hDiracDecBin->hReverb = NULL; - } - else if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) /* Indication of binaural rendering with room effect */ - { - mvr2r( hHrtfParambin->parametricEarlyPartEneCorrection, hDiracDecBin->earlyPartEneCorrection, nBins ); + hDiracDecBin->hDiffuseDist = NULL; /* Memory is allocated from stack during runtime when needed */ + + if ( hDiracDecBin->hTdDecorr == NULL ) + { + hDiracDecBin->useTdDecorr = 0; + } - /* reconfiguration needed when Reverb. parameters are changed -> close and open the handle again */ - if ( hDiracDecBin->hReverb != NULL && ( ( hDiracDecBin->hReverb->numBins != nBins ) || - ( hDiracDecBin->hReverb->blockSize != CLDFB_SLOTS_PER_SUBFRAME ) ) ) + if ( hDiracDecBin->h_freq_domain_decorr_ap_params != NULL ) { - ivas_binaural_reverb_close( &( hDiracDecBin->hReverb ) ); + ivas_dirac_dec_decorr_close( &hDiracDecBin->h_freq_domain_decorr_ap_params, &hDiracDecBin->h_freq_domain_decorr_ap_state ); } - if ( hDiracDecBin->hReverb == NULL ) +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( pos_idx == 0) /* open decorrelator only for the main direction */ { - if ( ( error = ivas_binaural_reverb_open( &hDiracDecBin->hReverb, - nBins, - CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, NULL, - st_ivas->hIntSetup.output_config, - output_Fs, - RENDERER_BINAURAL_PARAMETRIC_ROOM, - st_ivas->hHrtfFastConv, - st_ivas->hHrtfParambin ) ) != IVAS_ERR_OK ) +#endif + if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( hDiracDecBin->hTdDecorr ), &( hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) { return error; } + if ( !hDiracDecBin->useTdDecorr && !( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM ) ) + { + ivas_dirac_dec_get_frequency_axis( frequency_axis, output_Fs, nBins ); + if ( ( error = ivas_dirac_dec_decorr_open( &( hDiracDecBin->h_freq_domain_decorr_ap_params ), + &( hDiracDecBin->h_freq_domain_decorr_ap_state ), + nBins, + BINAURAL_CHANNELS, + BINAURAL_CHANNELS, + DIRAC_SYNTHESIS_PSD_LS, + frequency_axis, + BINAURAL_CHANNELS, + output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN } - } - else if ( renderer_type == RENDERER_STEREO_PARAMETRIC ) - { - set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); - hDiracDecBin->hReverb = NULL; - hDiracDecBin->renderStereoOutputInsteadOfBinaural = 1; - } - else /* Not valid renderer type for this renderer */ - { - assert( false ); - } - - hDiracDecBin->hDiffuseDist = NULL; /* Memory is allocated from stack during runtime when needed */ - - if ( hDiracDecBin->hTdDecorr == NULL ) - { - hDiracDecBin->useTdDecorr = 0; - } - - if ( hDiracDecBin->h_freq_domain_decorr_ap_params != NULL ) - { - ivas_dirac_dec_decorr_close( &hDiracDecBin->h_freq_domain_decorr_ap_params, &hDiracDecBin->h_freq_domain_decorr_ap_state ); - } - - if ( ( error = ivas_td_decorr_reconfig_dec( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate, st_ivas->nchan_transport, st_ivas->hDecoderConfig->output_Fs, &( hDiracDecBin->hTdDecorr ), &( hDiracDecBin->useTdDecorr ) ) ) != IVAS_ERR_OK ) - { - return error; - } - - if ( !hDiracDecBin->useTdDecorr && !( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM ) ) - { - ivas_dirac_dec_get_frequency_axis( frequency_axis, output_Fs, nBins ); - if ( ( error = ivas_dirac_dec_decorr_open( &( hDiracDecBin->h_freq_domain_decorr_ap_params ), - &( hDiracDecBin->h_freq_domain_decorr_ap_state ), - nBins, - BINAURAL_CHANNELS, - BINAURAL_CHANNELS, - DIRAC_SYNTHESIS_PSD_LS, - frequency_axis, - BINAURAL_CHANNELS, - output_Fs ) ) != IVAS_ERR_OK ) + else { - return error; + hDiracDecBin->useTdDecorr = st_ivas->hDiracDecBin[0]->useTdDecorr; /* copy the flag, but the implementation re-uses the decorrelated signal */ } - } +#endif - hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); + hDiracDecBin->reqularizationFactor = configure_reqularization_factor( st_ivas->ivas_format, st_ivas->hDecoderConfig->ivas_total_brate ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + st_ivas->hDiracDecBin[pos_idx] = hDiracDecBin; + } +#else st_ivas->hDiracDecBin = hDiracDecBin; +#endif /* allocate transport channels*/ if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) @@ -302,11 +339,36 @@ void ivas_dirac_dec_close_binaural_data( DIRAC_DEC_BIN_HANDLE *hBinaural /* i/o: decoder DirAC binaural data handle */ ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; +#endif if ( hBinaural == NULL || *hBinaural == NULL ) { return; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) + { + if ( hBinaural[pos_idx] != NULL ) + { + if ( hBinaural[pos_idx]->hReverb != NULL ) + { + ivas_binaural_reverb_close( &( hBinaural[pos_idx]->hReverb ) ); + } + + ivas_td_decorr_dec_close( &( hBinaural[pos_idx]->hTdDecorr ) ); + + if ( hBinaural[pos_idx]->h_freq_domain_decorr_ap_params != NULL ) + { + ivas_dirac_dec_decorr_close( &(hBinaural[pos_idx]->h_freq_domain_decorr_ap_params), &(hBinaural[pos_idx]->h_freq_domain_decorr_ap_state) ); + } + + free( hBinaural[pos_idx] ); + hBinaural[pos_idx] = NULL; + } + } +#else if ( ( *hBinaural )->hReverb != NULL ) { ivas_binaural_reverb_close( &( ( *hBinaural )->hReverb ) ); @@ -320,6 +382,7 @@ void ivas_dirac_dec_close_binaural_data( free( *hBinaural ); *hBinaural = NULL; +#endif return; } @@ -423,6 +486,7 @@ void ivas_dirac_dec_binaural_render( { int16_t n_samples_sf = slot_size * hSpatParamRendCom->subframe_nbslots[subframe_idx]; ivas_dirac_dec_binaural_internal( st_ivas, st_ivas->hCombinedOrientationData, output_f_local, nchan_transport, subframe_idx ); + for ( ch = 0; ch < nchan_out; ch++ ) { output_f_local[ch] += n_samples_sf; @@ -519,8 +583,11 @@ void ivas_dirac_dec_binaural( { ivas_spar_dec_set_render_map( st_ivas, DEFAULT_JBM_CLDFB_TIMESLOTS ); } - +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( st_ivas->hDiracDecBin[0]->useTdDecorr ) +#else if ( st_ivas->hDiracDecBin->useTdDecorr ) +#endif { float *decorr_signal[BINAURAL_CHANNELS]; int16_t output_frame; @@ -532,7 +599,11 @@ void ivas_dirac_dec_binaural( } output_frame = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_td_decorr_process( st_ivas->hDiracDecBin[0]->hTdDecorr, p_output, decorr_signal, output_frame ); +#else ivas_td_decorr_process( st_ivas->hDiracDecBin->hTdDecorr, p_output, decorr_signal, output_frame ); +#endif } if ( nchan_transport == 1 && st_ivas->nchan_transport != 2 && st_ivas->hSCE[0]->hCoreCoder[0] != NULL && st_ivas->hSCE[0]->hCoreCoder[0]->cng_sba_flag ) @@ -588,13 +659,29 @@ static void ivas_dirac_dec_binaural_internal( int16_t nBins, offsetSamples; int16_t i, j; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + int16_t pos_idx; + MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + float tmp_Cldfb_out_re[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float tmp_Cldfb_out_im[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + /* these allow re-using the reverb and freq-domain decorrelator signals from ivas_dirac_dec_binaural_process_output() in split rendering for the side renderings */ + float reverbRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float decorrRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float decorrIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; + float IIReneLimiter[CLDFB_NO_CHANNELS_MAX]; + + hDiracDecBin = st_ivas->hDiracDecBin[0]; +#else hDiracDecBin = st_ivas->hDiracDecBin; +#endif assert( hDiracDecBin ); hSpatParamRendCom = st_ivas->hSpatParamRendCom; nBins = hSpatParamRendCom->num_freq_bands; offsetSamples = hSpatParamRendCom->slots_rendered * nBins; - /* Setuo internal config */ + /* Setup internal config */ config_data.separateCenterChannelRendering = st_ivas->hOutSetup.separateChannelEnabled; config_data.ivas_format = st_ivas->ivas_format; config_data.mc_mode = st_ivas->mc_mode; @@ -726,7 +813,6 @@ static void ivas_dirac_dec_binaural_internal( hDiracDecBin->hDiffuseDist = &diffuseDistData; ivas_spar_param_to_masa_param_mapping( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe ); - ivas_sba_prototype_renderer( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe ); } @@ -742,14 +828,19 @@ static void ivas_dirac_dec_binaural_internal( if ( nchan_transport == 2 ) { +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + /* in case of split rendering, determine the prototype rotation based on the main direction and use the same prototypes for the offset directions */ +#endif adaptTransportSignalsHeadtracked( hCombinedOrientationData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, hSpatParamRendCom->subframe_nbslots[subframe], Rmat ); ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( hCombinedOrientationData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, hSpatParamRendCom->subframe_nbslots[subframe], Rmat ); } } +#ifndef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); +#endif if ( config_data.ivas_format == ISM_FORMAT ) { @@ -764,13 +855,110 @@ static void ivas_dirac_dec_binaural_internal( max_band_decorr = hDiracDecBin->h_freq_domain_decorr_ap_params->max_band_decorr; } +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_binaural_formulate_input_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe, + subFrameTotalEne, IIReneLimiter ); + ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter ); +#endif ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); - ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, max_band_decorr, numInChannels, config_data.processReverb, subframe ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + pMultiBinPoseData = &st_ivas->splitBinRend.splitrend.multiBinPoseData; + + if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) + { + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, + reverbRe, reverbIm, decorrRe, decorrIm, 1 ); + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) + { + mvr2r(tmp_Cldfb_out_re[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + mvr2r(tmp_Cldfb_out_im[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + } + } + } + else + { + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, NULL, NULL, + reverbRe, reverbIm, decorrRe, decorrIm, 1 ); + } +#endif hDiracDecBin->hDiffuseDist = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) + { + /* quaternion-based rotation from ivas_binRenderer_internal.c:ivas_binRenderer(), but using absolute rotation instead of delta rotations */ + IVAS_QUATERNION Quaternions_rot, Quaternions_abs, *Quaternions_ref; + float Rmat_local[3][3]; + + if ( hCombinedOrientationData ) + { + Quaternions_ref = &hCombinedOrientationData->Quaternions[0]; + Quaternions_rot.w = -3.0f; /* signal to use Euler */ + Quaternions_abs.w = -3.0f; /* signal to use Euler */ + Quat2EulerDegree( *Quaternions_ref, &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + for ( pos_idx = 1; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + { + Quaternions_rot.x = Quaternions_abs.x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternions_rot.y = Quaternions_abs.y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternions_rot.z = Quaternions_abs.z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; + + QuatToRotMat( Quaternions_rot, Rmat_local ); + + hDiracDecBin = st_ivas->hDiracDecBin[pos_idx]; + assert(hDiracDecBin != NULL && "No DiracDecBin handle for this position"); + if ( config_data.ivas_format == SBA_FORMAT ) + { + hDiracDecBin->hDiffuseDist = &diffuseDistData; + } + + /* re-use input covariance for the side renderings */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + mvr2r(st_ivas->hDiracDecBin[0]->ChEne[ch], hDiracDecBin->ChEne[ch], hSpatParamRendCom->num_freq_bands); + } + mvr2r(st_ivas->hDiracDecBin[0]->ChCrossRe, hDiracDecBin->ChCrossRe, hSpatParamRendCom->num_freq_bands); + mvr2r(st_ivas->hDiracDecBin[0]->ChCrossIm, hDiracDecBin->ChCrossIm, hSpatParamRendCom->num_freq_bands); + + ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat_local, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter ); + + ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0 ); + + /* re-use reverb and decorr from main direction for the sides */ + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, st_ivas->cldfbSynDec, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, + reverbRe, reverbIm, + decorrRe, decorrIm, 0); + + /* copy from temporary buffer to the main split rendering buffer */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) + { + mvr2r(tmp_Cldfb_out_re[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + mvr2r(tmp_Cldfb_out_im[ch][i], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX); + } + } + + hDiracDecBin->hDiffuseDist = NULL; + } + } + } + + /* update this counter only after the last rendering of split directions */ +#endif hSpatParamRendCom->slots_rendered += hSpatParamRendCom->subframe_nbslots[subframe]; hSpatParamRendCom->subframes_rendered++; @@ -832,36 +1020,27 @@ static void ivas_dirac_dec_decorrelate_slot( return; } - -static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( +static void ivas_dirac_dec_binaural_formulate_input_covariance_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], - float Rmat[3][3], const int16_t subframe, - const int16_t isHeadtracked ) + float *subFrameTotalEne, + float *IIReneLimiter ) { int16_t ch, slot, bin; - int16_t separateCenterChannelRendering; - int16_t nBins, idx; - float frameMeanDiffusenessEneWeight[CLDFB_NO_CHANNELS_MAX]; + int16_t nBins; float IIReneLimiterFactor; float qualityBasedSmFactor; float lowBitRateEQ[CLDFB_NO_CHANNELS_MAX]; uint8_t applyLowBitRateEQ; - int16_t dirac_read_idx; - float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; - PARAMBIN_HRTF_GAIN_CACHE gainCache[MAX_GAIN_CACHE_SIZE]; IVAS_FORMAT ivas_format; - MC_MODE mc_mode; int32_t ivas_total_brate; int16_t nchan_transport; - separateCenterChannelRendering = hConfig->separateCenterChannelRendering; ivas_format = hConfig->ivas_format; - mc_mode = hConfig->mc_mode; ivas_total_brate = hConfig->ivas_total_brate; nchan_transport = hConfig->nchan_transport; qualityBasedSmFactor = hConfig->qualityBasedSmFactor; @@ -870,20 +1049,9 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric set_zero( hDiracDecBin->ChCrossRe, nBins ); set_zero( hDiracDecBin->ChCrossIm, nBins ); - set_zero( hDiracDecBin->ChCrossReOut, nBins ); - set_zero( hDiracDecBin->ChCrossImOut, nBins ); for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { set_zero( hDiracDecBin->ChEne[ch], nBins ); - set_zero( hDiracDecBin->ChEneOut[ch], nBins ); - } - set_zero( hDiracDecBin->frameMeanDiffuseness, nBins ); - - set_zero( frameMeanDiffusenessEneWeight, CLDFB_NO_CHANNELS_MAX ); - - for ( idx = 0; idx < MAX_GAIN_CACHE_SIZE; idx++ ) - { - gainCache[idx].azi = -1000; /* Use -1000 as value for uninitialized cache. */ } /* Determine EQ for low bit rates (13.2 and 16.4 kbps) */ @@ -909,7 +1077,6 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric /* Formulate input and target covariance matrices for this subframe */ set_zero( subFrameTotalEne, CLDFB_NO_CHANNELS_MAX ); - dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; /* Calculate input covariance matrix */ for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) @@ -964,6 +1131,96 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric } } + /* Temporal IIR-type smoothing of covariance matrices. Also apply encoding quality based smoothing factor. */ + if ( ivas_format == MASA_FORMAT && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) + { + IIReneLimiterFactor = 16.0f + ( 1.0f - qualityBasedSmFactor ); + } + else + { + IIReneLimiterFactor = 8.0f + ( 1.0f - qualityBasedSmFactor ); + } + for ( bin = 0; bin < nBins; bin++ ) + { + float eneRatio; + + /* Temporally smooth cov mtx estimates for resulting mixing matrix stability. The design principle is that + * the energy history (IIR) must not be more than double of the current frame energy. This provides more + * robust performance at energy offsets when compared to typical IIR averaging. */ + eneRatio = ( hDiracDecBin->ChEne[0][bin] + hDiracDecBin->ChEne[1][bin] ) / fmaxf( 1e-12f, ( hDiracDecBin->ChEnePrev[0][bin] + hDiracDecBin->ChEnePrev[1][bin] ) ); + IIReneLimiter[bin] = fminf( 1.0f, eneRatio * IIReneLimiterFactor ); + + hDiracDecBin->ChCrossRe[bin] *= qualityBasedSmFactor; + hDiracDecBin->ChCrossIm[bin] *= qualityBasedSmFactor; + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEne[ch][bin] *= qualityBasedSmFactor; + } + + hDiracDecBin->ChCrossRe[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossRePrev[bin]; + hDiracDecBin->ChCrossIm[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossImPrev[bin]; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEne[ch][bin] += IIReneLimiter[bin] * hDiracDecBin->ChEnePrev[ch][bin]; + } + + /* Store energy values and coefficients for next round */ + hDiracDecBin->ChCrossRePrev[bin] = hDiracDecBin->ChCrossRe[bin]; + hDiracDecBin->ChCrossImPrev[bin] = hDiracDecBin->ChCrossIm[bin]; + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + hDiracDecBin->ChEnePrev[ch][bin] = hDiracDecBin->ChEne[ch][bin]; + } + } + + return; +} + +static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( + DIRAC_DEC_BIN_HANDLE hDiracDecBin, + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, + PARAMBIN_REND_CONFIG_HANDLE hConfig, + float Rmat[3][3], + const int16_t subframe, + const int16_t isHeadtracked, + const float *subFrameTotalEne, + const float *IIReneLimiter ) +{ + int16_t ch, bin; + int16_t separateCenterChannelRendering; + int16_t nBins, idx; + float frameMeanDiffusenessEneWeight[CLDFB_NO_CHANNELS_MAX]; + float qualityBasedSmFactor; + int16_t dirac_read_idx; + PARAMBIN_HRTF_GAIN_CACHE gainCache[MAX_GAIN_CACHE_SIZE]; + IVAS_FORMAT ivas_format; + MC_MODE mc_mode; + + separateCenterChannelRendering = hConfig->separateCenterChannelRendering; + ivas_format = hConfig->ivas_format; + mc_mode = hConfig->mc_mode; + qualityBasedSmFactor = hConfig->qualityBasedSmFactor; + qualityBasedSmFactor *= qualityBasedSmFactor; + nBins = hSpatParamRendCom->num_freq_bands; /* Actually bins */ + + set_zero( hDiracDecBin->ChCrossReOut, nBins ); + set_zero( hDiracDecBin->ChCrossImOut, nBins ); + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + set_zero( hDiracDecBin->ChEneOut[ch], nBins ); + } + set_zero( hDiracDecBin->frameMeanDiffuseness, nBins ); + + set_zero( frameMeanDiffusenessEneWeight, CLDFB_NO_CHANNELS_MAX ); + + for ( idx = 0; idx < MAX_GAIN_CACHE_SIZE; idx++ ) + { + gainCache[idx].azi = -1000; /* Use -1000 as value for uninitialized cache. */ + } + + dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; + /* Determine target covariance matrix containing target binaural properties */ for ( bin = 0; bin < nBins; bin++ ) { @@ -1187,56 +1444,30 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric hDiracDecBin->frameMeanDiffuseness[bin] /= fmaxf( 1e-12f, frameMeanDiffusenessEneWeight[bin] ); } - /* Temporal IIR-type smoothing of covariance matrices. Also apply encoding quality based smoothing factor. */ - if ( ivas_format == MASA_FORMAT && ivas_total_brate < MASA_STEREO_MIN_BITRATE ) - { - IIReneLimiterFactor = 16.0f + ( 1.0f - qualityBasedSmFactor ); - } - else - { - IIReneLimiterFactor = 8.0f + ( 1.0f - qualityBasedSmFactor ); - } for ( bin = 0; bin < nBins; bin++ ) { - float eneRatio, IIReneLimiter; - - /* Temporally smooth cov mtx estimates for resulting mixing matrix stability. The design principle is that - * the energy history (IIR) must not be more than double of the current frame energy. This provides more - * robust performance at energy offsets when compared to typical IIR averaging. */ - eneRatio = ( hDiracDecBin->ChEne[0][bin] + hDiracDecBin->ChEne[1][bin] ) / fmaxf( 1e-12f, ( hDiracDecBin->ChEnePrev[0][bin] + hDiracDecBin->ChEnePrev[1][bin] ) ); - IIReneLimiter = fminf( 1.0f, eneRatio * IIReneLimiterFactor ); - - hDiracDecBin->ChCrossRe[bin] *= qualityBasedSmFactor; - hDiracDecBin->ChCrossIm[bin] *= qualityBasedSmFactor; hDiracDecBin->ChCrossReOut[bin] *= qualityBasedSmFactor; hDiracDecBin->ChCrossImOut[bin] *= qualityBasedSmFactor; for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { - hDiracDecBin->ChEne[ch][bin] *= qualityBasedSmFactor; hDiracDecBin->ChEneOut[ch][bin] *= qualityBasedSmFactor; } - hDiracDecBin->ChCrossRe[bin] += IIReneLimiter * hDiracDecBin->ChCrossRePrev[bin]; - hDiracDecBin->ChCrossIm[bin] += IIReneLimiter * hDiracDecBin->ChCrossImPrev[bin]; - hDiracDecBin->ChCrossReOut[bin] += IIReneLimiter * hDiracDecBin->ChCrossReOutPrev[bin]; - hDiracDecBin->ChCrossImOut[bin] += IIReneLimiter * hDiracDecBin->ChCrossImOutPrev[bin]; + hDiracDecBin->ChCrossReOut[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossReOutPrev[bin]; + hDiracDecBin->ChCrossImOut[bin] += IIReneLimiter[bin] * hDiracDecBin->ChCrossImOutPrev[bin]; for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { - hDiracDecBin->ChEne[ch][bin] += IIReneLimiter * hDiracDecBin->ChEnePrev[ch][bin]; - hDiracDecBin->ChEneOut[ch][bin] += IIReneLimiter * hDiracDecBin->ChEneOutPrev[ch][bin]; + hDiracDecBin->ChEneOut[ch][bin] += IIReneLimiter[bin] * hDiracDecBin->ChEneOutPrev[ch][bin]; } /* Store energy values and coefficients for next round */ - hDiracDecBin->ChCrossRePrev[bin] = hDiracDecBin->ChCrossRe[bin]; - hDiracDecBin->ChCrossImPrev[bin] = hDiracDecBin->ChCrossIm[bin]; hDiracDecBin->ChCrossReOutPrev[bin] = hDiracDecBin->ChCrossReOut[bin]; hDiracDecBin->ChCrossImOutPrev[bin] = hDiracDecBin->ChCrossImOut[bin]; for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { - hDiracDecBin->ChEnePrev[ch][bin] = hDiracDecBin->ChEne[ch][bin]; hDiracDecBin->ChEneOutPrev[ch][bin] = hDiracDecBin->ChEneOut[ch][bin]; } } @@ -1436,14 +1667,27 @@ static void ivas_dirac_dec_binaural_process_output( const int16_t max_band_decorr, const int16_t numInChannels, const int16_t processReverb, - const int16_t subframe ) + const int16_t subframe +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + , + float outRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float outIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float reverbRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float reverbIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float decorrRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float decorrIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const uint8_t recompute +#endif + ) { int16_t slot, bin, chA, chB; int16_t nBins; float outSlotRe[CLDFB_NO_CHANNELS_MAX], outSlotIm[CLDFB_NO_CHANNELS_MAX]; float decSlotRe[BINAURAL_CHANNELS][CLDFB_NO_CHANNELS_MAX], decSlotIm[BINAURAL_CHANNELS][CLDFB_NO_CHANNELS_MAX]; +#ifndef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN float reverbRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; +#endif float interpVal; float *decSlotRePointer; float *decSlotImPointer; @@ -1457,7 +1701,14 @@ static void ivas_dirac_dec_binaural_process_output( if ( processReverb ) { /* Process second / room effect part of binaural output when needed */ - ivas_binaural_reverb_processSubframe( hDiracDecBin->hReverb, numInChannels, nSlots, inRe, inIm, reverbRe, reverbIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( recompute == 1 ) + { +#endif + ivas_binaural_reverb_processSubframe( hDiracDecBin->hReverb, numInChannels, nSlots, inRe, inIm, reverbRe, reverbIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + } +#endif } interpVal = 0.0f; @@ -1466,7 +1717,27 @@ static void ivas_dirac_dec_binaural_process_output( interpVal += 1.0f / (float) nSlots; if ( !hDiracDecBin->useTdDecorr && max_band_decorr > 0 ) { - ivas_dirac_dec_decorrelate_slot( hDiracDecBin, nBins, slot, inRe, inIm, decSlotRe, decSlotIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( recompute == 1 ) + { +#endif + ivas_dirac_dec_decorrelate_slot( hDiracDecBin, nBins, slot, inRe, inIm, decSlotRe, decSlotIm ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( chA = 0; chA < BINAURAL_CHANNELS; chA++ ) + { + mvr2r(decSlotRe[chA], decorrRe[chA][slot], CLDFB_NO_CHANNELS_MAX); + mvr2r(decSlotIm[chA], decorrIm[chA][slot], CLDFB_NO_CHANNELS_MAX); + } + } + else + { + for ( chA = 0; chA < BINAURAL_CHANNELS; chA++ ) + { + mvr2r(decorrRe[chA][slot], decSlotRe[chA], CLDFB_NO_CHANNELS_MAX); + mvr2r(decorrIm[chA][slot], decSlotIm[chA], CLDFB_NO_CHANNELS_MAX); + } + } +#endif } for ( chA = 0; chA < BINAURAL_CHANNELS; chA++ ) @@ -1531,8 +1802,22 @@ static void ivas_dirac_dec_binaural_process_output( outSlotRePr = &( outSlotRe[0] ); outSlotImPr = &( outSlotIm[0] ); +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + if ( outRe != NULL && outIm != NULL ) + { + /* provide the data outside in CLDFB domain => mainly for split rendering */ + mvr2r(outSlotRePr, outRe[chA][slot], CLDFB_NO_CHANNELS_MAX); + mvr2r(outSlotImPr, outIm[chA][slot], CLDFB_NO_CHANNELS_MAX); + } + if ( recompute == 1 ) + { + /* Inverse filter bank */ + cldfbSynthesis( &outSlotRePr, &outSlotImPr, &( output_f[chA][nBins * slot + offsetSamples] ), nBins, cldfbSynDec[chA] ); + } +#else /* Inverse filter bank */ cldfbSynthesis( &outSlotRePr, &outSlotImPr, &( output_f[chA][nBins * slot + offsetSamples] ), nBins, cldfbSynDec[chA] ); +#endif } } diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 6dd53bdd025d3d6d75fe8c70f4b771223fa4c46d..10ea4d47a733443a8e1bccf5147f220d88858bbf 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -3232,7 +3232,14 @@ static DecoderDummy *initDecoderDummy( decDummy->hoa_dec_mtx = NULL; decDummy->hVBAPdata = NULL; // note: not used at the moment decDummy->hMasa = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) + { + decDummy->hDiracDecBin[i] = NULL; + } +#else decDummy->hDiracDecBin = NULL; +#endif decDummy->hDirACRend = NULL; decDummy->hSpatParamRendCom = NULL; decDummy->hQMetaData = NULL; @@ -3437,7 +3444,11 @@ static void freeDecoderDummy( free( pDecDummy->hHrtfParambin ); /* Parametric binaural renderer handle */ +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + ivas_dirac_dec_close_binaural_data( pDecDummy->hDiracDecBin ); +#else ivas_dirac_dec_close_binaural_data( &pDecDummy->hDiracDecBin ); +#endif /* TC buffer */ ivas_jbm_dec_tc_buffer_close( &pDecDummy->hTcBuffer ); @@ -8114,6 +8125,13 @@ static ivas_error renderInputMasa( //#endif // renderMasaToBinauralRoom( masaInput, outConfig, outAudio ); // break; +#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN + case IVAS_REND_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: + case IVAS_REND_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + /* TODO: implement */ + renderMasaToBinaural( masaInput, outAudio ); + break; +#endif default: return IVAS_ERR_INVALID_OUTPUT_FORMAT; } diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index 76f114b689ccec347587e915a2ea5450dec81ba2..3d93ee980a4f9b2fc06a3028248b0e8209cd6b92 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -37,8 +37,8 @@ from tests.split_rendering.utils import * @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajectory): - if in_fmt != "HOA3": - pytest.xfail("Split Rendering currently only supported with HOA3") + if in_fmt in ("MONO", "STEREO",): + pytest.xfail("Split Rendering currently not supported with MONO or STEREO") post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") @@ -56,8 +56,8 @@ def test_ambisonics_full_chain_split(test_info, in_fmt, render_config, trajector @pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) def test_ambisonics_external_split(test_info, in_fmt, render_config, trajectory): - if in_fmt != "HOA3": - pytest.xfail("Split Rendering currently only supported with HOA3") + if in_fmt in ("MONO", "STEREO",): + pytest.xfail("Split Rendering currently not supported with MONO or STEREO") post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed")