binaural rendering for jbm with 7.1.4 upmix, matching non-jbm decoding (5697ddda) · Commits · IVAS Codec Public Collaboration / IVAS Codec

lib_dec/ivas_mc_paramupmix_dec.c

+202 −3

Original line number	Diff line number	Diff line
		@@ -58,7 +58,11 @@ static void ps_pred_process( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, float qmf_m
		#ifdef JBM_PARAMUPMIX
		static void ps_pred_process_sf( MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix, DECODER_TC_BUFFER_HANDLE hTcBuffer, float qmf_mod_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_mod_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_re[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float qmf_side_im[JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float *param_interpol, const int16_t ch, const int16_t slots_rendered );

		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		static void ivas_mc_paramupmix_dec_sf( Decoder_Struct st_ivas, float output_f[MAX_OUTPUT_CHANNELS], int16_t slot_index_start );
		#else
		static void ivas_mc_paramupmix_dec_sf( Decoder_Struct st_ivas, float output_f[MAX_OUTPUT_CHANNELS] );
		#endif

		static void ivas_param_upmix_dec_decorr_subframes( Decoder_Struct *st_ivas, const int16_t nSamplesForRendering );
		#endif
		@@ -182,13 +186,16 @@ void ivas_mc_paramupmix_dec(
		int16_t noparamupmix_delay;
		AUDIO_CONFIG output_config;
		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		float Cldfb_RealBuffer_subfr[3+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_subfr[3+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		int16_t subframeIdx, idx_in, index_slot, maxBand;
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		float Cldfb_RealBuffer_subfr[4+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_subfr[4+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_RealBuffer_Binaural[1][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_Binaural[1][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		int16_t idx_lfe;
		#else
		float Cldfb_RealBuffer_subfr[3+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_subfr[3+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];
		#endif
		@@ -306,11 +313,15 @@ void ivas_mc_paramupmix_dec(
		/* slot loop for gathering the input data */
		for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ )
		{
		cldfbAnalysis_ts( &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * (index_slot+slot_idx)] ), Cldfb_RealBuffer_subfr[idx_in][slot_idx], Cldfb_ImagBuffer_subfr[idx_in][slot_idx], hMCParamUpmix->num_freq_bands, st_ivas->cldfbAnaDec[2*MC_PARAMUPMIX_COMBINATIONS+idx_in] );
		cldfbAnalysis_ts( &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * (index_slot+slot_idx)] ),
		Cldfb_RealBuffer_subfr[idx_in][slot_idx],
		Cldfb_ImagBuffer_subfr[idx_in][slot_idx],
		maxBand, st_ivas->cldfbAnaDec[2*MC_PARAMUPMIX_COMBINATIONS+idx_in] );
		}
		idx_in++;
		}
		}
		/* copy and reorder cldfb analysis of coupled channels */
		for (ch=0; ch<MAX_PARAM_SPATIAL_SUBFRAMES; ch++)
		{
		for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ )
		@@ -322,6 +333,37 @@ void ivas_mc_paramupmix_dec(
		}
		idx_in += 2;
		}
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		/LFE handling for split rendering cases/
		if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) \|\|
		( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) )
		{
		for ( idx_lfe = 0; idx_lfe < st_ivas->hIntSetup.num_lfe; idx_lfe++ )
		{
		ch = st_ivas->hIntSetup.index_lfe[idx_lfe];
		/* slot loop for gathering the input data */
		for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ )
		{
		cldfbAnalysis_ts( &( output_f[ch][maxBand * index_slot] ),
		Cldfb_RealBuffer_subfr[idx_in][slot_idx],
		Cldfb_ImagBuffer_subfr[idx_in][slot_idx],
		maxBand, st_ivas->cldfbAnaDec[idx_in] );
		}
		idx_in++;
		}
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		if ( st_ivas->splitBinRend.hCldfbDataOut != NULL )
		{
		for ( ch = 0; ch < ( st_ivas->hIntSetup.nchan_out_woLFE + st_ivas->hIntSetup.num_lfe ); ch++ )
		{
		mvr2r( Cldfb_RealBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_RealBuffer[ch][( subframeIdx * JBM_CLDFB_SLOTS_IN_SUBFRAME ) + slot_idx], maxBand );
		mvr2r( Cldfb_ImagBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_ImagBuffer[ch][( subframeIdx * JBM_CLDFB_SLOTS_IN_SUBFRAME ) + slot_idx], maxBand );
		}
		st_ivas->splitBinRend.hCldfbDataOut->config = st_ivas->hIntSetup.output_config;
		}
		#endif
		}
		#endif
		/* Implement binaural rendering */
		ivas_binRenderer( st_ivas->hBinRenderer,
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		@@ -444,6 +486,9 @@ void ivas_mc_paramupmix_dec_render(
		int16_t slots_to_render, first_sf, last_sf, subframe_idx;
		uint16_t slot_size, ch;
		float *output_f_local[MAX_OUTPUT_CHANNELS];
		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		int16_t slot_index_start;
		#endif
		MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix;

		hMCParamUpmix = st_ivas->hMCParamUpmix;
		@@ -477,12 +522,14 @@ void ivas_mc_paramupmix_dec_render(
		#ifdef DEBUGGING
		assert( slots_to_render == 0 );
		#endif
		#ifndef PARAMUPMIX_BINAURAL_UPDATES
		/* Implement binaural rendering */
		if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV \|\| st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM )
		{
		ivas_binaural_cldfb_sf( st_ivas, *nSamplesRendered, slot_size, output_f_local );
		}
		else
		#endif
		{

		for ( ch = 0; ch < MC_PARAMUPMIX_COMBINATIONS; ch++ )
		@@ -491,11 +538,19 @@ void ivas_mc_paramupmix_dec_render(
		mvr2r( hMCParamUpmix->beta_prev[ch], hMCParamUpmix->beta_sf[ch], IVAS_MAX_NUM_BANDS );
		}

		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		slot_index_start = 0;
		#endif
		for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ )
		{
		int16_t n_samples_sf = slot_size * st_ivas->hTcBuffer->subframe_nbslots[subframe_idx];

		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local, slot_index_start );
		slot_index_start += st_ivas->hTcBuffer->subframe_nbslots[subframe_idx];
		#else
		ivas_mc_paramupmix_dec_sf( st_ivas, output_f_local );
		#endif
		for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ )
		{
		output_f_local[ch] += n_samples_sf;
		@@ -915,6 +970,9 @@ static void ps_pred_process_sf(
		static void ivas_mc_paramupmix_dec_sf(
		Decoder_Struct st_ivas, / i/o: IVAS decoder handle */
		float output_f[MAX_OUTPUT_CHANNELS] / i/o: synthesized core-coder transport channels */
		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		,int16_t slot_index_start
		#endif
		)
		{
		int16_t i, ch, slot_idx, k;
		@@ -925,6 +983,24 @@ static void ivas_mc_paramupmix_dec_sf(
		int16_t noparamupmix_delay, n_samples_rendered;
		MC_PARAMUPMIX_DEC_HANDLE hMCParamUpmix;
		hMCParamUpmix = st_ivas->hMCParamUpmix;
		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		int16_t subframeIdx, idx_in, maxBand;
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		float Cldfb_RealBuffer_subfr[4+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_subfr[4+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_RealBuffer_Binaural[1][BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_Binaural[1][BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		int16_t idx_lfe;
		#else
		float Cldfb_RealBuffer_subfr[3+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_subfr[3+MC_PARAMUPMIX_MAX_TRANSPORT_CHANS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX];
		#endif
		/* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */
		int16_t chidx1[MC_PARAMUPMIX_COMBINATIONS] = {0, 1, 4, 5};
		int16_t chidx2[MC_PARAMUPMIX_COMBINATIONS] = {2, 3, 6, 7};
		#endif
		assert( hMCParamUpmix );

		push_wmops( "ivas_mc_paramupmix_dec_sf" );
		@@ -973,6 +1049,126 @@ static void ivas_mc_paramupmix_dec_sf(
		}
		}
		}
		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV \|\| st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM )
		{
		maxBand = (int16_t) ( ( CLDFB_NO_CHANNELS_MAX * st_ivas->hDecoderConfig->output_Fs ) / 48000 );
		subframeIdx = st_ivas->hTcBuffer->subframes_rendered;
		/* fastconv binaural rendering and CLDFB synthesis */
		/* cldfb analysis of non-coupled, non-LFE channels */
		idx_in = 0;
		for ( ch = 0; ch < 12-2*MC_PARAMUPMIX_COMBINATIONS; ch++ )
		{
		if ( st_ivas->hIntSetup.index_lfe[0] != ch )
		{
		pPcm_temp[ch] = output_f[ch];
		/* slot loop for gathering the input data */
		for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ )
		{
		cldfbAnalysis_ts( &( pPcm_temp[ch][hMCParamUpmix->num_freq_bands * slot_idx] ),
		Cldfb_RealBuffer_subfr[idx_in][slot_idx],
		Cldfb_ImagBuffer_subfr[idx_in][slot_idx],
		maxBand, st_ivas->cldfbAnaDec[2*MC_PARAMUPMIX_COMBINATIONS+idx_in] );
		}
		idx_in++;
		}
		}
		/* copy and reorder cldfb analysis of coupled channels */
		for (ch=0; ch<MAX_PARAM_SPATIAL_SUBFRAMES; ch++)
		{
		for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ )
		{
		mvr2r(Cldfb_RealBuffer[chidx1[ch]][slot_idx], Cldfb_RealBuffer_subfr[idx_in][slot_idx], CLDFB_NO_CHANNELS_MAX);
		mvr2r(Cldfb_ImagBuffer[chidx1[ch]][slot_idx], Cldfb_ImagBuffer_subfr[idx_in][slot_idx], CLDFB_NO_CHANNELS_MAX);
		mvr2r(Cldfb_RealBuffer[chidx2[ch]][slot_idx], Cldfb_RealBuffer_subfr[idx_in+1][slot_idx], CLDFB_NO_CHANNELS_MAX);
		mvr2r(Cldfb_ImagBuffer[chidx2[ch]][slot_idx], Cldfb_ImagBuffer_subfr[idx_in+1][slot_idx], CLDFB_NO_CHANNELS_MAX);
		}
		idx_in += 2;
		}
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		/LFE handling for split rendering cases/
		if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) \|\|
		( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) )
		{
		for ( idx_lfe = 0; idx_lfe < st_ivas->hIntSetup.num_lfe; idx_lfe++ )
		{
		ch = st_ivas->hIntSetup.index_lfe[idx_lfe];
		for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered]; slot_idx++ )
		{
		cldfbAnalysis_ts( &( output_f[ch][maxBand * slot_idx] ),
		Cldfb_RealBuffer_subfr[idx_in][slot_idx],
		Cldfb_ImagBuffer_subfr[idx_in][slot_idx],
		maxBand, st_ivas->cldfbAnaDec[idx_in] );
		}
		idx_in++;
		}
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		if ( st_ivas->splitBinRend.hCldfbDataOut != NULL )
		{
		for ( ch = 0; ch < ( st_ivas->hIntSetup.nchan_out_woLFE + st_ivas->hIntSetup.num_lfe ); ch++ )
		{
		mvr2r( Cldfb_RealBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_RealBuffer[ch][slot_index_start + slot_idx], maxBand );
		mvr2r( Cldfb_ImagBuffer_subfr[ch][slot_idx], st_ivas->splitBinRend.hCldfbDataOut->Cldfb_ImagBuffer[ch][slot_index_start + slot_idx], maxBand );
		}
		st_ivas->splitBinRend.hCldfbDataOut->config = st_ivas->hIntSetup.output_config;
		}
		#endif
		}
		#endif
		/* Implement binaural rendering */
		ivas_binRenderer( st_ivas->hBinRenderer,
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		&st_ivas->splitBinRend.splitrend.multiBinPoseData,
		#endif
		st_ivas->hCombinedOrientationData,
		subframeIdx, st_ivas->hTcBuffer->subframe_nbslots[subframeIdx],
		Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural,
		Cldfb_RealBuffer_subfr,
		Cldfb_ImagBuffer_subfr );

		#ifdef SPLIT_REND_WITH_HEAD_ROT
		if ( ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) \|\|
		( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) )
		{
		int16_t pos_idx;
		for ( pos_idx = 0; pos_idx < st_ivas->hBinRenderer->numPoses; pos_idx++ )
		{
		for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[subframeIdx]; slot_idx++ )
		{
		for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ )
		{
		mvr2r( Cldfb_RealBuffer_Binaural[pos_idx][ch][slot_idx], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[( pos_idx * BINAURAL_CHANNELS ) + ch][slot_index_start + slot_idx], maxBand );
		mvr2r( Cldfb_ImagBuffer_Binaural[pos_idx][ch][slot_idx], st_ivas->splitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[( pos_idx * BINAURAL_CHANNELS ) + ch][slot_index_start + slot_idx], maxBand );
		}
		}
		}
		}
		#endif


		/* Implement CLDFB synthesis */
		for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
		{
		float *RealBuffer[MAX_PARAM_SPATIAL_SUBFRAMES];
		float *ImagBuffer[MAX_PARAM_SPATIAL_SUBFRAMES];

		for ( slot_idx = 0; slot_idx < st_ivas->hTcBuffer->subframe_nbslots[subframeIdx]; slot_idx++ )
		{
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[0][ch][slot_idx];
		ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[0][ch][slot_idx];
		#else
		RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[ch][slot_idx];
		ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[ch][slot_idx];
		#endif
		}

		cldfbSynthesis( RealBuffer, ImagBuffer, &( output_f[ch][0] ), maxBand * st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], st_ivas->cldfbSynDec[ch] );
		}
		}
		else
		{
		#endif
		/* boxes = { 0 1 2 3 [4 6] [5 7] [8 10] [9 11] }; */
		pPcm_temp[0] = output_f[4];
		pPcm_temp[1] = output_f[6];
		@@ -1021,6 +1217,9 @@ static void ivas_mc_paramupmix_dec_sf(
		mvr2r( tmp_buf, &hMCParamUpmix->pcm_delay[ch][noparamupmix_delay - n_samples_rendered], n_samples_rendered );
		}
		}
		#ifdef PARAMUPMIX_BINAURAL_UPDATES
		}
		#endif

		st_ivas->hTcBuffer->slots_rendered += st_ivas->hTcBuffer->subframe_nbslots[st_ivas->hTcBuffer->subframes_rendered];
		st_ivas->hTcBuffer->subframes_rendered++;