From fcfdadfa560bf39a545527c2e26d5b88c89fc80c Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 19 Jul 2023 17:47:12 +0200 Subject: [PATCH 1/5] fix for issue 194: introduce a delay buffer for LFE to binaural rendering --- lib_com/options.h | 1 + lib_rend/lib_rend.c | 115 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index b02e63cd1e..a0d5d2aa1b 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -183,6 +183,7 @@ #define FIX_MEM_REALLOC_IND_LIST /* VA: issue 601: failure of the automatic memory re-allocation mechanism when ind_list[] buffer is depleted in MASA mode with 2 TC*/ #define FIX_581_CLANG_OFFSET_TO_NULL /* FhG: issue 581: fix CLANG error about applying an offset to a NULL pointer */ #define JBM_PARAMUPMIX /* Dlb: Issue 471: Integrate the Multichannel Parametric Upmix into the JBM path */ +#define FIX_194_LFE_DELAY_EXTREND /* FhG: Issue 194: Fix delay alignment of LFE in external renderer */ /* Fixes for bugs found during split rendering contribution development */ #define REND_STATIC_MEM_OPT /* Dlb: Static memory optimisation for external renderer */ diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index e60268e8a8..c565e34b68 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -185,6 +185,10 @@ typedef struct lfe_routing lfeRouting; #ifdef REND_STATIC_MEM_OPT float *bufferData; +#endif +#ifdef FIX_194_LFE_DELAY_EXTREND + int16_t binauralDelaySmp; + float *lfeDelayBuffer; #endif MCMASA_ANA_HANDLE hMcMasa; } input_mc; @@ -305,6 +309,30 @@ static void freeInputBaseBufferData( float **data ) } #endif +#ifdef FIX_194_LFE_DELAY_EXTREND +static ivas_error allocateMcLfeDelayBuffer( float **lfeDelayBuffer, const int16_t data_size ) +{ + *lfeDelayBuffer = (float *) malloc( data_size * sizeof( float ) ); + if ( *lfeDelayBuffer == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Cannot allocate memory for LFE delay buffer" ); + } + + return IVAS_ERR_OK; +} + +static void freeMcLfeDelayBuffer( float **lfeDelayBuffer ) +{ + if ( *lfeDelayBuffer != NULL ) + { + free( *lfeDelayBuffer ); + *lfeDelayBuffer = NULL; + } + + return; +} +#endif + static IVAS_QUATERNION quaternionInit( void ) { @@ -2174,6 +2202,9 @@ static ivas_error initMcBinauralRendering( ivas_error error; #ifdef SPLIT_REND_WITH_HEAD_ROT int16_t i; +#endif +#ifdef FIX_194_LFE_DELAY_EXTREND + int32_t binauralDelayNs; #endif int32_t outSampleRate; @@ -2292,6 +2323,15 @@ static ivas_error initMcBinauralRendering( } } +#ifdef FIX_194_LFE_DELAY_EXTREND + /* determine binaural delay ( used for aligning LFE to output signal ) */ + binauralDelayNs = max( ( inputMc->crendWrapper != NULL ) ? inputMc->crendWrapper->binaural_latency_ns : 0, + ( &inputMc->tdRendWrapper != NULL ) ? inputMc->tdRendWrapper.binaural_latency_ns : 0 ); + inputMc->binauralDelaySmp = (int16_t) roundf( (float) binauralDelayNs * *inputMc->base.ctx.pOutSampleRate / 1000000000.f ); + + assert( ( inputMc->binauralDelaySmp < L_FRAME48k >> 2 ) && "Invalid delay for LFE binaural rendering!" ); + +#endif /* FIX_194_LFE_DELAY_EXTREND */ return IVAS_ERR_OK; } @@ -2408,6 +2448,13 @@ static ivas_error setRendInputActiveMc( return IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED; } +#ifdef FIX_194_LFE_DELAY_EXTREND + if ( ( error = allocateMcLfeDelayBuffer( &inputMc->lfeDelayBuffer, L_FRAME48k >> 2 ) ) != IVAS_ERR_OK ) + { + return error; + } + +#endif #ifdef REND_STATIC_MEM_OPT if ( ( error = allocateInputBaseBufferData( &inputMc->bufferData, MAX_BUFFER_LENGTH ) ) != IVAS_ERR_OK ) { @@ -2428,6 +2475,9 @@ static ivas_error setRendInputActiveMc( inputMc->hMcMasa = NULL; initRotGains( inputMc->rot_gains_prev ); inputMc->lfeRouting = defaultLfeRouting( inConfig, inputMc->customLsInput, outConfig, *inputMc->base.ctx.pCustomLsOut ); +#ifdef FIX_194_LFE_DELAY_EXTREND + set_zero( inputMc->lfeDelayBuffer, L_FRAME48k >> 2 ); +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT for ( i = 0; i < (int16_t) ( sizeof( inputMc->splitTdRendWrappers ) / sizeof( *inputMc->splitTdRendWrappers ) ); ++i ) @@ -2472,6 +2522,9 @@ static void clearInputMc( rendCtx = inputMc->base.ctx; +#ifdef FIX_194_LFE_DELAY_EXTREND + freeMcLfeDelayBuffer( &inputMc->lfeDelayBuffer ); +#endif #ifdef REND_STATIC_MEM_OPT freeInputBaseBufferData( &inputMc->bufferData ); #endif @@ -6287,11 +6340,17 @@ static ivas_error renderLfeToBinaural( IVAS_REND_AudioBuffer outAudio ) { int16_t i; +#ifdef FIX_194_LFE_DELAY_EXTREND + int16_t ear_idx; +#endif int16_t lfe_idx; #ifdef SPLIT_REND_WITH_HEAD_ROT int16_t pose_idx, num_poses; #endif float gain; +#ifdef FIX_194_LFE_DELAY_EXTREND + float *tmpLfeBuffer; +#endif float *readPtr, *writePtr; #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -6318,8 +6377,38 @@ static ivas_error renderLfeToBinaural( return IVAS_ERR_OK; } +#ifdef FIX_194_LFE_DELAY_EXTREND + /* Render LFE first to a temporary buffer to avoid applying gain multiple times */ + tmpLfeBuffer = malloc( mcInput->base.inputBuffer.config.numSamplesPerChannel * sizeof( float ) ); + if ( tmpLfeBuffer == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Failed to allocate memory for temporary LFE to Binaural buffer" ); + } + + /* 1. read from the LFE delay buffer and acc. to temp buffer + 2. read remaining samples from the input LFE channel and acc. to temp buffer */ + readPtr = mcInput->lfeDelayBuffer; + writePtr = tmpLfeBuffer; + for ( i = 0; i < mcInput->binauralDelaySmp; i++ ) + { + *writePtr++ += gain * ( *readPtr++ ); + } + readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); + for ( ; i < ( mcInput->base.inputBuffer.config.numSamplesPerChannel - mcInput->binauralDelaySmp ); i++ ) + { + *writePtr++ += gain * ( *readPtr++ ); + } + + /* save leftover samples in the LFE delay buffer for next frame */ + writePtr = mcInput->lfeDelayBuffer; + for ( i = 0; i < mcInput->binauralDelaySmp; i++ ) + { + *writePtr++ += gain * ( *readPtr++ ); + } + +#endif /* FIX_194_LFE_DELAY_EXTREND */ #ifdef SPLIT_REND_WITH_HEAD_ROT - /* Copy LFE to left and right binaural channels */ + /* Copy LFE to left and right binaural channels for all poses */ if ( mcInput->base.ctx.pSplitRendWrapper != NULL ) { num_poses = mcInput->base.ctx.pSplitRendWrapper->multiBinPoseData.num_poses; @@ -6331,6 +6420,14 @@ static ivas_error renderLfeToBinaural( for ( pose_idx = 0; pose_idx < num_poses; ++pose_idx ) { +#ifdef FIX_194_LFE_DELAY_EXTREND + for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) + { + mvr2r( tmpLfeBuffer, + getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS + ear_idx, 0 ), + mcInput->base.inputBuffer.config.numSamplesPerChannel ); + } +#else readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); writePtr = getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS, 0 ); for ( i = 0; i < mcInput->base.inputBuffer.config.numSamplesPerChannel; i++ ) @@ -6344,9 +6441,18 @@ static ivas_error renderLfeToBinaural( { *writePtr++ += gain * ( *readPtr++ ); } +#endif /* FIX_194_LFE_DELAY_EXTREND */ } -#else /* SPLIT_REND_WITH_HEAD_ROT */ +#else /* SPLIT_REND_WITH_HEAD_ROT */ /* Copy LFE to left and right ears */ +#ifdef FIX_194_LFE_DELAY_EXTREND + for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) + { + mvr2r( tmpLfeBuffer, + getSmplPtr( outAudio, ear_idx, 0 ), + mcInput->base.inputBuffer.config.numSamplesPerChannel ); + } +#else readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); writePtr = getSmplPtr( outAudio, 0, 0 ); for ( i = 0; i < mcInput->base.inputBuffer.config.numSamplesPerChannel; i++ ) @@ -6360,8 +6466,11 @@ static ivas_error renderLfeToBinaural( { *writePtr++ += gain * ( *readPtr++ ); } +#endif /* FIX_194_LFE_DELAY_EXTREND */ #endif /* SPLIT_REND_WITH_HEAD_ROT */ - pop_wmops(); +#ifdef FIX_194_LFE_DELAY_EXTREND + free( tmpLfeBuffer ); +#endif /* FIX_194_LFE_DELAY_EXTREND */ return IVAS_ERR_OK; } -- GitLab From 60a1e33f48d2140966210a37425591d4b39509c5 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 19 Jul 2023 17:50:05 +0200 Subject: [PATCH 2/5] [fix] accidentally removed pop_wmops() + whitespace --- lib_rend/lib_rend.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index c565e34b68..bb0d673f58 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -331,8 +331,8 @@ static void freeMcLfeDelayBuffer( float **lfeDelayBuffer ) return; } -#endif +#endif static IVAS_QUATERNION quaternionInit( void ) { @@ -6472,6 +6472,8 @@ static ivas_error renderLfeToBinaural( free( tmpLfeBuffer ); #endif /* FIX_194_LFE_DELAY_EXTREND */ + pop_wmops(); + return IVAS_ERR_OK; } -- GitLab From e2749cf7d6d7fa4e4edf6b39d073907c63a19565 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Wed, 19 Jul 2023 18:08:24 +0200 Subject: [PATCH 3/5] [fix] pointer not set to NULL causing MSAN errors --- lib_rend/lib_rend.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index bb0d673f58..9750087862 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -3718,6 +3718,9 @@ ivas_error IVAS_REND_Open( hIvasRend->inputsMc[i].tdRendWrapper.hBinRendererTd = NULL; #ifdef REND_STATIC_MEM_OPT hIvasRend->inputsMc[i].bufferData = NULL; +#endif +#ifdef FIX_194_LFE_DELAY_EXTREND + hIvasRend->inputsMc[i].lfeDelayBuffer = NULL; #endif hIvasRend->inputsMc[i].nonDiegeticPan = nonDiegeticPan; hIvasRend->inputsMc[i].nonDiegeticPanGain = nonDiegeticPanGain; -- GitLab From 08136bc6676740dd6ee5a2bac34b0f4efeb9bd01 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 20 Jul 2023 11:59:49 +0200 Subject: [PATCH 4/5] [fix] LFE buffer sizes and uninitialised reads --- lib_rend/lib_rend.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 9750087862..c83484295b 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -65,6 +65,10 @@ #define MAX_BUFFER_LENGTH ( MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL * MAX_INPUT_CHANNELS ) #endif /* REND_STATIC_MEM_OPT */ +#ifdef FIX_194_LFE_DELAY_EXTREND +#define MAX_BIN_DELAY_SAMPLES 50 /* Maximum supported rendering latency for binaural IRs */ +#endif + #else #define MAX_BUFFER_LENGTH ( MAX_BUFFER_LENGTH_PER_CHANNEL * MAX_INPUT_CHANNELS ) #endif @@ -2326,10 +2330,13 @@ static ivas_error initMcBinauralRendering( #ifdef FIX_194_LFE_DELAY_EXTREND /* determine binaural delay ( used for aligning LFE to output signal ) */ binauralDelayNs = max( ( inputMc->crendWrapper != NULL ) ? inputMc->crendWrapper->binaural_latency_ns : 0, - ( &inputMc->tdRendWrapper != NULL ) ? inputMc->tdRendWrapper.binaural_latency_ns : 0 ); + inputMc->tdRendWrapper.binaural_latency_ns ); inputMc->binauralDelaySmp = (int16_t) roundf( (float) binauralDelayNs * *inputMc->base.ctx.pOutSampleRate / 1000000000.f ); - assert( ( inputMc->binauralDelaySmp < L_FRAME48k >> 2 ) && "Invalid delay for LFE binaural rendering!" ); + if ( inputMc->binauralDelaySmp > MAX_BIN_DELAY_SAMPLES ) + { + return IVAS_ERROR( IVAS_ERR_WRONG_PARAMS, "Invalid delay for LFE binaural rendering!)" ); + } #endif /* FIX_194_LFE_DELAY_EXTREND */ return IVAS_ERR_OK; @@ -2449,7 +2456,7 @@ static ivas_error setRendInputActiveMc( } #ifdef FIX_194_LFE_DELAY_EXTREND - if ( ( error = allocateMcLfeDelayBuffer( &inputMc->lfeDelayBuffer, L_FRAME48k >> 2 ) ) != IVAS_ERR_OK ) + if ( ( error = allocateMcLfeDelayBuffer( &inputMc->lfeDelayBuffer, MAX_BIN_DELAY_SAMPLES ) ) != IVAS_ERR_OK ) { return error; } @@ -2476,7 +2483,7 @@ static ivas_error setRendInputActiveMc( initRotGains( inputMc->rot_gains_prev ); inputMc->lfeRouting = defaultLfeRouting( inConfig, inputMc->customLsInput, outConfig, *inputMc->base.ctx.pCustomLsOut ); #ifdef FIX_194_LFE_DELAY_EXTREND - set_zero( inputMc->lfeDelayBuffer, L_FRAME48k >> 2 ); + set_zero( inputMc->lfeDelayBuffer, MAX_BIN_DELAY_SAMPLES ); #endif #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -6388,25 +6395,25 @@ static ivas_error renderLfeToBinaural( return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Failed to allocate memory for temporary LFE to Binaural buffer" ); } - /* 1. read from the LFE delay buffer and acc. to temp buffer - 2. read remaining samples from the input LFE channel and acc. to temp buffer */ + /* 1. read from the LFE delay buffer into temp buffer + 2. read remaining samples from the input LFE channel into temp buffer */ readPtr = mcInput->lfeDelayBuffer; writePtr = tmpLfeBuffer; for ( i = 0; i < mcInput->binauralDelaySmp; i++ ) { - *writePtr++ += gain * ( *readPtr++ ); + *writePtr++ = gain * ( *readPtr++ ); } readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); for ( ; i < ( mcInput->base.inputBuffer.config.numSamplesPerChannel - mcInput->binauralDelaySmp ); i++ ) { - *writePtr++ += gain * ( *readPtr++ ); + *writePtr++ = gain * ( *readPtr++ ); } /* save leftover samples in the LFE delay buffer for next frame */ writePtr = mcInput->lfeDelayBuffer; for ( i = 0; i < mcInput->binauralDelaySmp; i++ ) { - *writePtr++ += gain * ( *readPtr++ ); + *writePtr++ = gain * ( *readPtr++ ); } #endif /* FIX_194_LFE_DELAY_EXTREND */ -- GitLab From dea5994716b240c1b0ad56d39e9e5d875ab0805e Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Thu, 20 Jul 2023 12:44:18 +0200 Subject: [PATCH 5/5] Clean up LFE delay compensation --- lib_rend/lib_rend.c | 69 +++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 43 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index c83484295b..a9a0469790 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -6349,19 +6349,21 @@ static ivas_error renderLfeToBinaural( #endif IVAS_REND_AudioBuffer outAudio ) { - int16_t i; -#ifdef FIX_194_LFE_DELAY_EXTREND - int16_t ear_idx; -#endif int16_t lfe_idx; #ifdef SPLIT_REND_WITH_HEAD_ROT int16_t pose_idx, num_poses; #endif float gain; #ifdef FIX_194_LFE_DELAY_EXTREND - float *tmpLfeBuffer; -#endif + int16_t ear_idx; + float tmpLfeBuffer[MAX_BUFFER_LENGTH_PER_CHANNEL]; + int16_t frame_size, num_cpy_smpl_cur_frame, num_cpy_smpl_prev_frame; + const float *lfeInput; + float *writePtr; +#else + int16_t i; float *readPtr, *writePtr; +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT assert( ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) && "Must be binaural output" ); @@ -6388,35 +6390,21 @@ static ivas_error renderLfeToBinaural( } #ifdef FIX_194_LFE_DELAY_EXTREND - /* Render LFE first to a temporary buffer to avoid applying gain multiple times */ - tmpLfeBuffer = malloc( mcInput->base.inputBuffer.config.numSamplesPerChannel * sizeof( float ) ); - if ( tmpLfeBuffer == NULL ) - { - return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Failed to allocate memory for temporary LFE to Binaural buffer" ); - } - - /* 1. read from the LFE delay buffer into temp buffer - 2. read remaining samples from the input LFE channel into temp buffer */ - readPtr = mcInput->lfeDelayBuffer; - writePtr = tmpLfeBuffer; - for ( i = 0; i < mcInput->binauralDelaySmp; i++ ) - { - *writePtr++ = gain * ( *readPtr++ ); - } - readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); - for ( ; i < ( mcInput->base.inputBuffer.config.numSamplesPerChannel - mcInput->binauralDelaySmp ); i++ ) - { - *writePtr++ = gain * ( *readPtr++ ); - } - - /* save leftover samples in the LFE delay buffer for next frame */ - writePtr = mcInput->lfeDelayBuffer; - for ( i = 0; i < mcInput->binauralDelaySmp; i++ ) - { - *writePtr++ = gain * ( *readPtr++ ); - } + /* --- Prepare LFE signal to be added to binaural output --- */ + lfeInput = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); + frame_size = mcInput->base.inputBuffer.config.numSamplesPerChannel; + num_cpy_smpl_prev_frame = mcInput->binauralDelaySmp; + num_cpy_smpl_cur_frame = frame_size - num_cpy_smpl_prev_frame; + /* Assuming LFE should be delayed by less that the duration of one frame */ + assert( mcInput->binauralDelaySmp < frame_size ); + /* Get delayed LFE signal from previous frame, apply gain and save in tmp buffer */ + v_multc( mcInput->lfeDelayBuffer, gain, tmpLfeBuffer, num_cpy_smpl_prev_frame ); + /* Continue filling tmp buffer, now with LFE signal from current frame */ + v_multc( lfeInput, gain, tmpLfeBuffer + num_cpy_smpl_prev_frame, num_cpy_smpl_cur_frame ); + /* Save remaining LFE samples of current frame for next frame */ + mvr2r( lfeInput + num_cpy_smpl_cur_frame, mcInput->lfeDelayBuffer, num_cpy_smpl_prev_frame ); +#endif -#endif /* FIX_194_LFE_DELAY_EXTREND */ #ifdef SPLIT_REND_WITH_HEAD_ROT /* Copy LFE to left and right binaural channels for all poses */ if ( mcInput->base.ctx.pSplitRendWrapper != NULL ) @@ -6433,9 +6421,8 @@ static ivas_error renderLfeToBinaural( #ifdef FIX_194_LFE_DELAY_EXTREND for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) { - mvr2r( tmpLfeBuffer, - getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS + ear_idx, 0 ), - mcInput->base.inputBuffer.config.numSamplesPerChannel ); + writePtr = getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS + ear_idx, 0 ); + v_add( writePtr, tmpLfeBuffer, writePtr, frame_size ); } #else readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); @@ -6458,9 +6445,8 @@ static ivas_error renderLfeToBinaural( #ifdef FIX_194_LFE_DELAY_EXTREND for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) { - mvr2r( tmpLfeBuffer, - getSmplPtr( outAudio, ear_idx, 0 ), - mcInput->base.inputBuffer.config.numSamplesPerChannel ); + writePtr = getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS + ear_idx, 0 ); + v_add( writePtr, tmpLfeBuffer, writePtr, frame_size ); } #else readPtr = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); @@ -6478,9 +6464,6 @@ static ivas_error renderLfeToBinaural( } #endif /* FIX_194_LFE_DELAY_EXTREND */ #endif /* SPLIT_REND_WITH_HEAD_ROT */ -#ifdef FIX_194_LFE_DELAY_EXTREND - free( tmpLfeBuffer ); -#endif /* FIX_194_LFE_DELAY_EXTREND */ pop_wmops(); -- GitLab