From 9dda5a52569d25b0f43680eedced9ed51d895378 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 24 Jun 2024 16:46:11 +0200 Subject: [PATCH 1/8] [fix] enale FOA/HOA2 rendering with FastConv in external renderer --- apps/renderer.c | 11 +++++++++-- lib_com/options.h | 1 + lib_dec/ivas_binRenderer_internal.c | 16 +++++++++++++++- lib_rend/ivas_rotation.c | 3 ++- lib_rend/lib_rend.c | 3 ++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/apps/renderer.c b/apps/renderer.c index cf97acf947..c0a6f90785 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1055,10 +1055,17 @@ int main( exit( -1 ); } - CLDFBframeSize_smpls = frameSize_smpls * 2; - cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); +#ifndef FIX_1113_EXTREND_ISAR + CLDFBframeSize_smpls = frameSize_smpls * 2; + cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); +#endif } +#ifdef FIX_1113_EXTREND_ISAR + CLDFBframeSize_smpls = frameSize_smpls * 2; + cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); +#endif + if ( ( error = IVAS_REND_SetOrientationTrackingMode( hIvasRend, args.orientation_tracking ) ) != IVAS_ERR_OK ) { fprintf( stderr, "Error in IVAS_REND_SetOrientationTrackingMode(): %s\n", ivas_error_to_string( error ) ); diff --git a/lib_com/options.h b/lib_com/options.h index 4d8e7d5f74..76c4615f6c 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -165,6 +165,7 @@ #define FIX_1111_TDM_LSP_BUFFER /* VA: issue 1111: remove unused buffer `tdm_lspQ_PCh[]' */ #define FIX_1101_CLEANING_JBM_CALL /* VA: issue 1101: remove obsolete call of ivas_jbm_dec_tc_buffer_open() */ #define FIX_1053_AEID_FILE_TEST /* Philips: Tests for the -aeid file and fix for memory error */ +#define FIX_1113_EXTREND_ISAR /* FhG: issue 1113: fix external renderer asserts for FOA/HOA2 and CLDFB config */ /* #################### End BE switches ################################## */ diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index fbf8c916a0..8a65defbce 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -998,8 +998,12 @@ ivas_error ivas_rend_openCldfbRend( hBinRenderer->conv_band = convBand; } +#ifdef FIX_1113_EXTREND_ISAR + ivas_output_init( hBinRenderer->hInputSetup, inConfig ); +#else hBinRenderer->hInputSetup->is_loudspeaker_setup = 0; hBinRenderer->hInputSetup->output_config = inConfig; +#endif if ( ( error = getAudioConfigNumChannels( inConfig, &hBinRenderer->hInputSetup->nchan_out_woLFE ) ) != IVAS_ERR_OK ) { return error; @@ -1846,8 +1850,14 @@ void ivas_binRenderer( } } - /* HOA decoding to CICP19 if needed*/ +/* HOA decoding to CICP19 if needed*/ +#ifdef FIX_1113_EXTREND_ISAR + if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && + ( hBinRenderer->nInChannels != 16 && hBinRenderer->nInChannels != 9 && hBinRenderer->nInChannels != 4 ) ) + // if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && hBinRenderer->hInputSetup->output_config == IVAS_AUDIO_CONFIG_7_1_4 ) +#else if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && hBinRenderer->nInChannels != 16 ) +#endif { ivas_sba2mc_cldfb( *( hBinRenderer->hInputSetup ), RealBuffer, ImagBuffer, hBinRenderer->nInChannels, hBinRenderer->conv_band, numTimeSlots, hBinRenderer->hoa_dec_mtx ); } @@ -1905,7 +1915,11 @@ void ivas_binRenderer( } else { +#ifdef FIX_1113_EXTREND_ISAR + rotateFrame_shd_cldfb( RealBuffer, ImagBuffer, Rmat_local, hBinRenderer->hInputSetup->nchan_out_woLFE, numTimeSlots, hBinRenderer->hInputSetup->ambisonics_order ); +#else rotateFrame_shd_cldfb( RealBuffer, ImagBuffer, Rmat_local, hBinRenderer->hInputSetup->nchan_out_woLFE, numTimeSlots, 3 ); +#endif } ivas_binRenderer_filterModule( Cldfb_RealBuffer_Binaural[pos_idx], Cldfb_ImagBuffer_Binaural[pos_idx], RealBuffer, ImagBuffer, numTimeSlots, hBinRenderer, pos_idx ); diff --git a/lib_rend/ivas_rotation.c b/lib_rend/ivas_rotation.c index 198100dfff..71ee98d090 100644 --- a/lib_rend/ivas_rotation.c +++ b/lib_rend/ivas_rotation.c @@ -501,8 +501,9 @@ void rotateFrame_shd_cldfb( int16_t l = 0, m1 = 0, m2 = 0; float realRot[2 * HEADROT_ORDER + 1], imagRot[2 * HEADROT_ORDER + 1]; float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM]; - +#ifndef FIX_1113_EXTREND_ISAR assert( nInChannels == HEADROT_SHMAT_DIM && "Number of channels must be 16!" ); +#endif /* initialize rotation matrices with zeros */ for ( i = 0; i < HEADROT_SHMAT_DIM; i++ ) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 6e3257b6c8..ff170b4873 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2511,9 +2511,10 @@ static ivas_error updateSbaPanGains( case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: { if ( hRendCfg->split_rend_config.rendererSelection == ISAR_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) - { +#ifndef FIX_1113_EXTREND_ISAR assert( inConfig == IVAS_AUDIO_CONFIG_HOA3 && ( *rendCtx.pOutSampleRate == 48000 ) && "split binaural fast conv mode is currently supported with HOA3 input and 48k sampling rate only" ); +#endif if ( ( error = ivas_rend_openCldfbRend( &inputSba->cldfbRendWrapper, inConfig, outConfig, &rendCtx.pSplitRendWrapper->multiBinPoseData, *rendCtx.pOutSampleRate ) ) != IVAS_ERR_OK ) { return error; -- GitLab From 02af984f583b5e30c2b30495dd3e728e2326739a Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 24 Jun 2024 16:49:15 +0200 Subject: [PATCH 2/8] clang-format --- apps/renderer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/renderer.c b/apps/renderer.c index c0a6f90785..a43540cf03 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1056,8 +1056,8 @@ int main( } #ifndef FIX_1113_EXTREND_ISAR - CLDFBframeSize_smpls = frameSize_smpls * 2; - cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); + CLDFBframeSize_smpls = frameSize_smpls * 2; + cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); #endif } -- GitLab From f7a738a1e7e110bb089aaf2b679a2ca12737b699 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 24 Jun 2024 16:54:44 +0200 Subject: [PATCH 3/8] [fix] update assert instead of removing it to avoid compilation error --- lib_rend/ivas_rotation.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib_rend/ivas_rotation.c b/lib_rend/ivas_rotation.c index 71ee98d090..8791bc6856 100644 --- a/lib_rend/ivas_rotation.c +++ b/lib_rend/ivas_rotation.c @@ -501,7 +501,10 @@ void rotateFrame_shd_cldfb( int16_t l = 0, m1 = 0, m2 = 0; float realRot[2 * HEADROT_ORDER + 1], imagRot[2 * HEADROT_ORDER + 1]; float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM]; -#ifndef FIX_1113_EXTREND_ISAR +#ifdef FIX_1113_EXTREND_ISAR + assert( ( nInChannels == HOA3_CHANNELS || nInChannels == HOA2_CHANNELS || nInChannels == FOA_CHANNELS ) && + "Number of channels must correspond to an ambisonics order!" ); +#else assert( nInChannels == HEADROT_SHMAT_DIM && "Number of channels must be 16!" ); #endif -- GitLab From c110be5b2f2ff0386cfaa041c1a2b558e43c3fa2 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 24 Jun 2024 17:24:43 +0200 Subject: [PATCH 4/8] [cleanup] - revert change in renderer.c - needs fix on API level - correct IVAS_REND_{Get,Feed}RenderConfig function return types - remove commented if condition. --- apps/renderer.c | 6 ------ lib_dec/ivas_binRenderer_internal.c | 1 - lib_rend/lib_rend.c | 8 ++++++++ lib_rend/lib_rend.h | 8 ++++++++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/apps/renderer.c b/apps/renderer.c index a43540cf03..61633a3c0a 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1055,16 +1055,10 @@ int main( exit( -1 ); } -#ifndef FIX_1113_EXTREND_ISAR CLDFBframeSize_smpls = frameSize_smpls * 2; cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); -#endif } -#ifdef FIX_1113_EXTREND_ISAR - CLDFBframeSize_smpls = frameSize_smpls * 2; - cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); -#endif if ( ( error = IVAS_REND_SetOrientationTrackingMode( hIvasRend, args.orientation_tracking ) ) != IVAS_ERR_OK ) { diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index 8a65defbce..a51be6122b 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -1854,7 +1854,6 @@ void ivas_binRenderer( #ifdef FIX_1113_EXTREND_ISAR if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && ( hBinRenderer->nInChannels != 16 && hBinRenderer->nInChannels != 9 && hBinRenderer->nInChannels != 4 ) ) - // if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && hBinRenderer->hInputSetup->output_config == IVAS_AUDIO_CONFIG_7_1_4 ) #else if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && hBinRenderer->nInChannels != 16 ) #endif diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index ff170b4873..296e273948 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -4138,7 +4138,11 @@ ivas_error IVAS_REND_InitConfig( * *-------------------------------------------------------------------*/ +#ifdef FIX_1113_EXTREND_ISAR +ivas_error IVAS_REND_GetRenderConfig( +#else int16_t IVAS_REND_GetRenderConfig( +#endif IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render configuration handle */ ) @@ -4201,7 +4205,11 @@ int16_t IVAS_REND_GetRenderConfig( * *-------------------------------------------------------------------*/ +#ifdef FIX_1113_EXTREND_ISAR +ivas_error IVAS_REND_FeedRenderConfig( +#else int16_t IVAS_REND_FeedRenderConfig( +#endif IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ const IVAS_RENDER_CONFIG_DATA renderConfig /* i : Render configuration struct */ ) diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index 5cd2b07f61..615ba65eee 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -238,12 +238,20 @@ ivas_error IVAS_REND_InitConfig( const IVAS_AUDIO_CONFIG outAudioConfig /* i : output audioConfig */ ); +#ifdef FIX_1113_EXTREND_ISAR +ivas_error IVAS_REND_GetRenderConfig( +#else int16_t IVAS_REND_GetRenderConfig( +#endif IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render configuration handle */ ); +#ifdef FIX_1113_EXTREND_ISAR +ivas_error IVAS_REND_FeedRenderConfig( +#else int16_t IVAS_REND_FeedRenderConfig( +#endif IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ const IVAS_RENDER_CONFIG_DATA renderConfig /* i : Render configuration struct */ ); -- GitLab From 845fb82e48f82b90a55cf35db3cc29c9371237da Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Mon, 24 Jun 2024 17:56:21 +0200 Subject: [PATCH 5/8] [fix] unintentional whitespace changes --- apps/renderer.c | 1 - lib_rend/ivas_rotation.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/renderer.c b/apps/renderer.c index 61633a3c0a..cf97acf947 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1059,7 +1059,6 @@ int main( cldfb_in_flag = get_cldfb_in_flag( args.outConfig.audioConfig, &renderConfig ); } - if ( ( error = IVAS_REND_SetOrientationTrackingMode( hIvasRend, args.orientation_tracking ) ) != IVAS_ERR_OK ) { fprintf( stderr, "Error in IVAS_REND_SetOrientationTrackingMode(): %s\n", ivas_error_to_string( error ) ); diff --git a/lib_rend/ivas_rotation.c b/lib_rend/ivas_rotation.c index 8791bc6856..c28d7430ae 100644 --- a/lib_rend/ivas_rotation.c +++ b/lib_rend/ivas_rotation.c @@ -501,6 +501,7 @@ void rotateFrame_shd_cldfb( int16_t l = 0, m1 = 0, m2 = 0; float realRot[2 * HEADROT_ORDER + 1], imagRot[2 * HEADROT_ORDER + 1]; float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM]; + #ifdef FIX_1113_EXTREND_ISAR assert( ( nInChannels == HOA3_CHANNELS || nInChannels == HOA2_CHANNELS || nInChannels == FOA_CHANNELS ) && "Number of channels must correspond to an ambisonics order!" ); -- GitLab From 5281dd3b601c495aa819ae2e1c026ebcf68e8814 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 25 Jun 2024 09:32:49 +0200 Subject: [PATCH 6/8] retain assert for 48kHz sampling rate --- lib_rend/lib_rend.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 296e273948..c2133d2f6d 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2512,7 +2512,9 @@ static ivas_error updateSbaPanGains( { if ( hRendCfg->split_rend_config.rendererSelection == ISAR_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) { -#ifndef FIX_1113_EXTREND_ISAR +#ifdef FIX_1113_EXTREND_ISAR + assert( *rendCtx.pOutSampleRate == 48000 && "split binaural fast conv mode is currently supported with 48k sampling rate only" ); +#else assert( inConfig == IVAS_AUDIO_CONFIG_HOA3 && ( *rendCtx.pOutSampleRate == 48000 ) && "split binaural fast conv mode is currently supported with HOA3 input and 48k sampling rate only" ); #endif if ( ( error = ivas_rend_openCldfbRend( &inputSba->cldfbRendWrapper, inConfig, outConfig, &rendCtx.pSplitRendWrapper->multiBinPoseData, *rendCtx.pOutSampleRate ) ) != IVAS_ERR_OK ) -- GitLab From 184af2d3ef95a69d8756cb2ea57b2e9ade75fe25 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 25 Jun 2024 13:26:14 +0200 Subject: [PATCH 7/8] [fix] update setting of binaural_latency_ns value according to input format --- lib_dec/ivas_binRenderer_internal.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index 42ad7f512a..6a9072b571 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -1019,7 +1019,27 @@ ivas_error ivas_rend_openCldfbRend( } #ifdef FIX_1113_CLDFB_REND_IN_ISAR +#ifdef FIX_1113_EXTREND_ISAR + if ( inConfig == IVAS_AUDIO_CONFIG_FOA ) + { + pCldfbRend->binaural_latency_ns = (int32_t) ( pCldfbRend->hHrtfFastConv->FASTCONV_FOA_latency_s * 1000000000.f ); + } + else if ( inConfig == IVAS_AUDIO_CONFIG_HOA2 ) + { + pCldfbRend->binaural_latency_ns = (int32_t) ( pCldfbRend->hHrtfFastConv->FASTCONV_HOA2_latency_s * 1000000000.f ); + } + else if ( inConfig == IVAS_AUDIO_CONFIG_HOA3 ) + { + pCldfbRend->binaural_latency_ns = (int32_t) ( pCldfbRend->hHrtfFastConv->FASTCONV_HOA3_latency_s * 1000000000.f ); + } + else + { + /* should never happen for SBA */ + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } +#else pCldfbRend->binaural_latency_ns = (int32_t) ( pCldfbRend->hHrtfFastConv->FASTCONV_HOA3_latency_s * 1000000000.f ); +#endif #else pCldfbRend->binaural_latency_ns = (int32_t) ( FASTCONV_HOA3_latency_s * 1000000000.f ); #endif -- GitLab From 8c7c89fbdd192fa08e18337bcabb08ce15fd1b16 Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Tue, 23 Jul 2024 10:20:48 +0200 Subject: [PATCH 8/8] [fix] comparison of nInChannels to defines instead of hardcoded values --- lib_dec/ivas_binRenderer_internal.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index 6a9072b571..ae81aeae0a 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -885,8 +885,12 @@ static void ivas_binaural_obtain_DMX( set_zero( outRealRightPtr, CLDFB_NO_CHANNELS_MAX ); set_zero( outImagRightPtr, CLDFB_NO_CHANNELS_MAX ); - /*Ambisonics input requires different processing*/ +/*Ambisonics input requires different processing*/ +#ifdef FIX_1113_CLDFB_REND_IN_ISAR + if ( hBinRenderer->nInChannels == HOA3_CHANNELS ) +#else if ( hBinRenderer->nInChannels == 16 ) +#endif { float *inRealPtr_W, *inImagPtr_W; float *inRealPtr_Y, *inImagPtr_Y; @@ -1176,15 +1180,27 @@ ivas_error ivas_binRenderer_open( } else { +#ifdef FIX_1113_CLDFB_REND_IN_ISAR + if ( hBinRenderer->nInChannels == HOA3_CHANNELS ) +#else if ( hBinRenderer->nInChannels == 16 ) +#endif { st_ivas->binaural_latency_ns = (int32_t) ( st_ivas->hHrtfFastConv->FASTCONV_HOA3_latency_s * 1000000000.f ); } +#ifdef FIX_1113_CLDFB_REND_IN_ISAR + else if ( hBinRenderer->nInChannels == HOA2_CHANNELS ) +#else else if ( hBinRenderer->nInChannels == 9 ) +#endif { st_ivas->binaural_latency_ns = (int32_t) ( st_ivas->hHrtfFastConv->FASTCONV_HOA2_latency_s * 1000000000.f ); } +#ifdef FIX_1113_CLDFB_REND_IN_ISAR + else if ( hBinRenderer->nInChannels == FOA_CHANNELS ) +#else else if ( hBinRenderer->nInChannels == 4 ) +#endif { st_ivas->binaural_latency_ns = (int32_t) ( st_ivas->hHrtfFastConv->FASTCONV_FOA_latency_s * 1000000000.f ); } @@ -1861,7 +1877,7 @@ void ivas_binRenderer( /* HOA decoding to CICP19 if needed*/ #ifdef FIX_1113_EXTREND_ISAR if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && - ( hBinRenderer->nInChannels != 16 && hBinRenderer->nInChannels != 9 && hBinRenderer->nInChannels != 4 ) ) + ( hBinRenderer->nInChannels != HOA3_CHANNELS && hBinRenderer->nInChannels != HOA2_CHANNELS && hBinRenderer->nInChannels != FOA_CHANNELS ) ) #else if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 && hBinRenderer->nInChannels != 16 ) #endif -- GitLab