From a3ceaf4e5701ab25630be92ca715c5706402e4b3 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 8 Mar 2024 13:02:14 +0200 Subject: [PATCH 01/29] Adds split rendering support for OMASA --- lib_com/options.h | 2 ++ lib_dec/ivas_init_dec.c | 19 ++++++++++++++++ lib_dec/ivas_jbm_dec.c | 13 ++++++++++- lib_dec/ivas_omasa_dec.c | 49 +++++++++++++++++++++++++++++++++++++++- lib_dec/lib_dec.c | 24 ++++++++++++++++++++ 5 files changed, 105 insertions(+), 2 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index 9bfbf6050c..28b587bf46 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -163,6 +163,8 @@ #define FIX_1002_DEC_PHASE_ECU_USAN_OF_PHASE /* Eri: issue #1002, usan-value-out-of-range-for-int16, kept BE for PLC-conditions */ #define FIX_960_SYN_OUTPUT /* VA: issue 960: unused function syn_output() is removed */ +#define FIX_766_OMASA_SPLIT_REND /* Nokia: issue #766: split-rendering support for OMASA */ + /* #################### End BE switches ################################## */ /* #################### Start NON-BE switches ############################ */ diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index b92dc19ded..1d144b76b2 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -95,8 +95,16 @@ static ivas_error ivas_dec_reconfig_split_rend( isCldfbNeeded = 0; #ifndef SPLIT_REND_WITH_HEAD_ROT cldfbMode = CLDFB_ANALYSIS; +#else +#ifdef FIX_766_OMASA_SPLIT_REND + if ( ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) || + ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) + { + cldfb_in_flag = 0; /* this allocates cldfb analysis for td-rend results */ + } #else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) +#endif { cldfb_in_flag = 0; } @@ -193,8 +201,14 @@ static ivas_error ivas_dec_reconfig_split_rend( } #ifdef SPLIT_REND_WITH_HEAD_ROT +#ifdef FIX_766_OMASA_SPLIT_REND + if ( ( st_ivas->renderer_type != RENDERER_BINAURAL_OBJECTS_TD ) && + ( st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV || st_ivas->ivas_format != SBA_ISM_FORMAT ) && + !( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) /* td-rend not needed? */ +#else if ( ( st_ivas->renderer_type != RENDERER_BINAURAL_OBJECTS_TD ) && ( st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV || st_ivas->ivas_format != SBA_ISM_FORMAT ) ) +#endif #else if ( st_ivas->renderer_type != RENDERER_BINAURAL_OBJECTS_TD ) #endif @@ -253,7 +267,12 @@ static ivas_error ivas_dec_init_split_rend( } #ifdef SPLIT_REND_WITH_HEAD_ROT +#ifdef FIX_766_OMASA_SPLIT_REND + if ( ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) || + ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) +#else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) +#endif { cldfb_in_flag = 0; } diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 128754273c..0223675d59 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -932,7 +932,18 @@ void ivas_jbm_dec_feed_tc_to_renderer( { v_multc( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], OMASA_TDREND_MATCHING_GAIN, st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available ); } - delay_signal( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available, st_ivas->hMasaIsmData->delayBuffer[n], st_ivas->hMasaIsmData->delayBuffer_size ); +#ifdef FIX_766_OMASA_SPLIT_REND +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { +#endif +#endif + delay_signal( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available, st_ivas->hMasaIsmData->delayBuffer[n], st_ivas->hMasaIsmData->delayBuffer_size ); +#ifdef FIX_766_OMASA_SPLIT_REND +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif +#endif } } } diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index 763c37a709..73c82e2747 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -749,6 +749,49 @@ ivas_error ivas_omasa_dirac_td_binaural_jbm( /* reset combined orientation access index before calling the td renderer */ ivas_combined_orientation_set_to_start_index( st_ivas->hCombinedOrientationData ); +#ifdef FIX_766_OMASA_SPLIT_REND +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + int16_t slot_idx, num_cldfb_bands, nchan_transport_orig, cldfb_slots; + float Cldfb_RealBuffer[CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer[CLDFB_NO_CHANNELS_MAX]; + + float data_rend_objects[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; /* float[8 * 2][960] */ + float *p_rend_obj[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS]; + + for ( n = 0; n < MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS; n++ ) + { + p_rend_obj[n] = &data_rend_objects[n][0]; + } + + num_cldfb_bands = st_ivas->hSplitBinRend.splitrend.hCldfbHandles->cldfbAna[0]->no_channels; + nchan_transport_orig = st_ivas->nchan_transport; + st_ivas->nchan_transport = st_ivas->nchan_ism; + + if ( ( error = ivas_td_binaural_renderer_sf_splitBinaural( st_ivas, p_rend_obj, *nSamplesRendered ) ) != IVAS_ERR_OK ) /* objects are read from st_ivas->hTcBuffer->tc[2..(1+n_isms)] */ + { + return error; + } + st_ivas->nchan_transport = nchan_transport_orig; + cldfb_slots = *nSamplesRendered / num_cldfb_bands; + + for ( n = 0; n < st_ivas->hSplitBinRend.splitrend.multiBinPoseData.num_poses * BINAURAL_CHANNELS; ++n ) + { + for ( slot_idx = 0; slot_idx < cldfb_slots; slot_idx++ ) + { + cldfbAnalysis_ts( &( p_rend_obj[n][num_cldfb_bands * slot_idx] ), Cldfb_RealBuffer, Cldfb_ImagBuffer, num_cldfb_bands, st_ivas->hSplitBinRend.splitrend.hCldfbHandles->cldfbAna[n] ); + + /* this differs from OSBA by: no scaling by 0.5, and omit slot offset in right-hand side buffer as this is always 0 */ + v_add( st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[n][slot_idx], Cldfb_RealBuffer, st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[n][slot_idx], num_cldfb_bands ); + v_add( st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[n][slot_idx], Cldfb_ImagBuffer, st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[n][slot_idx], num_cldfb_bands ); + } + } + } + else + { +#endif +#endif if ( ( error = ivas_td_binaural_renderer_sf( st_ivas, p_sepobj, *nSamplesRendered ) ) != IVAS_ERR_OK ) { return error; @@ -757,7 +800,11 @@ ivas_error ivas_omasa_dirac_td_binaural_jbm( { v_add( output_f[n], p_sepobj[n], output_f[n], *nSamplesRendered ); } - +#ifdef FIX_766_OMASA_SPLIT_REND +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif +#endif return IVAS_ERR_OK; } diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index 4e9dc2ac03..687cc4f5b1 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -1108,7 +1108,19 @@ ivas_error IVAS_DEC_GetSplitBinauralBitstream( } else { +#ifdef FIX_766_OMASA_SPLIT_REND + /* TODO: check correctness: this is needed for OMASA-DISC, because the td-rend granularity is 240 samples at 48kHz, leading to wrong slot count. */ + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + slots_rendered = st_ivas->hTcBuffer->n_samples_rendered / NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ); + } + else + { + slots_rendered = st_ivas->hTcBuffer->n_samples_rendered / st_ivas->hTcBuffer->n_samples_granularity; + } +#else slots_rendered = st_ivas->hTcBuffer->n_samples_rendered / st_ivas->hTcBuffer->n_samples_granularity; +#endif } @@ -1137,7 +1149,19 @@ ivas_error IVAS_DEC_GetSplitBinauralBitstream( } else { +#ifdef FIX_766_OMASA_SPLIT_REND + /* TODO: check correctness: this is needed for OMASA-DISC, because the td-rend granularity is 240 samples at 48kHz, leading to wrong slot count. */ + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + slots_rendered_new = st_ivas->hTcBuffer->n_samples_rendered / NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ); + } + else + { + slots_rendered_new = st_ivas->hTcBuffer->n_samples_rendered / st_ivas->hTcBuffer->n_samples_granularity; + } +#else slots_rendered_new = st_ivas->hTcBuffer->n_samples_rendered / st_ivas->hTcBuffer->n_samples_granularity; +#endif } for ( i = 0; i < BINAURAL_CHANNELS * numPoses; ++i ) -- GitLab From 7f3bfafe336a8bef486be6939fd26261ae007553 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Fri, 8 Mar 2024 12:13:30 +0100 Subject: [PATCH 02/29] apply clang formatting --- lib_dec/ivas_init_dec.c | 4 ++-- lib_dec/ivas_omasa_dec.c | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 1d144b76b2..6aaf2182ce 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -100,7 +100,7 @@ static ivas_error ivas_dec_reconfig_split_rend( if ( ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) || ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) { - cldfb_in_flag = 0; /* this allocates cldfb analysis for td-rend results */ + cldfb_in_flag = 0; /* this allocates cldfb analysis for td-rend results */ } #else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) @@ -204,7 +204,7 @@ static ivas_error ivas_dec_reconfig_split_rend( #ifdef FIX_766_OMASA_SPLIT_REND if ( ( st_ivas->renderer_type != RENDERER_BINAURAL_OBJECTS_TD ) && ( st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV || st_ivas->ivas_format != SBA_ISM_FORMAT ) && - !( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) /* td-rend not needed? */ + !( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) /* td-rend not needed? */ #else if ( ( st_ivas->renderer_type != RENDERER_BINAURAL_OBJECTS_TD ) && ( st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV || st_ivas->ivas_format != SBA_ISM_FORMAT ) ) diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index 73c82e2747..6e63c57a4c 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -757,7 +757,7 @@ ivas_error ivas_omasa_dirac_td_binaural_jbm( float Cldfb_RealBuffer[CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer[CLDFB_NO_CHANNELS_MAX]; - float data_rend_objects[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; /* float[8 * 2][960] */ + float data_rend_objects[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; /* float[8 * 2][960] */ float *p_rend_obj[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS]; for ( n = 0; n < MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS; n++ ) @@ -769,7 +769,7 @@ ivas_error ivas_omasa_dirac_td_binaural_jbm( nchan_transport_orig = st_ivas->nchan_transport; st_ivas->nchan_transport = st_ivas->nchan_ism; - if ( ( error = ivas_td_binaural_renderer_sf_splitBinaural( st_ivas, p_rend_obj, *nSamplesRendered ) ) != IVAS_ERR_OK ) /* objects are read from st_ivas->hTcBuffer->tc[2..(1+n_isms)] */ + if ( ( error = ivas_td_binaural_renderer_sf_splitBinaural( st_ivas, p_rend_obj, *nSamplesRendered ) ) != IVAS_ERR_OK ) /* objects are read from st_ivas->hTcBuffer->tc[2..(1+n_isms)] */ { return error; } @@ -792,14 +792,14 @@ ivas_error ivas_omasa_dirac_td_binaural_jbm( { #endif #endif - if ( ( error = ivas_td_binaural_renderer_sf( st_ivas, p_sepobj, *nSamplesRendered ) ) != IVAS_ERR_OK ) - { - return error; - } - for ( n = 0; n < BINAURAL_CHANNELS; n++ ) - { - v_add( output_f[n], p_sepobj[n], output_f[n], *nSamplesRendered ); - } + if ( ( error = ivas_td_binaural_renderer_sf( st_ivas, p_sepobj, *nSamplesRendered ) ) != IVAS_ERR_OK ) + { + return error; + } + for ( n = 0; n < BINAURAL_CHANNELS; n++ ) + { + v_add( output_f[n], p_sepobj[n], output_f[n], *nSamplesRendered ); + } #ifdef FIX_766_OMASA_SPLIT_REND #ifdef SPLIT_REND_WITH_HEAD_ROT } -- GitLab From 5bd06c2a41e2db4ea3b00f994872d7857f62ad9a Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Mon, 11 Mar 2024 13:25:14 +0100 Subject: [PATCH 03/29] add OMASA into split rendering tests --- tests/renderer/constants.py | 60 +++++++++++++++++++ tests/split_rendering/README.md | 5 +- tests/split_rendering/constants.py | 13 ++++ tests/split_rendering/test_split_rendering.py | 24 ++++++++ tests/split_rendering/utils.py | 2 +- 5 files changed, 101 insertions(+), 3 deletions(-) diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 9592103e1d..0e7b13c8ac 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -114,6 +114,14 @@ FORMAT_TO_FILE_SMOKETEST = { "NDP_ISM4": NCHAN_TO_FILE[4], "MASA1": NCHAN_TO_FILE[1], "MASA2": NCHAN_TO_FILE[2], + "OMASA_1_1": NCHAN_TO_FILE[2], + "OMASA_1_2": NCHAN_TO_FILE[3], + "OMASA_1_3": NCHAN_TO_FILE[4], + "OMASA_1_4": NCHAN_TO_FILE[5], + "OMASA_2_1": NCHAN_TO_FILE[3], + "OMASA_2_2": NCHAN_TO_FILE[4], + "OMASA_2_3": NCHAN_TO_FILE[5], + "OMASA_2_4": NCHAN_TO_FILE[6], "OSBA_1_1": NCHAN_TO_FILE[5], "OSBA_2_1": NCHAN_TO_FILE[6], "OSBA_3_1": NCHAN_TO_FILE[7], @@ -149,6 +157,14 @@ FORMAT_TO_FILE_COMPARETEST = { "ISM4": TESTV_DIR.joinpath("stv4ISM48s.wav"), "MASA1": TESTV_DIR.joinpath("stv1MASA1TC48c.wav"), "MASA2": TESTV_DIR.joinpath("stv2MASA2TC48c.wav"), + "OMASA_1_1": TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA1TC48c.wav"), + "OMASA_1_2": TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA1TC48c.wav"), + "OMASA_1_3": TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA1TC48c.wav"), + "OMASA_1_4": TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA1TC48c.wav"), + "OMASA_2_1": TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA2TC48c.wav"), + "OMASA_2_2": TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA2TC48c.wav"), + "OMASA_2_3": TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA2TC48c.wav"), + "OMASA_2_4": TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA2TC48c.wav"), "OSBA_1_1": TESTV_DIR.joinpath("stvOSBA_1ISM_FOA48c.wav"), "OSBA_1_2": TESTV_DIR.joinpath("stvOSBA_1ISM_2OA48c.wav"), "OSBA_1_3": TESTV_DIR.joinpath("stvOSBA_1ISM_3OA48c.wav"), @@ -192,6 +208,50 @@ FORMAT_TO_METADATA_FILES = { ], "MASA1": [str(TESTV_DIR.joinpath("stv1MASA1TC48c.met"))], "MASA2": [str(TESTV_DIR.joinpath("stv2MASA2TC48c.met"))], + "OMASA_1_1": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA1TC48c.met")), + ], + "OMASA_1_2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA1TC48c.met")), + ], + "OMASA_1_3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA1TC48c.met")), + ], + "OMASA_1_4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + str(TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA1TC48c.met")), + ], + "OMASA_2_1": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvOMASA_1ISM_1MASA2TC48c.met")), + ], + "OMASA_2_2": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvOMASA_2ISM_2MASA2TC48c.met")), + ], + "OMASA_2_3": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvOMASA_3ISM_1MASA2TC48c.met")), + ], + "OMASA_2_4": [ + str(TESTV_DIR.joinpath("stvISM1.csv")), + str(TESTV_DIR.joinpath("stvISM2.csv")), + str(TESTV_DIR.joinpath("stvISM3.csv")), + str(TESTV_DIR.joinpath("stvISM4.csv")), + str(TESTV_DIR.joinpath("stvOMASA_4ISM_2MASA2TC48c.met")), + ], } diff --git a/tests/split_rendering/README.md b/tests/split_rendering/README.md index 0ff6b8dd0c..b63214c13b 100644 --- a/tests/split_rendering/README.md +++ b/tests/split_rendering/README.md @@ -1,5 +1,5 @@