From 625ace491c8b55e9cd02a3562421571c78236405 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Wed, 11 Oct 2023 15:19:54 +0300 Subject: [PATCH 1/9] Rewrites MASA ext renderer using direct renderer specific functions instead of decoder functionality. Performs refactors on other code where necessary. --- apps/renderer.c | 25 +- lib_dec/ivas_dirac_dec.c | 7 +- lib_dec/ivas_rom_dec.c | 140 - lib_dec/ivas_rom_dec.h | 20 - lib_rend/ivas_dirac_dec_binaural_functions.c | 283 + lib_rend/ivas_dirac_decorr_dec.c | 3 +- lib_rend/ivas_dirac_output_synthesis_dec.c | 20 +- lib_rend/ivas_dirac_rend.c | 421 +- lib_rend/ivas_prot_rend.h | 33 +- lib_rend/ivas_rom_rend.c | 143 + lib_rend/ivas_rom_rend.h | 22 + lib_rend/ivas_stat_rend.h | 28 + lib_rend/lib_rend.c | 8801 +++++++++--------- 13 files changed, 5424 insertions(+), 4522 deletions(-) diff --git a/apps/renderer.c b/apps/renderer.c index 9f0c242529..981dbef367 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1762,15 +1762,22 @@ int main( exit( -1 ); } - if ( masaReaders[i] != NULL ) + if ( isCurrentFrameMultipleOf20ms ) { - /* This will update data in hMasaMetadata[i] */ - MasaFileReader_readNextFrame( masaReaders[i] ); - - if ( ( error = IVAS_REND_FeedInputMasaMetadata( hIvasRend, masaIds[i], hMasaMetadata[i] ) ) != IVAS_ERR_OK ) + if ( masaReaders[i] != NULL ) { - fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); - exit( -1 ); + /* This will update data in hMasaMetadata[i] */ + if ( ( error = MasaFileReader_readNextFrame( masaReaders[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in MASA Metadata File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( ( error = IVAS_REND_FeedInputMasaMetadata( hIvasRend, masaIds[i], hMasaMetadata[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } } } } @@ -2444,9 +2451,7 @@ static IVAS_AUDIO_CONFIG parseAudioConfig( switch ( charBuf[4] ) { case '1': - fprintf( stderr, "1TC MASA support is not functional and is pending on DirAC renderer refactoring.\n" ); - exit( EXIT_FAILURE ); - /*return IVAS_AUDIO_CONFIG_MASA1;*/ // ToDo: temporarily disabled to avoid compilation warnings + return IVAS_AUDIO_CONFIG_MASA1; case '2': return IVAS_AUDIO_CONFIG_MASA2; default: diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index 6a4bc7fe9e..f81492869c 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -41,6 +41,7 @@ #include "ivas_cnst.h" #include "ivas_rom_com.h" #include "ivas_rom_dec.h" +#include "ivas_rom_rend.h" #ifdef DEBUGGING #include "debug.h" #endif @@ -841,7 +842,7 @@ ivas_error ivas_dirac_dec_config( { common_rend_config_flag = st_ivas->hSpatParamRendCom == NULL ? DIRAC_OPEN : flag_config_inp; if ( ( error = ivas_spat_hSpatParamRendCom_config( &st_ivas->hSpatParamRendCom, common_rend_config_flag, dec_param_estim_new, - st_ivas->ivas_format, st_ivas->mc_mode, output_Fs, hodirac_flag ) ) != IVAS_ERR_OK ) + st_ivas->ivas_format, st_ivas->mc_mode, output_Fs, hodirac_flag, 0 ) ) != IVAS_ERR_OK ) { return error; } @@ -1928,7 +1929,7 @@ void ivas_dirac_dec_render_sf( ivas_dirac_dec_compute_directional_responses( hSpatParamRendCom, hDirACRend, st_ivas->hVBAPdata, - st_ivas->hMasa, + st_ivas->hMasa == NULL ? NULL : st_ivas->hMasa->data.band_mapping, st_ivas->hMasaIsmData, azimuth, elevation, @@ -1943,7 +1944,7 @@ void ivas_dirac_dec_render_sf( ivas_dirac_dec_compute_directional_responses( hSpatParamRendCom, hDirACRend, st_ivas->hVBAPdata, - st_ivas->hMasa, + st_ivas->hMasa == NULL ? NULL : st_ivas->hMasa->data.band_mapping, st_ivas->hMasaIsmData, azimuth, elevation, diff --git a/lib_dec/ivas_rom_dec.c b/lib_dec/ivas_rom_dec.c index 1c04fb93ef..8f0f1cca60 100644 --- a/lib_dec/ivas_rom_dec.c +++ b/lib_dec/ivas_rom_dec.c @@ -379,17 +379,6 @@ const uint16_t * const sym_freq_ECSQ_tab_abs_lsbs[1 + 4] = * DirAC ROM tables *----------------------------------------------------------------------------------*/ -const float ls_azimuth_4d4[8] = { 45.0f, -45.0f, 135.0f, -135.0f, 45.0f, -45.0f, 135.0f, -135.0f }; - -const float ls_elevation_4d4[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 35.0f, 35.0f, 35.0f, 35.0f }; - -const float diffuse_response_CICP6[5] = { 0.4219f, 0.4219f, 0.3704f, 0.5034f, 0.5034f }; - -const float diffuse_response_CICP14[7] = { 0.3817f, 0.3817f, 0.2819f, 0.5399f, 0.5399f, 0.1519f, 0.1519f }; - -const float diffuse_response_CICP16[9] = { 0.3456f, 0.3456f, 0.3035f, 0.4124f, 0.4124f, 0.2702f, 0.2702f, 0.3023f, 0.3023f }; - - const float dirac_dithering_azi_scale[DIRAC_DIFFUSE_LEVELS] = { 6.716062e-01f, 1.011837e+00f, 1.799065e+00f, 2.824915e+00f, 4.800879e+00f, 9.206031e+00f, 1.469832e+01f, 2.566224e+01f @@ -400,135 +389,6 @@ const float dirac_dithering_ele_scale[DIRAC_DIFFUSE_LEVELS] = 6.716062e-01f, 1.011804e+00f, 1.796875e+00f, 2.804382e+00f, 4.623130e+00f, 7.802667e+00f, 1.045446e+01f, 1.379538e+01f }; -const int16_t ap_pre_delay[DIRAC_DECORR_NUM_SPLIT_BANDS] = { 7, 2, 1 }; - -const int16_t ap_filter_length[DIRAC_DECORR_NUM_SPLIT_BANDS] = { 15, 6, 3 }; - -const float ap_lattice_delta_phi[DIRAC_MAX_NUM_DECORR_FILTERS*DIRAC_MAX_DECORR_FILTER_LEN] = -{ - 1.802519f, 0.922986f, 1.813685f, 1.272828f, 0.856928f, 0.366571f, 1.531249f, 1.318158f, 0.123812f, 0.897173f, 0.958696f, 1.256384f, 0.179677f, 0.668918f, 1.440292f, 1.573058f, 1.396481f, 1.191463f, 0.444143f, 1.666942f, - 1.273955f, 1.747171f, 1.408330f, 1.002782f, 1.559302f, 1.782992f, 1.474896f, 0.813181f, 1.457724f, 0.588531f, 1.384302f, 0.156493f, 0.600048f, 1.661632f, 0.538958f, 0.645429f, 0.565237f, 0.024684f, 0.264229f, 0.062140f, - 1.235343f, 0.851725f, 1.820211f, 0.116148f, 0.972111f, 0.488703f, 1.777672f, 1.452170f, 0.814134f, 1.272649f, 1.281416f, 0.101871f, 0.897888f, 0.199760f, 0.085732f, 1.686579f, 0.964558f, 0.057281f, 0.910252f, 1.662302f, - 0.955234f, 0.834348f, 1.672478f, 1.324896f, 0.444544f, 1.721172f, 0.153356f, 1.602240f, 0.171880f, 1.169774f, 0.543628f, 1.409581f, 1.763724f, 1.686754f, 1.210390f, 0.402691f, 0.983618f, 0.862997f, 1.220409f, 0.890061f, - 0.031641f, 0.461590f, 1.719550f, 1.357698f, 1.112262f, 1.166531f, 0.246097f, 1.387325f, 0.177485f, 1.446268f, 0.799476f, 1.667227f, 1.723465f, 1.505920f, 0.245874f, 1.155854f, 0.831394f, 0.677194f, 0.568871f, 1.652070f, - 0.019803f, 1.197794f, 0.635553f, 0.531682f, 0.878194f, 0.048050f, 0.080480f, 1.566743f, 0.724210f, 0.853668f, 1.741191f, 0.698465f, 1.553550f, 0.130290f, 0.688346f, 1.331091f, 0.599759f, 1.125466f, 1.764818f, 1.042879f, - 1.486589f, 1.627971f, 1.871181f, 0.102359f, 0.035021f, 1.403176f, 1.468675f, 0.190347f, 0.553282f, 1.031227f, 1.232390f, 1.255724f, 1.504443f, 0.683526f, 0.600958f, 1.746936f, 1.529243f, 1.448196f, 0.646850f, 0.116053f, - 1.283295f, 0.355220f, 1.380620f, 1.858453f, 0.818804f, 0.219006f, 0.476292f, 0.420029f, 1.291187f, 0.568738f, 1.174088f, 0.628805f, 1.753154f, 1.459582f, 1.354449f, 1.755790f, 0.441757f, 0.856240f, 1.647962f, 0.686353f, - 1.395289f, 0.699934f, 0.239310f, 0.239535f, 0.958190f, 0.748780f, 0.513784f, 1.666344f, 1.461995f, 1.599060f, 0.893107f, 0.341873f, 1.387703f, 1.808363f, 0.676542f, 1.424958f, 0.310574f, 0.836247f, 1.011101f, 1.686200f, - 1.823432f, 0.994827f, 1.635555f, 0.684380f, 1.017029f, 1.440371f, 1.694641f, 0.607132f, 1.197331f, 0.862161f, 0.666449f, 1.047956f, 0.159627f, 0.043131f, 1.251515f, 1.618724f, 0.216906f, 0.152250f, 0.471610f, 0.744260f, - 0.576810f, 1.632177f, 1.556912f, 1.866317f, 0.568088f, 1.541817f, 1.726725f, 0.275154f, 0.814958f, 0.863399f, 1.333040f, 0.148277f, 0.197893f, 1.048665f, 1.158090f, 1.692225f, 0.884294f, 0.289619f, 0.380633f, 1.728234f, - 1.433213f, 1.749505f, 1.533837f, 0.669701f, 0.372580f, 1.052390f, 1.116645f, 0.181320f, 1.139126f, 0.222671f, 0.604393f, 1.811797f, 1.743315f, 1.368792f, 1.861434f, 0.751908f, 0.159811f, 1.566503f, 0.443273f, 1.667530f, - 1.083060f, 1.243136f, 0.717777f, 0.675019f, 0.690490f, 0.672228f, 1.060789f, 0.423566f, 1.198457f, 0.485768f, 0.993953f, 0.443540f, 0.361702f, 1.552042f, 0.863562f, 1.517677f, 1.061899f, 0.691413f, 1.642818f, 1.756590f, - 0.278323f, 0.790363f, 0.172303f, 0.417138f, 0.009343f, 0.783325f, 1.369303f, 1.041067f, 0.467102f, 0.992773f, 1.525170f, 0.871213f, 0.243906f, 1.542036f, 0.449148f, 0.843633f, 0.191800f, 1.614246f, 1.038188f, 1.415620f, - 0.551081f, 0.382599f, 1.410121f, 0.102084f, 0.137286f, 0.671081f, 0.254860f, 1.758068f, 1.079013f, 0.129143f, 1.410873f, 0.150485f, 0.601119f, 0.760737f, 0.975905f, 0.223261f, 0.710162f, 1.677048f, 0.996836f, 1.849865f, - 1.536222f, 0.089016f, 0.960881f, 0.388690f, 0.379955f, 1.002223f, 1.271420f, 1.410632f, 0.254397f, 1.535559f, 1.133703f, 1.305280f, 1.466565f, 0.274167f, 0.399688f, 1.359638f, 1.766289f, 1.401348f, 1.310883f, 0.261030f, - 1.314825f, 1.538635f, 1.317986f, 1.243167f, 1.749461f, 1.689706f, 0.024853f, 0.634754f, 1.036317f, 1.828101f, 1.676951f, 0.023606f, 0.857000f, 0.076471f, 1.622198f, 0.254469f, 1.451625f, 1.720881f, 0.763812f, 0.186982f, - 0.056994f, 0.590507f, 0.375291f, 1.609261f, 0.607721f, 0.026355f, 0.483366f, 0.823931f, 0.792878f, 0.163577f, 0.753588f, 0.730789f, 0.135991f, 1.031660f, 1.554135f, 1.192863f, 0.016693f, 0.125796f, 1.017920f, 1.591773f, - 0.575956f, 0.112943f, 0.249506f, 1.399570f, 0.053241f, 1.410759f, 0.251638f, 1.059086f, 0.025315f, 1.422914f, 1.030412f, 0.848758f, 0.317396f, 1.375456f, 1.116858f, 1.682310f, 0.279550f, 0.325974f, 0.937704f, 1.744329f, - 0.447773f, 1.024286f, 1.001528f, 1.863684f, 1.278323f, 0.860699f, 1.346331f, 1.692596f, 0.022627f, 1.033613f, 0.546354f, 0.395804f, 1.486546f, 1.381045f, 1.312260f, 0.245976f, 1.607429f, 1.818793f, 0.964359f, 1.496598f, - 0.669967f, 1.535929f, 1.841878f, 0.979127f, 0.614002f, 1.879218f, 0.512531f, 1.167061f, 0.081697f, 1.773427f, 1.535668f, 0.757729f, 0.220395f, 1.538243f, 1.281162f, 0.302159f, 0.889871f, 0.798522f, 1.476288f, 1.665941f, - 0.915365f, 1.394094f, 0.757041f, 0.350064f, 1.199679f, 1.319499f, 1.128405f, 0.632337f, 0.790673f, 0.461582f, 1.693343f, 1.537442f, 0.346527f, 0.433782f, 1.754552f, 0.550903f, 0.686724f, 0.764433f, 1.792750f, 1.489998f -}; - -const float ap_lattice_coeffs_1[DIRAC_DECORR_FILTER_LEN_1*DIRAC_MAX_NUM_DECORR_FILTERS] = -{ - 0.795329f, 0.502700f, 0.204456f, 0.416566f, 0.459648f, 0.270454f, -0.201944f, 0.027997f, 0.067811f, -0.052627f, -0.038779f, -0.057387f, 0.020480f, 0.367697f, -0.593705f, - 0.533667f, 0.202500f, -0.001953f, 0.195081f, -0.184458f, -0.233867f, 0.228100f, -0.329293f, -0.338714f, -0.079700f, 0.052389f, -0.009468f, 0.178807f, 0.190843f, -0.478877f, - 0.044022f, 0.788498f, 0.133529f, -0.173657f, 0.545391f, 0.681431f, 0.332868f, 0.294685f, 0.325261f, 0.047617f, 0.157401f, 0.116272f, 0.218980f, -0.189248f, -0.317401f, - -0.753681f, 0.772385f, -0.507384f, 0.276980f, -0.692775f, 0.232302f, -0.354759f, -0.015620f, -0.223059f, 0.310506f, -0.035034f, 0.085783f, -0.499998f, 0.215260f, 0.201415f, - -0.495551f, -0.301660f, 0.196510f, 0.326147f, -0.594364f, 0.314921f, 0.668671f, 0.076643f, 0.045711f, -0.124790f, -0.203272f, -0.297190f, 0.125806f, -0.179483f, -0.201757f, - 0.161128f, 0.396050f, 0.266897f, 0.356586f, 0.488145f, -0.056254f, 0.139280f, -0.296405f, -0.112844f, 0.037405f, -0.367425f, -0.216292f, -0.277360f, 0.389420f, 0.115115f, - -0.240449f, -0.271015f, 0.426720f, -0.011059f, 0.151813f, 0.253490f, 0.225764f, 0.498716f, -0.136377f, 0.443004f, -0.305017f, -0.031310f, -0.010765f, 0.170349f, 0.496478f, - -0.139875f, -0.241998f, -0.104850f, 0.294343f, -0.067728f, -0.492202f, -0.487610f, 0.036395f, 0.109393f, 0.396155f, -0.352845f, -0.205913f, -0.082999f, -0.463033f, -0.309296f, - 0.025886f, -0.092456f, -0.125139f, -0.156117f, -0.004887f, 0.178440f, 0.089586f, 0.044827f, 0.238219f, -0.312120f, -0.390688f, -0.178543f, 0.454418f, 0.387012f, -0.388874f, - -0.197797f, 0.035540f, 0.455388f, -0.054410f, 0.380035f, 0.290964f, 0.048804f, 0.078637f, 0.221740f, -0.217548f, 0.121289f, -0.396681f, -0.218482f, -0.127265f, -0.269507f, - -0.344218f, -0.465038f, -0.421415f, -0.026031f, 0.221547f, 0.361993f, -0.348243f, 0.294983f, 0.366175f, 0.070663f, -0.086050f, 0.252129f, 0.156066f, -0.062800f, 0.408972f, - 0.242461f, -0.301764f, -0.066160f, 0.388651f, -0.462227f, -0.158880f, 0.230796f, -0.093179f, 0.047076f, 0.073402f, -0.335018f, 0.022940f, 0.354611f, 0.072391f, 0.019473f, - 0.336900f, -0.480534f, 0.170267f, 0.259663f, -0.393576f, -0.348588f, -0.108962f, 0.278842f, 0.385490f, 0.362249f, -0.318739f, 0.362305f, 0.288936f, 0.291204f, -0.278019f, - -0.449302f, -0.288513f, -0.413973f, -0.405279f, -0.295152f, -0.245110f, 0.002530f, 0.287890f, 0.348229f, -0.178354f, -0.206517f, 0.351081f, -0.482205f, 0.021360f, -0.492207f, - 0.048642f, 0.453282f, 0.109160f, 0.232599f, -0.473781f, 0.140502f, 0.352527f, -0.098606f, 0.147172f, -0.055797f, 0.107739f, -0.231026f, 0.357310f, 0.348031f, 0.232404f, - -0.324788f, 0.162480f, 0.057647f, -0.060734f, -0.009742f, -0.224185f, -0.282355f, -0.065443f, 0.064697f, 0.280370f, -0.284906f, -0.470501f, 0.019484f, -0.442308f, 0.377214f, - 0.447384f, -0.290498f, 0.345528f, 0.370207f, -0.313120f, 0.119592f, 0.300014f, 0.406995f, -0.277922f, 0.447039f, 0.194824f, 0.157703f, -0.223402f, -0.147167f, 0.379073f, - -0.216342f, 0.137967f, -0.397180f, 0.073905f, -0.273110f, -0.443037f, -0.168327f, 0.346264f, 0.037543f, 0.065387f, 0.163901f, -0.122523f, 0.365477f, -0.316321f, 0.117273f, - -0.301282f, 0.169625f, -0.336466f, 0.269914f, -0.420160f, -0.331296f, 0.498523f, -0.393009f, -0.462184f, -0.323097f, 0.470977f, -0.359463f, 0.264315f, 0.216797f, 0.493400f, - 0.151489f, -0.321653f, 0.464413f, -0.355673f, 0.420401f, 0.184297f, -0.302128f, 0.136536f, -0.252849f, 0.193906f, 0.298775f, 0.238808f, -0.386298f, -0.181999f, -0.077326f, - 0.124840f, -0.263394f, 0.389606f, 0.004502f, 0.339804f, 0.415204f, 0.377751f, 0.400221f, 0.352426f, -0.003808f, -0.184530f, 0.433348f, -0.237554f, 0.147684f, 0.407210f, - 0.067616f, -0.249313f, -0.354010f, 0.320937f, 0.426000f, 0.309576f, -0.189689f, -0.209447f, 0.158967f, -0.081929f, 0.386828f, 0.178582f, -0.407143f, 0.222189f, -0.149097f -}; - -const float ap_lattice_coeffs_2[DIRAC_DECORR_FILTER_LEN_2*DIRAC_MAX_NUM_DECORR_FILTERS] = -{ - 0.633692f, 0.681207f, -0.049418f, 0.286715f, 0.146022f, 0.135402f, - -0.410145f, -0.206766f, -0.656968f, -0.101746f, 0.436299f, 0.339818f, - -0.131383f, -0.773746f, -0.301627f, 0.327561f, 0.332227f, 0.205858f, - 0.717602f, -0.552370f, -0.150136f, 0.054556f, 0.239519f, -0.648477f, - -0.721848f, 0.189377f, 0.068185f, 0.006216f, 0.077025f, 0.038678f, - 0.423415f, -0.108658f, 0.432050f, -0.414641f, 0.277840f, 0.418486f, - 0.077811f, -0.283063f, -0.357008f, 0.004634f, -0.442640f, -0.372315f, - 0.403336f, 0.243023f, 0.314367f, 0.444513f, 0.347517f, 0.496043f, - 0.108408f, 0.469382f, -0.170286f, 0.326310f, 0.172157f, 0.440334f, - 0.058417f, 0.339673f, -0.194965f, 0.491219f, -0.281296f, -0.043120f, - -0.482487f, -0.335005f, -0.336159f, 0.196469f, -0.164623f, 0.442491f, - -0.135190f, -0.354385f, 0.452133f, -0.311221f, -0.347640f, 0.498342f, - -0.288999f, 0.376431f, -0.177924f, 0.195542f, 0.333872f, -0.152697f, - 0.053230f, 0.457767f, 0.442476f, 0.235254f, -0.345159f, -0.286098f, - 0.155438f, 0.405393f, 0.217073f, -0.107960f, 0.010698f, 0.368987f, - -0.498358f, -0.495816f, -0.215479f, -0.093869f, 0.320276f, -0.013842f, - 0.489406f, 0.470814f, -0.065834f, 0.346871f, 0.027279f, 0.150086f, - 0.484013f, -0.497391f, 0.168796f, -0.493841f, -0.173528f, 0.334676f, - 0.264235f, -0.424651f, -0.314926f, 0.253086f, 0.397381f, -0.491565f, - -0.453727f, -0.463358f, -0.019128f, 0.000344f, 0.315432f, 0.472345f, - 0.095139f, 0.283375f, -0.225088f, -0.119762f, -0.476871f, 0.037525f, - 0.336951f, 0.494511f, -0.062603f, 0.177652f, 0.463892f, 0.489286f -}; - -const float ap_lattice_coeffs_3[DIRAC_DECORR_FILTER_LEN_3*DIRAC_MAX_NUM_DECORR_FILTERS] = -{ - 0.018977f, -0.212205f, 0.422719f, - -0.400657f, -0.106890f, -0.024589f, - 0.140005f, 0.279582f, 0.032357f, - 0.632535f, 0.578535f, -0.734606f, - 0.017182f, 0.013244f, -0.027715f, - -0.353356f, -0.482160f, -0.491265f, - 0.457024f, 0.165122f, 0.469723f, - -0.195705f, 0.440105f, -0.477366f, - 0.360186f, -0.490565f, 0.484623f, - -0.173791f, 0.007543f, 0.278186f, - 0.434416f, 0.060363f, -0.193717f, - -0.033709f, 0.496222f, 0.002939f, - -0.480848f, -0.109552f, -0.023198f, - 0.324679f, -0.292075f, -0.356148f, - -0.366595f, 0.380917f, -0.301741f, - 0.110318f, 0.383789f, 0.303984f, - -0.499685f, -0.349584f, 0.334749f, - -0.020224f, -0.430078f, -0.154705f, - -0.371129f, 0.334080f, 0.346913f, - -0.166781f, -0.229089f, 0.117956f, - 0.341292f, 0.490463f, 0.493655f, - -0.367726f, 0.426528f, -0.045774f -}; - -const float * const ap_lattice_coeffs[DIRAC_DECORR_NUM_SPLIT_BANDS] = -{ - &ap_lattice_coeffs_1[0], - &ap_lattice_coeffs_2[0], - &ap_lattice_coeffs_3[0], -}; - -const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1] = -{ - 0.0f, 0.125f, 0.375f, 1.0f -}; - -const int16_t sba_map_tc[11] = -{ - 0, 1, 2, 3, 4, 8, 9, 15, 5, 6, 7 -}; -const int16_t sba_map_tc_512[11] = -{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15 -}; - /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables diff --git a/lib_dec/ivas_rom_dec.h b/lib_dec/ivas_rom_dec.h index 2ab0e12690..993f6abb0f 100644 --- a/lib_dec/ivas_rom_dec.h +++ b/lib_dec/ivas_rom_dec.h @@ -92,35 +92,15 @@ extern const uint16_t *const sym_freq_ECSQ_tab_abs_lsbs[1 + 4]; * DirAC ROM tables *----------------------------------------------------------------------------------*/ -extern const float ls_azimuth_4d4[8]; -extern const float ls_elevation_4d4[8]; -extern const float diffuse_response_CICP6[5]; -extern const float diffuse_response_CICP14[7]; -extern const float diffuse_response_CICP16[9]; - - extern const float dirac_dithering_azi_scale[DIRAC_DIFFUSE_LEVELS]; extern const float dirac_dithering_ele_scale[DIRAC_DIFFUSE_LEVELS]; -extern const int16_t ap_pre_delay[DIRAC_DECORR_NUM_SPLIT_BANDS]; -extern const int16_t ap_filter_length[DIRAC_DECORR_NUM_SPLIT_BANDS]; -extern const float ap_lattice_delta_phi[DIRAC_MAX_NUM_DECORR_FILTERS * DIRAC_MAX_DECORR_FILTER_LEN]; -extern const float ap_lattice_coeffs_1[DIRAC_DECORR_FILTER_LEN_1 * DIRAC_MAX_NUM_DECORR_FILTERS]; -extern const float ap_lattice_coeffs_2[DIRAC_DECORR_FILTER_LEN_2 * DIRAC_MAX_NUM_DECORR_FILTERS]; -extern const float ap_lattice_coeffs_3[DIRAC_DECORR_FILTER_LEN_3 * DIRAC_MAX_NUM_DECORR_FILTERS]; -extern const float *const ap_lattice_coeffs[DIRAC_DECORR_NUM_SPLIT_BANDS]; -extern const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1]; - -extern const int16_t sba_map_tc[11]; -extern const int16_t sba_map_tc_512[11]; - /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables *----------------------------------------------------------------------------------*/ extern const float dmxmtx_table[BINAURAL_CHANNELS][11]; - #ifdef SPLIT_REND_WITH_HEAD_ROT /*----------------------------------------------------------------------* * Binuaral split rendering ROM tables diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 55299076d5..b80467ace5 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -3445,3 +3445,286 @@ void ivas_omasa_preProcessStereoTransportsForMovedObjects( return; } + + +static void ivas_masa_ext_rend_parambin_internal( + MASA_EXT_REND_HANDLE hMasaExtRend, + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, + float *output_f[], + const int16_t subframe ) +{ + DIRAC_DEC_BIN_HANDLE hDiracDecBin; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + PARAMBIN_REND_CONFIG config_data; + int16_t slot, ch, numInChannels; + float Cldfb_RealBuffer_in[6][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_in[6][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float Rmat[3][3]; + int16_t max_band_decorr; + int16_t nBins; + int16_t i, j; + int16_t nchan_transport; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t pos_idx; + MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + float tmp_Cldfb_out_re[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float tmp_Cldfb_out_im[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + /* these allow re-using the reverb and freq-domain decorrelator signals from ivas_dirac_dec_binaural_process_output() in split rendering for the side renderings */ + float reverbRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float decorrRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float decorrIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float subFrameTotalEne[CLDFB_NO_CHANNELS_MAX]; + float IIReneLimiter[CLDFB_NO_CHANNELS_MAX]; + + hDiracDecBin = hMasaExtRend->hDiracDecBin[0]; +#else + hDiracDecBin = hMasaExtRend->hDiracDecBin; +#endif + assert( hDiracDecBin ); + hSpatParamRendCom = hMasaExtRend->hSpatParamRendCom; + nBins = hSpatParamRendCom->num_freq_bands; + + /* Setup internal config. MASA EXT renderer is quite strict. */ + config_data.separateCenterChannelRendering = 0; + config_data.ivas_format = MASA_FORMAT; + config_data.mc_mode = MC_MODE_NONE; + config_data.ivas_total_brate = IVAS_512k; /* Maximum bitrate set for external renderer */ + config_data.nchan_transport = hMasaExtRend->nchan_input; + config_data.qualityBasedSmFactor = 1.0f; + config_data.processReverb = hMasaExtRend->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ? 1 : 0; + config_data.ism_mode = ISM_MODE_NONE; + + /* Set nchan_transport to number of transport channels in MASA input */ + nchan_transport = hMasaExtRend->nchan_input; + + /* The input channel number at this processing function (not nchan_transport) */ + numInChannels = BINAURAL_CHANNELS; + + Rmat[0][0] = 1.0f; + Rmat[0][1] = 0.0f; + Rmat[0][2] = 0.0f; + + Rmat[1][0] = 0.0f; + Rmat[1][1] = 1.0f; + Rmat[1][2] = 0.0f; + + Rmat[2][0] = 0.0f; + Rmat[2][1] = 0.0f; + Rmat[2][2] = 1.0f; + + /* CLDFB Analysis of input */ + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + for ( ch = 0; ch < numInChannels; ch++ ) + { + if ( ch == 0 || nchan_transport == 2 ) + { + cldfbAnalysis_ts( + &( output_f[ch][nBins * slot] ), + Cldfb_RealBuffer_in[ch][slot], + Cldfb_ImagBuffer_in[ch][slot], + nBins, hMasaExtRend->cldfbAnaRend[ch] ); + } + else /* when nchan_transport == 1 and ch == 1 */ + { + /* At mono input duplicate the channel to dual-mono, and apply gain + correction to ensure same overall level as in stereo mode */ + v_multc( Cldfb_RealBuffer_in[0][slot], INV_SQRT_2, Cldfb_RealBuffer_in[0][slot], nBins ); + v_multc( Cldfb_ImagBuffer_in[0][slot], INV_SQRT_2, Cldfb_ImagBuffer_in[0][slot], nBins ); + + mvr2r( Cldfb_RealBuffer_in[0][slot], Cldfb_RealBuffer_in[1][slot], nBins ); + mvr2r( Cldfb_ImagBuffer_in[0][slot], Cldfb_ImagBuffer_in[1][slot], nBins ); + } + } + } + + if ( hCombinedOrientationData ) + { + for ( i = 0; i < 3; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + Rmat[i][j] = hCombinedOrientationData->Rmat[subframe][i][j]; + } + } + + if ( nchan_transport == 2 ) + { +#ifdef SPLIT_REND_WITH_HEAD_ROT + /* in case of split rendering, determine the prototype rotation based on the main direction and use the same prototypes for the offset directions */ +#endif + adaptTransportSignalsHeadtracked( hCombinedOrientationData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, hSpatParamRendCom->subframe_nbslots[subframe], Rmat ); + + ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( hCombinedOrientationData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, hSpatParamRendCom->subframe_nbslots[subframe], Rmat ); + } + } + + +#ifndef SPLIT_REND_WITH_HEAD_ROT + ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, NULL ); +#endif + + /* Always using CLDFB decorrelation in MASA EXT renderer */ + max_band_decorr = hDiracDecBin->h_freq_domain_decorr_ap_params->max_band_decorr; + + +#ifdef SPLIT_REND_WITH_HEAD_ROT + ivas_dirac_dec_binaural_formulate_input_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe, + subFrameTotalEne, IIReneLimiter ); + ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter, NULL ); +#endif + + ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + 0, NULL ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + // pMultiBinPoseData = &st_ivas->hSplitBinRend.splitrend.multiBinPoseData; // Todo split rend MASA ext: Need to get split rend structure here + +#ifdef FIX_658_SPLIT_REND_MASA + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) +#else + if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) +#endif + { + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, hMasaExtRend->cldfbSynRend, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, + reverbRe, reverbIm, decorrRe, decorrIm, 1 ); + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) + { + // Todo split rend MASA ext: Need to get split rend CLDFB output structure here + // mvr2r( tmp_Cldfb_out_re[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + // mvr2r( tmp_Cldfb_out_im[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + } + } + } + else + { + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, hMasaExtRend->cldfbSynRend, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, NULL, NULL, + reverbRe, reverbIm, decorrRe, decorrIm, 1 ); + } +#else + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, hMasaExtRend->cldfbSynRend, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, max_band_decorr, numInChannels, config_data.processReverb, subframe ); +#endif + + hDiracDecBin->hDiffuseDist = NULL; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) + { + /* quaternion-based rotation from ivas_binRenderer_internal.c:ivas_binRenderer(), but using absolute rotation instead of delta rotations */ + IVAS_QUATERNION Quaternions_rot, Quaternions_abs, *Quaternions_ref; + float Rmat_local[3][3]; + + if ( hCombinedOrientationData ) + { + Quaternions_ref = &hCombinedOrientationData->Quaternions[0]; + Quaternions_rot.w = -3.0f; /* signal to use Euler */ + Quaternions_abs.w = -3.0f; /* signal to use Euler */ + Quat2EulerDegree( *Quaternions_ref, &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + for ( pos_idx = 1; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + { + Quaternions_rot.x = Quaternions_abs.x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternions_rot.y = Quaternions_abs.y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternions_rot.z = Quaternions_abs.z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; + + QuatToRotMat( Quaternions_rot, Rmat_local ); + + hDiracDecBin = hMasaExtRend->hDiracDecBin[pos_idx]; + assert( hDiracDecBin != NULL && "No DiracDecBin handle for this position" ); + + /* re-use input covariance for the side renderings */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + mvr2r( hMasaExtRend->hDiracDecBin[0]->ChEne[ch], hDiracDecBin->ChEne[ch], hSpatParamRendCom->num_freq_bands ); + } + mvr2r( hMasaExtRend->hDiracDecBin[0]->ChCrossRe, hDiracDecBin->ChCrossRe, hSpatParamRendCom->num_freq_bands ); + mvr2r( hMasaExtRend->hDiracDecBin[0]->ChCrossIm, hDiracDecBin->ChCrossIm, hSpatParamRendCom->num_freq_bands ); + + ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat_local, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + subFrameTotalEne, IIReneLimiter, NULL ); + + ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, subframe, + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, + nchanSeparateChannels, NULL ); + + + /* re-use reverb and decorr from main direction for the sides */ + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, hMasaExtRend->cldfbSynRend, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, + max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, + reverbRe, reverbIm, decorrRe, decorrIm, 0 ); + + /* copy from temporary buffer to the main split rendering buffer */ + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) + { + // Todo split rend MASA ext: Need to get split rend structure here + // mvr2r( tmp_Cldfb_out_re[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + // mvr2r( tmp_Cldfb_out_im[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + } + } + + hDiracDecBin->hDiffuseDist = NULL; + } + } + } + + /* update this counter only after the last rendering of split directions */ +#endif + hSpatParamRendCom->slots_rendered += hSpatParamRendCom->subframe_nbslots[subframe]; + hSpatParamRendCom->subframes_rendered++; + + return; +} + + +// Todo split rend MASA ext: If needed, make a copy of this function for split rendering purposes +void ivas_masa_ext_rend_parambin_render( + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ + float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ + const int16_t num_subframes /* i : number of subframes to render */ +) +{ + int16_t subframe; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + float *p_output[BINAURAL_CHANNELS]; + int16_t ch; + + hSpatParamRendCom = hMasaExtRend->hSpatParamRendCom; + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + p_output[ch] = output_f[ch]; + } + + hSpatParamRendCom->subframes_rendered = hSpatParamRendCom->dirac_read_idx; + + for ( subframe = 0; subframe < num_subframes; subframe++ ) + { + int16_t n_samples_sf = hSpatParamRendCom->slot_size * CLDFB_SLOTS_PER_SUBFRAME; + hSpatParamRendCom->slots_rendered = 0; + + ivas_masa_ext_rend_parambin_internal( hMasaExtRend, hCombinedOrientationData, p_output, hSpatParamRendCom->dirac_read_idx ); + + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + p_output[ch] += n_samples_sf; + } + hSpatParamRendCom->dirac_read_idx = ( hSpatParamRendCom->dirac_read_idx + 1 ) % hSpatParamRendCom->dirac_md_buffer_length; + } + + return; +} diff --git a/lib_rend/ivas_dirac_decorr_dec.c b/lib_rend/ivas_dirac_decorr_dec.c index e831188881..89c2c4a086 100644 --- a/lib_rend/ivas_dirac_decorr_dec.c +++ b/lib_rend/ivas_dirac_decorr_dec.c @@ -38,9 +38,8 @@ #include "prot.h" #include "ivas_prot.h" #include "ivas_prot_rend.h" -#include "ivas_stat_dec.h" #include "ivas_cnst.h" -#include "ivas_rom_dec.h" +#include "ivas_rom_rend.h" #ifdef DEBUGGING #include "debug.h" #endif diff --git a/lib_rend/ivas_dirac_output_synthesis_dec.c b/lib_rend/ivas_dirac_output_synthesis_dec.c index f10b7b2c22..8ee3b1e4f5 100644 --- a/lib_rend/ivas_dirac_output_synthesis_dec.c +++ b/lib_rend/ivas_dirac_output_synthesis_dec.c @@ -1741,11 +1741,11 @@ static void ivas_dirac_dec_get_response_split_order( *------------------------------------------------------------------------*/ void ivas_dirac_dec_compute_directional_responses( - SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ - DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ - const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ - const MASA_DECODER_HANDLE hMasa, /* i : MASA decoder structure */ - MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ + DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ + const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ + const int16_t *masa_band_mapping, /* i : Band mapping for MASA, NULL assumes not using MASA in any form */ + MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ const int16_t *azimuth, const int16_t *elevation, const int16_t md_idx, @@ -1793,13 +1793,13 @@ void ivas_dirac_dec_compute_directional_responses( for ( k = 0; k < hSpatParamRendCom->num_freq_bands; ++k ) { - if ( hMasa != NULL && k == MASA_band_grouping_24[hMasa->data.band_mapping[codingBand + 1]] ) + if ( masa_band_mapping != NULL && k == MASA_band_grouping_24[masa_band_mapping[codingBand + 1]] ) { codingBand++; } - if ( hMasa != NULL && k > MASA_band_grouping_24[hMasa->data.band_mapping[codingBand]] && - k < MASA_band_grouping_24[hMasa->data.band_mapping[codingBand + 1]] && + if ( masa_band_mapping != NULL && k > MASA_band_grouping_24[masa_band_mapping[codingBand]] && + k < MASA_band_grouping_24[masa_band_mapping[codingBand + 1]] && k != hDirACRend->h_output_synthesis_psd_params.max_band_decorr ) { /* Panning gains have to be computed only for the first bin of the coding band in MASA, for other bins the previous values can be used */ @@ -1841,7 +1841,7 @@ void ivas_dirac_dec_compute_directional_responses( } } - if ( hMasa == NULL && hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) + if ( masa_band_mapping == NULL && hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) { mvr2r_inc( direct_response_hoa, 1, &hDirACRend->h_output_synthesis_psd_state.direct_responses[k], hSpatParamRendCom->num_freq_bands, num_channels_dir ); @@ -1850,7 +1850,7 @@ void ivas_dirac_dec_compute_directional_responses( mvr2r_inc( direct_response_dir2, 1, &hDirACRend->h_output_synthesis_psd_state.direct_responses[k + hSpatParamRendCom->num_freq_bands * num_channels_dir], hSpatParamRendCom->num_freq_bands, num_channels_dir ); } } - else if ( ( ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) && ( hMasa != NULL ) ) || + else if ( ( ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) && ( masa_band_mapping != NULL ) ) || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) { /* Synthesize the first direction */ diff --git a/lib_rend/ivas_dirac_rend.c b/lib_rend/ivas_dirac_rend.c index 148f11e82c..b3cf98df4a 100644 --- a/lib_rend/ivas_dirac_rend.c +++ b/lib_rend/ivas_dirac_rend.c @@ -39,7 +39,7 @@ #include "ivas_prot.h" #include "ivas_prot_rend.h" #include "ivas_cnst.h" -#include "ivas_rom_dec.h" +#include "ivas_rom_rend.h" #ifdef DEBUGGING #include "debug.h" #endif @@ -197,7 +197,8 @@ ivas_error ivas_spat_hSpatParamRendCom_config( const IVAS_FORMAT ivas_format, const MC_MODE mc_mode, const int32_t output_Fs, - const int16_t hodirac_flag ) + const int16_t hodirac_flag, + const int16_t masa_ext_rend_flag ) { ivas_error error; int16_t map_idx; @@ -266,7 +267,7 @@ ivas_error ivas_spat_hSpatParamRendCom_config( hSpatParamRendCom->dirac_md_buffer_length = 0; hSpatParamRendCom->dirac_bs_md_write_idx = 0; hSpatParamRendCom->dirac_read_idx = 0; - if ( mc_mode == MC_MODE_MCMASA ) + if ( mc_mode == MC_MODE_MCMASA || masa_ext_rend_flag == 1 ) { hSpatParamRendCom->dirac_md_buffer_length = MAX_PARAM_SPATIAL_SUBFRAMES; @@ -2084,3 +2085,417 @@ void rotateAziEle_DirAC( return; } + + +/* A reduced rewrite of the corresponding decoder side function */ +static void ivas_masa_ext_dirac_render_sf( + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: IVAS decoder structure */ + float *output_f[] /* i/o: synthesized core-coder transport channels/DirAC output */ +) +{ + int16_t i, ch, idx_in, idx_lfe; + DIRAC_REND_HANDLE hDirACRend; + float dirEne; + float surCohEner; + float surCohRatio[CLDFB_NO_CHANNELS_MAX]; + int16_t subframe_idx; + int16_t slot_idx, index_slot; + int16_t slot_idx_start, slot_idx_start_cldfb_synth, md_idx; + int16_t nchan_transport; + int16_t masa_band_mapping[MASA_FREQUENCY_BANDS + 1]; + + /*CLDFB: last output channels reserved to LFT for CICPx*/ + float Cldfb_RealBuffer[MAX_OUTPUT_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer[MAX_OUTPUT_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + + /* local copies of azi, ele, diffuseness */ + int16_t azimuth[CLDFB_NO_CHANNELS_MAX]; + int16_t elevation[CLDFB_NO_CHANNELS_MAX]; + float diffuseness_vector[CLDFB_NO_CHANNELS_MAX]; + + DIRAC_DEC_STACK_MEM DirAC_mem; + float *reference_power, *reference_power_smooth; + float *onset_filter, *onset_filter_subframe, *p_onset_filter = NULL; + uint16_t coherence_flag; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + + push_wmops( "ivas_masa_ext_dirac_render_sf" ); + + /* Initialize aux buffers */ + hDirACRend = hMasaExtRend->hDirACRend; + hSpatParamRendCom = hMasaExtRend->hSpatParamRendCom; + nchan_transport = hMasaExtRend->nchan_input; + + DirAC_mem = hDirACRend->stack_mem; + + reference_power = DirAC_mem.reference_power; + reference_power_smooth = DirAC_mem.reference_power + hSpatParamRendCom->num_freq_bands; + onset_filter = DirAC_mem.onset_filter; + onset_filter_subframe = ( DirAC_mem.onset_filter == NULL ) ? NULL : DirAC_mem.onset_filter + hSpatParamRendCom->num_freq_bands; + + coherence_flag = 1; /* There is always coherence assumed for ext rend of MASA */ + + /* Construct default MASA band mapping */ + for ( i = 0; i < MASA_FREQUENCY_BANDS + 1; i++ ) + { + masa_band_mapping[i] = i; + } + + /* Subframe loop */ + slot_idx_start = hSpatParamRendCom->slots_rendered; + slot_idx_start_cldfb_synth = 0; + + subframe_idx = hSpatParamRendCom->subframes_rendered; + md_idx = hSpatParamRendCom->render_to_md_map[subframe_idx]; + + /* copy parameters into local buffers*/ + mvs2s( hSpatParamRendCom->azimuth[hSpatParamRendCom->render_to_md_map[subframe_idx]], azimuth, hSpatParamRendCom->num_freq_bands ); + mvs2s( hSpatParamRendCom->elevation[hSpatParamRendCom->render_to_md_map[subframe_idx]], elevation, hSpatParamRendCom->num_freq_bands ); + mvr2r( hSpatParamRendCom->diffuseness_vector[hSpatParamRendCom->render_to_md_map[subframe_idx]], diffuseness_vector, hSpatParamRendCom->num_freq_bands ); + + if ( hDirACRend->synthesisConf != DIRAC_SYNTHESIS_GAIN_SHD ) + { + set_zero( reference_power_smooth, hSpatParamRendCom->num_freq_bands ); + } + else + { + set_zero( onset_filter_subframe, hSpatParamRendCom->num_freq_bands ); + } + + /* compute response */ + if ( hDirACRend->synthesisConf != DIRAC_SYNTHESIS_GAIN_SHD ) + { + ivas_dirac_dec_compute_power_factors( hSpatParamRendCom->num_freq_bands, + diffuseness_vector, + hDirACRend->h_output_synthesis_psd_params.max_band_decorr, + hDirACRend->h_output_synthesis_psd_state.direct_power_factor, + hDirACRend->h_output_synthesis_psd_state.diffuse_power_factor ); + + if ( coherence_flag ) + { + for ( i = 0; i < hSpatParamRendCom->num_freq_bands; i++ ) + { + dirEne = hDirACRend->h_output_synthesis_psd_state.direct_power_factor[i]; + surCohEner = hDirACRend->h_output_synthesis_psd_state.diffuse_power_factor[i] * hSpatParamRendCom->surroundingCoherence[md_idx][i]; + hDirACRend->h_output_synthesis_psd_state.diffuse_power_factor[i] -= surCohEner; + hDirACRend->h_output_synthesis_psd_state.direct_power_factor[i] += surCohEner; + + surCohRatio[i] = surCohEner / ( 1e-12f + dirEne + surCohEner ); + } + } + else + { + set_zero( surCohRatio, hSpatParamRendCom->num_freq_bands ); + } + } + else + { + ivas_dirac_dec_compute_gain_factors( hSpatParamRendCom->num_freq_bands, + hSpatParamRendCom->diffuseness_vector[md_idx], + hDirACRend->h_output_synthesis_psd_params.max_band_decorr, + hDirACRend->h_output_synthesis_psd_state.direct_power_factor, + hDirACRend->h_output_synthesis_psd_state.diffuse_power_factor ); + + if ( coherence_flag ) + { + for ( i = 0; i < hSpatParamRendCom->num_freq_bands; i++ ) + { + surCohRatio[i] = hSpatParamRendCom->surroundingCoherence[md_idx][i]; + } + } + else + { + set_zero( surCohRatio, hSpatParamRendCom->num_freq_bands ); + } + } + + ivas_dirac_dec_compute_directional_responses( hSpatParamRendCom, + hDirACRend, + hMasaExtRend->hVBAPdata, + masa_band_mapping, + NULL, + azimuth, + elevation, + md_idx, + surCohRatio, + 0, + NULL, + 0 ); + + + for ( slot_idx = 0; slot_idx < hSpatParamRendCom->subframe_nbslots[subframe_idx]; slot_idx++ ) + { + index_slot = slot_idx_start + slot_idx; + md_idx = hSpatParamRendCom->render_to_md_map[subframe_idx]; + + /* CLDFB Analysis*/ + for ( ch = 0; ch < nchan_transport; ch++ ) + { + cldfbAnalysis_ts( &( output_f[ch][hSpatParamRendCom->num_freq_bands * index_slot] ), + Cldfb_RealBuffer[ch][0], + Cldfb_ImagBuffer[ch][0], + hSpatParamRendCom->num_freq_bands, + hMasaExtRend->cldfbAnaRend[ch] ); + } + + /*-----------------------------------------------------------------* + * protoype signal computation + *-----------------------------------------------------------------*/ + + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) + { + protoSignalComputation_shd( Cldfb_RealBuffer, Cldfb_ImagBuffer, + hDirACRend->h_output_synthesis_psd_state.proto_direct_buffer_f, + hDirACRend->h_output_synthesis_psd_state.proto_diffuse_buffer_f, + reference_power, slot_idx, nchan_transport, + hDirACRend->num_outputs_diff, + hSpatParamRendCom->num_freq_bands, + 0 ); + } + else if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) + { + protoSignalComputation2( Cldfb_RealBuffer, Cldfb_ImagBuffer, hDirACRend->proto_frame_f, + hDirACRend->h_output_synthesis_psd_state.proto_direct_buffer_f, + reference_power, hDirACRend->h_output_synthesis_psd_state.proto_power_smooth, + 0, slot_idx, hSpatParamRendCom->num_freq_bands, hDirACRend->masa_stereo_type_detect ); + } + else + { + switch ( nchan_transport ) + { + case 2: + protoSignalComputation2( Cldfb_RealBuffer, Cldfb_ImagBuffer, + hDirACRend->proto_frame_f, + hDirACRend->h_output_synthesis_psd_state.proto_direct_buffer_f, + reference_power, + hDirACRend->h_output_synthesis_psd_state.proto_power_smooth, + hDirACRend->hOutSetup.is_loudspeaker_setup, + slot_idx, + hSpatParamRendCom->num_freq_bands, + hDirACRend->masa_stereo_type_detect ); + break; + case 1: + protoSignalComputation1( Cldfb_RealBuffer, Cldfb_ImagBuffer, + hDirACRend->proto_frame_f, + hDirACRend->h_output_synthesis_psd_state.proto_direct_buffer_f, + reference_power, + hDirACRend->h_output_synthesis_psd_state.proto_power_smooth, + slot_idx, + hDirACRend->num_protos_diff, + hSpatParamRendCom->num_freq_bands ); + break; + default: + return; + } + } + + /*-----------------------------------------------------------------* + * frequency domain decorrelation + *-----------------------------------------------------------------*/ + + if ( hDirACRend->proto_signal_decorr_on == 1 ) + { + /* decorrelate prototype frame */ + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) + { + ivas_dirac_dec_decorr_process( hSpatParamRendCom->num_freq_bands, + hDirACRend->num_outputs_diff, + hDirACRend->num_protos_diff, + hDirACRend->synthesisConf, + nchan_transport, + hDirACRend->h_output_synthesis_psd_state.proto_diffuse_buffer_f + slot_idx * 2 * hSpatParamRendCom->num_freq_bands * hDirACRend->num_outputs_diff, + hDirACRend->num_protos_diff, + hDirACRend->proto_index_diff, + hDirACRend->h_output_synthesis_psd_state.proto_diffuse_buffer_f + slot_idx * 2 * hSpatParamRendCom->num_freq_bands * hDirACRend->num_outputs_diff + 2 * hSpatParamRendCom->num_freq_bands * min( 4, nchan_transport ), + onset_filter, + hDirACRend->h_freq_domain_decorr_ap_params, + hDirACRend->h_freq_domain_decorr_ap_state ); + + v_multc( onset_filter, 0.25f, onset_filter, hSpatParamRendCom->num_freq_bands ); + v_add( onset_filter, onset_filter_subframe, onset_filter_subframe, hSpatParamRendCom->num_freq_bands ); + p_onset_filter = onset_filter_subframe; + } + else + { + ivas_dirac_dec_decorr_process( hSpatParamRendCom->num_freq_bands, + hDirACRend->num_outputs_diff, + hDirACRend->num_protos_diff, + hDirACRend->synthesisConf, + nchan_transport, + hDirACRend->proto_frame_f, + hDirACRend->num_protos_diff, + hDirACRend->proto_index_diff, + DirAC_mem.frame_dec_f, + onset_filter, + hDirACRend->h_freq_domain_decorr_ap_params, + hDirACRend->h_freq_domain_decorr_ap_state ); + + hDirACRend->proto_frame_dec_f = DirAC_mem.frame_dec_f; + p_onset_filter = onset_filter; + } + } + else + { + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) + { + set_f( onset_filter_subframe, 1.f, hSpatParamRendCom->num_freq_bands ); + p_onset_filter = onset_filter_subframe; + } + else + { + /* no frequency domain decorrelation: use prototype frame */ + hDirACRend->proto_frame_dec_f = hDirACRend->proto_frame_f; + p_onset_filter = NULL; + } + } + + /*-----------------------------------------------------------------* + * output synthesis + *-----------------------------------------------------------------*/ + + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_LS || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD ) + { + /*Compute diffuse prototypes*/ + ivas_dirac_dec_compute_diffuse_proto( hDirACRend, hSpatParamRendCom->num_freq_bands, slot_idx ); + } + + ivas_dirac_dec_output_synthesis_process_slot( reference_power, + p_onset_filter, + azimuth, + elevation, + hSpatParamRendCom->diffuseness_vector[md_idx], + hSpatParamRendCom, + hDirACRend, + 0, + 0, + hMasaExtRend->hVBAPdata, + hDirACRend->hOutSetup, + nchan_transport, + md_idx, + 0, + 0 ); + + if ( hDirACRend->synthesisConf != DIRAC_SYNTHESIS_GAIN_SHD ) + { + v_add( reference_power, reference_power_smooth, reference_power_smooth, hSpatParamRendCom->num_freq_bands ); + } + } + + ivas_dirac_dec_output_synthesis_get_interpolator( &hDirACRend->h_output_synthesis_psd_params, hSpatParamRendCom->subframe_nbslots[subframe_idx] ); + + + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) + { + ivas_dirac_dec_output_synthesis_process_subframe_gain_shd( Cldfb_RealBuffer, + Cldfb_ImagBuffer, + hSpatParamRendCom, + hDirACRend, + nchan_transport, + hSpatParamRendCom->subframe_nbslots[subframe_idx], + p_onset_filter, + diffuseness_vector, + 0, + 0 ); + } + else + { + ivas_dirac_dec_output_synthesis_process_subframe_psd_ls( Cldfb_RealBuffer, + Cldfb_ImagBuffer, + hSpatParamRendCom, + hDirACRend, + hSpatParamRendCom->subframe_nbslots[subframe_idx], + diffuseness_vector, + reference_power_smooth, + 1.0f, + 0 ); + } + + /*-----------------------------------------------------------------* + * CLDFB synthesis (and binaural rendering) + *-----------------------------------------------------------------*/ + + index_slot = slot_idx_start_cldfb_synth; + + { + float *RealBuffer[MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[MAX_PARAM_SPATIAL_SUBFRAMES]; + int16_t outchannels; + + idx_in = 0; + idx_lfe = 0; + + outchannels = hDirACRend->hOutSetup.nchan_out_woLFE + hDirACRend->hOutSetup.num_lfe; + + /* Note here that compared to decoder path, there is no separate channel ever for MASA ext rend path */ + + for ( ch = 0; ch < outchannels; ch++ ) + { + if ( ( hDirACRend->hOutSetup.num_lfe > 0 ) && ( hDirACRend->hOutSetup.index_lfe[idx_lfe] == ch ) ) + { + /* No LFE for MASA rendering */ + set_zero( &( output_f[ch][index_slot * hSpatParamRendCom->num_freq_bands] ), hSpatParamRendCom->subframe_nbslots[subframe_idx] * hSpatParamRendCom->num_freq_bands ); + + if ( idx_lfe < ( hDirACRend->hOutSetup.num_lfe - 1 ) ) + { + idx_lfe++; + } + } + else + { + /* open CLDFB buffer up to CLDFB_NO_CHANNELS_MAX bands for 48kHz */ + for ( i = 0; i < hSpatParamRendCom->subframe_nbslots[subframe_idx]; i++ ) + { + RealBuffer[i] = Cldfb_RealBuffer[idx_in][i]; + ImagBuffer[i] = Cldfb_ImagBuffer[idx_in][i]; + } + cldfbSynthesis( RealBuffer, ImagBuffer, &( output_f[ch][index_slot * hSpatParamRendCom->num_freq_bands] ), hSpatParamRendCom->num_freq_bands * hSpatParamRendCom->subframe_nbslots[subframe_idx], hMasaExtRend->cldfbSynRend[idx_in] ); + idx_in++; + } + } + } + hSpatParamRendCom->slots_rendered += hSpatParamRendCom->subframe_nbslots[subframe_idx]; + hSpatParamRendCom->subframes_rendered++; + + pop_wmops(); + + return; +} + +void ivas_masa_ext_dirac_render( + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: IVAS decoder structure */ + float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ + const int16_t nchan_transport, /* i : number of transport channels */ + const int16_t num_subframes /* i : number of subframes to render */ +) +{ + int16_t subframe_idx; + float *output_f_local[MAX_OUTPUT_CHANNELS]; + int16_t n, n_samples_sf; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + + hSpatParamRendCom = hMasaExtRend->hSpatParamRendCom; + + n_samples_sf = JBM_CLDFB_SLOTS_IN_SUBFRAME * hSpatParamRendCom->slot_size; + + for ( n = 0; n < MAX_OUTPUT_CHANNELS; n++ ) + { + output_f_local[n] = output_f[n]; + } + + hSpatParamRendCom->subframes_rendered = hSpatParamRendCom->dirac_read_idx; + + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + { + hSpatParamRendCom->slots_rendered = 0; + ivas_masa_ext_dirac_render_sf( hMasaExtRend, output_f_local ); + for ( n = 0; n < MAX_OUTPUT_CHANNELS; n++ ) + { + output_f_local[n] += n_samples_sf; + } + + hSpatParamRendCom->dirac_read_idx = ( hSpatParamRendCom->dirac_read_idx + 1 ) % hSpatParamRendCom->dirac_md_buffer_length; + } + + + return; +} diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index a8b9c029b0..8880d2a5cc 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -176,6 +176,13 @@ void ivas_dirac_dec_binaural_render( float *output_f[] /* o : rendered time signal */ ); +void ivas_masa_ext_rend_parambin_render( + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ + float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ + const int16_t num_subframes /* i : number of subframes to render */ +); + ivas_error ivas_dirac_dec_init_binaural_data( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ HRTFS_PARAMBIN_HANDLE hHrtfParambin /* i : HRTF structure for rendering */ @@ -458,18 +465,18 @@ void ivas_dirac_dec_compute_power_factors( ); void ivas_dirac_dec_compute_directional_responses( - SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ - DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ - const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ - const MASA_DECODER_HANDLE hMasa, /* i : MASA decoder structure */ - MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ + DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ + const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ + const int16_t *masa_band_mapping, /* i : Band mapping for MASA, NULL assumes not using MASA in any form */ + MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ const int16_t *azimuth, const int16_t *elevation, const int16_t md_idx, const float *surCohRatio, - const int16_t shd_rot_max_order, /* i : split-order rotation method */ - const float *p_Rmat, /* i : rotation matrix */ - const int16_t hodirac_flag /* i : flag to indicate HO-DirAC mode */ + const int16_t shd_rot_max_order, /* i : split-order rotation method */ + const float *p_Rmat, /* i : rotation matrix */ + const int16_t hodirac_flag /* i : flag to indicate HO-DirAC mode */ ); void ivas_dirac_dec_get_frequency_axis( @@ -485,7 +492,8 @@ ivas_error ivas_spat_hSpatParamRendCom_config( const IVAS_FORMAT ivas_format, const MC_MODE mc_mode, const int32_t output_Fs, - const int16_t hodirac_flag + const int16_t hodirac_flag, + const int16_t masa_ext_rend_flag ); void ivas_spat_hSpatParamRendCom_close( @@ -506,6 +514,13 @@ void ivas_dirac_deallocate_parameters( const int16_t params_flag /* i : set of parameters flag */ ); +void ivas_masa_ext_dirac_render( + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA renderer structure */ + float *output_f[], /* i/o: input/output signals in time domain */ + const int16_t nchan_transport, /* i : number of transport channels */ + const int16_t num_subframes /* i : number of subframes to render */ +); + /*----------------------------------------------------------------------------------* * HRTF diff --git a/lib_rend/ivas_rom_rend.c b/lib_rend/ivas_rom_rend.c index 9348357561..99ffe44285 100644 --- a/lib_rend/ivas_rom_rend.c +++ b/lib_rend/ivas_rom_rend.c @@ -39,6 +39,149 @@ /* clang-format off */ +/*----------------------------------------------------------------------------------* + * DirAC renderer ROM tables + *----------------------------------------------------------------------------------*/ + +const float ls_azimuth_4d4[8] = { 45.0f, -45.0f, 135.0f, -135.0f, 45.0f, -45.0f, 135.0f, -135.0f }; + +const float ls_elevation_4d4[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 35.0f, 35.0f, 35.0f, 35.0f }; + +const float diffuse_response_CICP6[5] = { 0.4219f, 0.4219f, 0.3704f, 0.5034f, 0.5034f }; + +const float diffuse_response_CICP14[7] = { 0.3817f, 0.3817f, 0.2819f, 0.5399f, 0.5399f, 0.1519f, 0.1519f }; + +const float diffuse_response_CICP16[9] = { 0.3456f, 0.3456f, 0.3035f, 0.4124f, 0.4124f, 0.2702f, 0.2702f, 0.3023f, 0.3023f }; + +const int16_t ap_pre_delay[DIRAC_DECORR_NUM_SPLIT_BANDS] = { 7, 2, 1 }; + +const int16_t ap_filter_length[DIRAC_DECORR_NUM_SPLIT_BANDS] = { 15, 6, 3 }; + +const float ap_lattice_delta_phi[DIRAC_MAX_NUM_DECORR_FILTERS*DIRAC_MAX_DECORR_FILTER_LEN] = +{ + 1.802519f, 0.922986f, 1.813685f, 1.272828f, 0.856928f, 0.366571f, 1.531249f, 1.318158f, 0.123812f, 0.897173f, 0.958696f, 1.256384f, 0.179677f, 0.668918f, 1.440292f, 1.573058f, 1.396481f, 1.191463f, 0.444143f, 1.666942f, + 1.273955f, 1.747171f, 1.408330f, 1.002782f, 1.559302f, 1.782992f, 1.474896f, 0.813181f, 1.457724f, 0.588531f, 1.384302f, 0.156493f, 0.600048f, 1.661632f, 0.538958f, 0.645429f, 0.565237f, 0.024684f, 0.264229f, 0.062140f, + 1.235343f, 0.851725f, 1.820211f, 0.116148f, 0.972111f, 0.488703f, 1.777672f, 1.452170f, 0.814134f, 1.272649f, 1.281416f, 0.101871f, 0.897888f, 0.199760f, 0.085732f, 1.686579f, 0.964558f, 0.057281f, 0.910252f, 1.662302f, + 0.955234f, 0.834348f, 1.672478f, 1.324896f, 0.444544f, 1.721172f, 0.153356f, 1.602240f, 0.171880f, 1.169774f, 0.543628f, 1.409581f, 1.763724f, 1.686754f, 1.210390f, 0.402691f, 0.983618f, 0.862997f, 1.220409f, 0.890061f, + 0.031641f, 0.461590f, 1.719550f, 1.357698f, 1.112262f, 1.166531f, 0.246097f, 1.387325f, 0.177485f, 1.446268f, 0.799476f, 1.667227f, 1.723465f, 1.505920f, 0.245874f, 1.155854f, 0.831394f, 0.677194f, 0.568871f, 1.652070f, + 0.019803f, 1.197794f, 0.635553f, 0.531682f, 0.878194f, 0.048050f, 0.080480f, 1.566743f, 0.724210f, 0.853668f, 1.741191f, 0.698465f, 1.553550f, 0.130290f, 0.688346f, 1.331091f, 0.599759f, 1.125466f, 1.764818f, 1.042879f, + 1.486589f, 1.627971f, 1.871181f, 0.102359f, 0.035021f, 1.403176f, 1.468675f, 0.190347f, 0.553282f, 1.031227f, 1.232390f, 1.255724f, 1.504443f, 0.683526f, 0.600958f, 1.746936f, 1.529243f, 1.448196f, 0.646850f, 0.116053f, + 1.283295f, 0.355220f, 1.380620f, 1.858453f, 0.818804f, 0.219006f, 0.476292f, 0.420029f, 1.291187f, 0.568738f, 1.174088f, 0.628805f, 1.753154f, 1.459582f, 1.354449f, 1.755790f, 0.441757f, 0.856240f, 1.647962f, 0.686353f, + 1.395289f, 0.699934f, 0.239310f, 0.239535f, 0.958190f, 0.748780f, 0.513784f, 1.666344f, 1.461995f, 1.599060f, 0.893107f, 0.341873f, 1.387703f, 1.808363f, 0.676542f, 1.424958f, 0.310574f, 0.836247f, 1.011101f, 1.686200f, + 1.823432f, 0.994827f, 1.635555f, 0.684380f, 1.017029f, 1.440371f, 1.694641f, 0.607132f, 1.197331f, 0.862161f, 0.666449f, 1.047956f, 0.159627f, 0.043131f, 1.251515f, 1.618724f, 0.216906f, 0.152250f, 0.471610f, 0.744260f, + 0.576810f, 1.632177f, 1.556912f, 1.866317f, 0.568088f, 1.541817f, 1.726725f, 0.275154f, 0.814958f, 0.863399f, 1.333040f, 0.148277f, 0.197893f, 1.048665f, 1.158090f, 1.692225f, 0.884294f, 0.289619f, 0.380633f, 1.728234f, + 1.433213f, 1.749505f, 1.533837f, 0.669701f, 0.372580f, 1.052390f, 1.116645f, 0.181320f, 1.139126f, 0.222671f, 0.604393f, 1.811797f, 1.743315f, 1.368792f, 1.861434f, 0.751908f, 0.159811f, 1.566503f, 0.443273f, 1.667530f, + 1.083060f, 1.243136f, 0.717777f, 0.675019f, 0.690490f, 0.672228f, 1.060789f, 0.423566f, 1.198457f, 0.485768f, 0.993953f, 0.443540f, 0.361702f, 1.552042f, 0.863562f, 1.517677f, 1.061899f, 0.691413f, 1.642818f, 1.756590f, + 0.278323f, 0.790363f, 0.172303f, 0.417138f, 0.009343f, 0.783325f, 1.369303f, 1.041067f, 0.467102f, 0.992773f, 1.525170f, 0.871213f, 0.243906f, 1.542036f, 0.449148f, 0.843633f, 0.191800f, 1.614246f, 1.038188f, 1.415620f, + 0.551081f, 0.382599f, 1.410121f, 0.102084f, 0.137286f, 0.671081f, 0.254860f, 1.758068f, 1.079013f, 0.129143f, 1.410873f, 0.150485f, 0.601119f, 0.760737f, 0.975905f, 0.223261f, 0.710162f, 1.677048f, 0.996836f, 1.849865f, + 1.536222f, 0.089016f, 0.960881f, 0.388690f, 0.379955f, 1.002223f, 1.271420f, 1.410632f, 0.254397f, 1.535559f, 1.133703f, 1.305280f, 1.466565f, 0.274167f, 0.399688f, 1.359638f, 1.766289f, 1.401348f, 1.310883f, 0.261030f, + 1.314825f, 1.538635f, 1.317986f, 1.243167f, 1.749461f, 1.689706f, 0.024853f, 0.634754f, 1.036317f, 1.828101f, 1.676951f, 0.023606f, 0.857000f, 0.076471f, 1.622198f, 0.254469f, 1.451625f, 1.720881f, 0.763812f, 0.186982f, + 0.056994f, 0.590507f, 0.375291f, 1.609261f, 0.607721f, 0.026355f, 0.483366f, 0.823931f, 0.792878f, 0.163577f, 0.753588f, 0.730789f, 0.135991f, 1.031660f, 1.554135f, 1.192863f, 0.016693f, 0.125796f, 1.017920f, 1.591773f, + 0.575956f, 0.112943f, 0.249506f, 1.399570f, 0.053241f, 1.410759f, 0.251638f, 1.059086f, 0.025315f, 1.422914f, 1.030412f, 0.848758f, 0.317396f, 1.375456f, 1.116858f, 1.682310f, 0.279550f, 0.325974f, 0.937704f, 1.744329f, + 0.447773f, 1.024286f, 1.001528f, 1.863684f, 1.278323f, 0.860699f, 1.346331f, 1.692596f, 0.022627f, 1.033613f, 0.546354f, 0.395804f, 1.486546f, 1.381045f, 1.312260f, 0.245976f, 1.607429f, 1.818793f, 0.964359f, 1.496598f, + 0.669967f, 1.535929f, 1.841878f, 0.979127f, 0.614002f, 1.879218f, 0.512531f, 1.167061f, 0.081697f, 1.773427f, 1.535668f, 0.757729f, 0.220395f, 1.538243f, 1.281162f, 0.302159f, 0.889871f, 0.798522f, 1.476288f, 1.665941f, + 0.915365f, 1.394094f, 0.757041f, 0.350064f, 1.199679f, 1.319499f, 1.128405f, 0.632337f, 0.790673f, 0.461582f, 1.693343f, 1.537442f, 0.346527f, 0.433782f, 1.754552f, 0.550903f, 0.686724f, 0.764433f, 1.792750f, 1.489998f +}; + +const float ap_lattice_coeffs_1[DIRAC_DECORR_FILTER_LEN_1*DIRAC_MAX_NUM_DECORR_FILTERS] = +{ + 0.795329f, 0.502700f, 0.204456f, 0.416566f, 0.459648f, 0.270454f, -0.201944f, 0.027997f, 0.067811f, -0.052627f, -0.038779f, -0.057387f, 0.020480f, 0.367697f, -0.593705f, + 0.533667f, 0.202500f, -0.001953f, 0.195081f, -0.184458f, -0.233867f, 0.228100f, -0.329293f, -0.338714f, -0.079700f, 0.052389f, -0.009468f, 0.178807f, 0.190843f, -0.478877f, + 0.044022f, 0.788498f, 0.133529f, -0.173657f, 0.545391f, 0.681431f, 0.332868f, 0.294685f, 0.325261f, 0.047617f, 0.157401f, 0.116272f, 0.218980f, -0.189248f, -0.317401f, + -0.753681f, 0.772385f, -0.507384f, 0.276980f, -0.692775f, 0.232302f, -0.354759f, -0.015620f, -0.223059f, 0.310506f, -0.035034f, 0.085783f, -0.499998f, 0.215260f, 0.201415f, + -0.495551f, -0.301660f, 0.196510f, 0.326147f, -0.594364f, 0.314921f, 0.668671f, 0.076643f, 0.045711f, -0.124790f, -0.203272f, -0.297190f, 0.125806f, -0.179483f, -0.201757f, + 0.161128f, 0.396050f, 0.266897f, 0.356586f, 0.488145f, -0.056254f, 0.139280f, -0.296405f, -0.112844f, 0.037405f, -0.367425f, -0.216292f, -0.277360f, 0.389420f, 0.115115f, + -0.240449f, -0.271015f, 0.426720f, -0.011059f, 0.151813f, 0.253490f, 0.225764f, 0.498716f, -0.136377f, 0.443004f, -0.305017f, -0.031310f, -0.010765f, 0.170349f, 0.496478f, + -0.139875f, -0.241998f, -0.104850f, 0.294343f, -0.067728f, -0.492202f, -0.487610f, 0.036395f, 0.109393f, 0.396155f, -0.352845f, -0.205913f, -0.082999f, -0.463033f, -0.309296f, + 0.025886f, -0.092456f, -0.125139f, -0.156117f, -0.004887f, 0.178440f, 0.089586f, 0.044827f, 0.238219f, -0.312120f, -0.390688f, -0.178543f, 0.454418f, 0.387012f, -0.388874f, + -0.197797f, 0.035540f, 0.455388f, -0.054410f, 0.380035f, 0.290964f, 0.048804f, 0.078637f, 0.221740f, -0.217548f, 0.121289f, -0.396681f, -0.218482f, -0.127265f, -0.269507f, + -0.344218f, -0.465038f, -0.421415f, -0.026031f, 0.221547f, 0.361993f, -0.348243f, 0.294983f, 0.366175f, 0.070663f, -0.086050f, 0.252129f, 0.156066f, -0.062800f, 0.408972f, + 0.242461f, -0.301764f, -0.066160f, 0.388651f, -0.462227f, -0.158880f, 0.230796f, -0.093179f, 0.047076f, 0.073402f, -0.335018f, 0.022940f, 0.354611f, 0.072391f, 0.019473f, + 0.336900f, -0.480534f, 0.170267f, 0.259663f, -0.393576f, -0.348588f, -0.108962f, 0.278842f, 0.385490f, 0.362249f, -0.318739f, 0.362305f, 0.288936f, 0.291204f, -0.278019f, + -0.449302f, -0.288513f, -0.413973f, -0.405279f, -0.295152f, -0.245110f, 0.002530f, 0.287890f, 0.348229f, -0.178354f, -0.206517f, 0.351081f, -0.482205f, 0.021360f, -0.492207f, + 0.048642f, 0.453282f, 0.109160f, 0.232599f, -0.473781f, 0.140502f, 0.352527f, -0.098606f, 0.147172f, -0.055797f, 0.107739f, -0.231026f, 0.357310f, 0.348031f, 0.232404f, + -0.324788f, 0.162480f, 0.057647f, -0.060734f, -0.009742f, -0.224185f, -0.282355f, -0.065443f, 0.064697f, 0.280370f, -0.284906f, -0.470501f, 0.019484f, -0.442308f, 0.377214f, + 0.447384f, -0.290498f, 0.345528f, 0.370207f, -0.313120f, 0.119592f, 0.300014f, 0.406995f, -0.277922f, 0.447039f, 0.194824f, 0.157703f, -0.223402f, -0.147167f, 0.379073f, + -0.216342f, 0.137967f, -0.397180f, 0.073905f, -0.273110f, -0.443037f, -0.168327f, 0.346264f, 0.037543f, 0.065387f, 0.163901f, -0.122523f, 0.365477f, -0.316321f, 0.117273f, + -0.301282f, 0.169625f, -0.336466f, 0.269914f, -0.420160f, -0.331296f, 0.498523f, -0.393009f, -0.462184f, -0.323097f, 0.470977f, -0.359463f, 0.264315f, 0.216797f, 0.493400f, + 0.151489f, -0.321653f, 0.464413f, -0.355673f, 0.420401f, 0.184297f, -0.302128f, 0.136536f, -0.252849f, 0.193906f, 0.298775f, 0.238808f, -0.386298f, -0.181999f, -0.077326f, + 0.124840f, -0.263394f, 0.389606f, 0.004502f, 0.339804f, 0.415204f, 0.377751f, 0.400221f, 0.352426f, -0.003808f, -0.184530f, 0.433348f, -0.237554f, 0.147684f, 0.407210f, + 0.067616f, -0.249313f, -0.354010f, 0.320937f, 0.426000f, 0.309576f, -0.189689f, -0.209447f, 0.158967f, -0.081929f, 0.386828f, 0.178582f, -0.407143f, 0.222189f, -0.149097f +}; + +const float ap_lattice_coeffs_2[DIRAC_DECORR_FILTER_LEN_2*DIRAC_MAX_NUM_DECORR_FILTERS] = +{ + 0.633692f, 0.681207f, -0.049418f, 0.286715f, 0.146022f, 0.135402f, + -0.410145f, -0.206766f, -0.656968f, -0.101746f, 0.436299f, 0.339818f, + -0.131383f, -0.773746f, -0.301627f, 0.327561f, 0.332227f, 0.205858f, + 0.717602f, -0.552370f, -0.150136f, 0.054556f, 0.239519f, -0.648477f, + -0.721848f, 0.189377f, 0.068185f, 0.006216f, 0.077025f, 0.038678f, + 0.423415f, -0.108658f, 0.432050f, -0.414641f, 0.277840f, 0.418486f, + 0.077811f, -0.283063f, -0.357008f, 0.004634f, -0.442640f, -0.372315f, + 0.403336f, 0.243023f, 0.314367f, 0.444513f, 0.347517f, 0.496043f, + 0.108408f, 0.469382f, -0.170286f, 0.326310f, 0.172157f, 0.440334f, + 0.058417f, 0.339673f, -0.194965f, 0.491219f, -0.281296f, -0.043120f, + -0.482487f, -0.335005f, -0.336159f, 0.196469f, -0.164623f, 0.442491f, + -0.135190f, -0.354385f, 0.452133f, -0.311221f, -0.347640f, 0.498342f, + -0.288999f, 0.376431f, -0.177924f, 0.195542f, 0.333872f, -0.152697f, + 0.053230f, 0.457767f, 0.442476f, 0.235254f, -0.345159f, -0.286098f, + 0.155438f, 0.405393f, 0.217073f, -0.107960f, 0.010698f, 0.368987f, + -0.498358f, -0.495816f, -0.215479f, -0.093869f, 0.320276f, -0.013842f, + 0.489406f, 0.470814f, -0.065834f, 0.346871f, 0.027279f, 0.150086f, + 0.484013f, -0.497391f, 0.168796f, -0.493841f, -0.173528f, 0.334676f, + 0.264235f, -0.424651f, -0.314926f, 0.253086f, 0.397381f, -0.491565f, + -0.453727f, -0.463358f, -0.019128f, 0.000344f, 0.315432f, 0.472345f, + 0.095139f, 0.283375f, -0.225088f, -0.119762f, -0.476871f, 0.037525f, + 0.336951f, 0.494511f, -0.062603f, 0.177652f, 0.463892f, 0.489286f +}; + +const float ap_lattice_coeffs_3[DIRAC_DECORR_FILTER_LEN_3*DIRAC_MAX_NUM_DECORR_FILTERS] = +{ + 0.018977f, -0.212205f, 0.422719f, + -0.400657f, -0.106890f, -0.024589f, + 0.140005f, 0.279582f, 0.032357f, + 0.632535f, 0.578535f, -0.734606f, + 0.017182f, 0.013244f, -0.027715f, + -0.353356f, -0.482160f, -0.491265f, + 0.457024f, 0.165122f, 0.469723f, + -0.195705f, 0.440105f, -0.477366f, + 0.360186f, -0.490565f, 0.484623f, + -0.173791f, 0.007543f, 0.278186f, + 0.434416f, 0.060363f, -0.193717f, + -0.033709f, 0.496222f, 0.002939f, + -0.480848f, -0.109552f, -0.023198f, + 0.324679f, -0.292075f, -0.356148f, + -0.366595f, 0.380917f, -0.301741f, + 0.110318f, 0.383789f, 0.303984f, + -0.499685f, -0.349584f, 0.334749f, + -0.020224f, -0.430078f, -0.154705f, + -0.371129f, 0.334080f, 0.346913f, + -0.166781f, -0.229089f, 0.117956f, + 0.341292f, 0.490463f, 0.493655f, + -0.367726f, 0.426528f, -0.045774f +}; + +const float * const ap_lattice_coeffs[DIRAC_DECORR_NUM_SPLIT_BANDS] = +{ + &ap_lattice_coeffs_1[0], + &ap_lattice_coeffs_2[0], + &ap_lattice_coeffs_3[0], +}; + +const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1] = +{ + 0.0f, 0.125f, 0.375f, 1.0f +}; + +const int16_t sba_map_tc[11] = +{ + 0, 1, 2, 3, 4, 8, 9, 15, 5, 6, 7 +}; +const int16_t sba_map_tc_512[11] = +{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15 +}; + /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables *----------------------------------------------------------------------------------*/ diff --git a/lib_rend/ivas_rom_rend.h b/lib_rend/ivas_rom_rend.h index 0e42c33286..165f4791c6 100644 --- a/lib_rend/ivas_rom_rend.h +++ b/lib_rend/ivas_rom_rend.h @@ -41,6 +41,28 @@ #include "ivas_cnst.h" #include "ivas_stat_rend.h" +/*----------------------------------------------------------------------------------* + * DirAC renderer ROM tables + *----------------------------------------------------------------------------------*/ + +extern const float ls_azimuth_4d4[8]; +extern const float ls_elevation_4d4[8]; +extern const float diffuse_response_CICP6[5]; +extern const float diffuse_response_CICP14[7]; +extern const float diffuse_response_CICP16[9]; + +extern const int16_t ap_pre_delay[DIRAC_DECORR_NUM_SPLIT_BANDS]; +extern const int16_t ap_filter_length[DIRAC_DECORR_NUM_SPLIT_BANDS]; +extern const float ap_lattice_delta_phi[DIRAC_MAX_NUM_DECORR_FILTERS * DIRAC_MAX_DECORR_FILTER_LEN]; +extern const float ap_lattice_coeffs_1[DIRAC_DECORR_FILTER_LEN_1 * DIRAC_MAX_NUM_DECORR_FILTERS]; +extern const float ap_lattice_coeffs_2[DIRAC_DECORR_FILTER_LEN_2 * DIRAC_MAX_NUM_DECORR_FILTERS]; +extern const float ap_lattice_coeffs_3[DIRAC_DECORR_FILTER_LEN_3 * DIRAC_MAX_NUM_DECORR_FILTERS]; +extern const float *const ap_lattice_coeffs[DIRAC_DECORR_NUM_SPLIT_BANDS]; +extern const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1]; + +extern const int16_t sba_map_tc[11]; +extern const int16_t sba_map_tc_512[11]; + /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables *----------------------------------------------------------------------------------*/ diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index ae8a382f3e..556e16bef0 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -1575,6 +1575,34 @@ typedef enum } ChannelType; +/*----------------------------------------------------------------------------------* + * MASA external renderer structure + *----------------------------------------------------------------------------------*/ + +typedef struct ivas_masa_external_rendering_struct +{ + int16_t nchan_input; + int16_t nchan_output; + RENDERER_TYPE renderer_type; + DIRAC_REND_HANDLE hDirACRend; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; +#ifdef SPLIT_REND_WITH_HEAD_ROT + DIRAC_DEC_BIN_HANDLE hDiracDecBin[MAX_HEAD_ROT_POSES]; +#else + DIRAC_DEC_BIN_HANDLE hDiracDecBin; +#endif + + REVERB_STRUCT_HANDLE hReverb; + HRTFS_PARAMBIN_HANDLE hHrtfParambin; + VBAP_HANDLE hVBAPdata; + float *hoa_dec_mtx; + + HANDLE_CLDFB_FILTER_BANK cldfbAnaRend[MASA_MAX_TRANSPORT_CHANNELS]; + HANDLE_CLDFB_FILTER_BANK cldfbSynRend[MAX_OUTPUT_CHANNELS]; + + int16_t current_subframe; +} MASA_EXT_REND_DATA, *MASA_EXT_REND_HANDLE; + /*----------------------------------------------------------------------------------* * Multichannel MASA (McMASA) analysis structure *----------------------------------------------------------------------------------*/ diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index f319e46bbd..aef5c7b237 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -212,17 +212,13 @@ typedef struct } input_split_post_rend; #endif -/* Due to API of some rendering methods, the renderer has to use the decoder struct. - Only struct members relevant for rendering will be initialized, therefore typedef as "dummy" decoder struct */ -typedef Decoder_Struct DecoderDummy; - typedef struct { input_base base; - DecoderDummy *decDummy; MASA_METADATA_FRAME masaMetadata; bool metadataHasBeenFed; float *bufferData; + MASA_EXT_REND_HANDLE hMasaExtRend; MASA_PREREND_HANDLE hMasaPrerend; } input_masa; @@ -267,6 +263,14 @@ struct IVAS_REND }; +/*-------------------------------------------------------------------* + * Local function prototypes + *-------------------------------------------------------------------*/ + +static ivas_error initMasaExtRenderer( input_masa *inputMasa, const AUDIO_CONFIG outConfig ); +static void freeMasaExtRenderer( MASA_EXT_REND_HANDLE *hMasaExtRendOut ); + + /*-------------------------------------------------------------------* * Local functions *-------------------------------------------------------------------*/ @@ -2936,439 +2940,368 @@ static void clearInputSba( } -static ivas_error initMasaDummyDecForMcOut( - input_masa *inputMasa, - const AUDIO_CONFIG outConfig ) +static ivas_error setRendInputActiveMasa( + void *input, + const AUDIO_CONFIG inConfig, + const IVAS_REND_InputId id, + RENDER_CONFIG_DATA *hRendCfg ) /* Todo: This is not used at all within MASA. Support might be better to do after refactoring. */ { ivas_error error; - int16_t numCldfbAnalyses; - int16_t numCldfbSyntheses; - int16_t i; - DecoderDummy *decDummy; - - decDummy = inputMasa->decDummy; - decDummy->hDecoderConfig->output_config = outConfig; - decDummy->sba_analysis_order = 1; - decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ - decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + rendering_context rendCtx; + AUDIO_CONFIG outConfig; + input_masa *inputMasa; + int16_t numInChannels; - ivas_output_init( &( decDummy->hOutSetup ), outConfig ); - ivas_output_init( &( decDummy->hIntSetup ), outConfig ); + inputMasa = (input_masa *) input; + rendCtx = inputMasa->base.ctx; + outConfig = *rendCtx.pOutConfig; + (void) hRendCfg; /* Suppress warning */ - decDummy->renderer_type = RENDERER_DIRAC; - if ( outConfig == IVAS_AUDIO_CONFIG_STEREO ) + if ( !isIoConfigPairSupported( inConfig, outConfig ) ) { - decDummy->renderer_type = RENDERER_STEREO_PARAMETRIC; + return IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED; } - decDummy->ivas_format = MASA_FORMAT; - decDummy->transport_config = IVAS_AUDIO_CONFIG_INVALID; - - /* Todo refactor: Access to qmetadata is not required by the algorithm. */ - if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + if ( ( error = allocateInputBaseBufferData( &inputMasa->bufferData, MAX_BUFFER_LENGTH ) ) != IVAS_ERR_OK ) { return error; } - decDummy->hQMetaData->coherence_flag = 1; + initRendInputBase( &inputMasa->base, inConfig, id, rendCtx, inputMasa->bufferData, MAX_BUFFER_LENGTH ); - if ( ( error = ivas_dirac_dec_config( decDummy, DIRAC_OPEN ) ) != IVAS_ERR_OK ) + if ( ( error = getAudioConfigNumChannels( inConfig, &numInChannels ) ) != IVAS_ERR_OK ) { return error; } - decDummy->hSpatParamRendCom->dirac_bs_md_write_idx = 0; - if ( decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) { - if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy, NULL ) ) != IVAS_ERR_OK ) + inputMasa->metadataHasBeenFed = false; + if ( ( error = masaPrerendOpen( &inputMasa->hMasaPrerend, inputMasa->base.inConfig == IVAS_AUDIO_CONFIG_MASA1 ? 1 : 2, *( inputMasa->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) { return error; } } - - numCldfbAnalyses = decDummy->nchan_transport; - numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; - - for ( i = 0; i < numCldfbAnalyses; i++ ) + else { - if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + // Todo split rend MASA ext: Maybe something should go in here + if ( ( error = initMasaExtRenderer( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { return error; } - } - for ( ; i < MAX_INTERN_CHANNELS; i++ ) - { - decDummy->cldfbAnaDec[i] = NULL; - } - for ( i = 0; i < numCldfbSyntheses; i++ ) - { - if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) - { - return error; - } - } - for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) - { - decDummy->cldfbSynDec[i] = NULL; + inputMasa->metadataHasBeenFed = false; } return IVAS_ERR_OK; } -static ivas_error initMasaDummyDecForSbaOut( - input_masa *inputMasa, - const AUDIO_CONFIG outConfig ) +static void clearInputMasa( + input_masa *inputMasa ) { - ivas_error error; - int16_t numCldfbAnalyses; - int16_t numCldfbSyntheses; - int16_t i; - DecoderDummy *decDummy; + rendering_context rendCtx; - decDummy = inputMasa->decDummy; - decDummy->hDecoderConfig->output_config = outConfig; - decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ - decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ - decDummy->sba_analysis_order = 1; + rendCtx = inputMasa->base.ctx; - ivas_output_init( &( decDummy->hOutSetup ), outConfig ); - ivas_output_init( &( decDummy->hIntSetup ), outConfig ); + freeInputBaseBufferData( &inputMasa->bufferData ); - decDummy->renderer_type = RENDERER_DIRAC; - decDummy->ivas_format = MASA_FORMAT; - decDummy->transport_config = IVAS_AUDIO_CONFIG_INVALID; + masaPrerendClose( &inputMasa->hMasaPrerend ); + freeMasaExtRenderer( &inputMasa->hMasaExtRend ); - /* Todo refactor: Access to qmetadata is not required by the algorithm. */ - if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) - { - return error; - } - decDummy->hQMetaData->coherence_flag = 1; + initRendInputBase( &inputMasa->base, IVAS_AUDIO_CONFIG_INVALID, 0, rendCtx, NULL, 0 ); - if ( ( error = ivas_dirac_dec_config( decDummy, DIRAC_OPEN ) ) != IVAS_ERR_OK ) - { - return error; - } - decDummy->hSpatParamRendCom->dirac_bs_md_write_idx = 0; + return; +} - numCldfbAnalyses = decDummy->nchan_transport; - numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; - for ( i = 0; i < numCldfbAnalyses; i++ ) +#ifdef SPLIT_REND_WITH_HEAD_ROT +static ivas_error initSplitRend( + SPLIT_REND_WRAPPER *pSplitRendWrapper, + IVAS_REND_AudioBuffer *pSplitRendEncBuffer, + const IVAS_SPLIT_REND_CONFIG_DATA *pSplit_rend_config, + IVAS_REND_HeadRotData headRotData, + const int32_t outputSampleRate, + const AUDIO_CONFIG outConfig, + const int16_t cldfb_in_flag, + const int16_t is_5ms_frame ) +{ + ivas_error error; + IVAS_REND_AudioBufferConfig bufConfig; + + if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) { - if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + if ( pSplit_rend_config->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { - return error; + ivas_renderSplitGetMultiBinPoseData( pSplit_rend_config, &pSplitRendWrapper->multiBinPoseData, headRotData.sr_pose_pred_axis ); + } + else if ( pSplit_rend_config->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) + { + ivas_renderSplitUpdateNoCorrectionPoseData( pSplit_rend_config, &pSplitRendWrapper->multiBinPoseData ); } - } - for ( ; i < MAX_INTERN_CHANNELS; i++ ) - { - decDummy->cldfbAnaDec[i] = NULL; - } - for ( i = 0; i < numCldfbSyntheses; i++ ) - { - if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_split_renderer_open( pSplitRendWrapper, pSplit_rend_config, outputSampleRate, cldfb_in_flag, outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM, is_5ms_frame ) ) != IVAS_ERR_OK ) { return error; } + + /*allocate for CLDFB in and change to TD during process if needed*/ + bufConfig.numSamplesPerChannel = MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL; + bufConfig.numChannels = BINAURAL_CHANNELS * pSplitRendWrapper->multiBinPoseData.num_poses; + bufConfig.is_cldfb = 1; + pSplitRendEncBuffer->config = bufConfig; + + if ( ( pSplitRendEncBuffer->data = malloc( bufConfig.numChannels * bufConfig.numSamplesPerChannel * sizeof( float ) ) ) == NULL ) + { + return IVAS_ERR_FAILED_ALLOC; + } } - for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + else { - decDummy->cldfbSynDec[i] = NULL; + IVAS_REND_AudioBufferConfig bufConfig2; + + bufConfig2.numSamplesPerChannel = 0; + bufConfig2.numChannels = 0; + bufConfig2.is_cldfb = 0; + pSplitRendEncBuffer->config = bufConfig2; + pSplitRendEncBuffer->data = NULL; } return IVAS_ERR_OK; } +#endif -static ivas_error initMasaDummyDecForBinauralOut( - input_masa *inputMasa, - const AUDIO_CONFIG outConfig ) +/*------------------------------------------------------------------------- + * IVAS_REND_Open() + * + * + *------------------------------------------------------------------------*/ + +ivas_error IVAS_REND_Open( + IVAS_REND_HANDLE *phIvasRend, + const int32_t outputSampleRate, + const AUDIO_CONFIG outConfig, + const int16_t nonDiegeticPan, + const float nonDiegeticPanGain, + const int16_t num_subframes ) { - ivas_error error; int16_t i; - DecoderDummy *decDummy; - - decDummy = inputMasa->decDummy; - decDummy->hDecoderConfig->output_config = outConfig; - decDummy->sba_analysis_order = 1; - decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ - decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ - - ivas_output_init( &( decDummy->hOutSetup ), outConfig ); - #ifdef SPLIT_REND_WITH_HEAD_ROT - if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) -#else - if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL ) + int16_t j; #endif + IVAS_REND_HANDLE hIvasRend; + ivas_error error; + int16_t numOutChannels; + + /* Validate function arguments */ + if ( phIvasRend == NULL ) { - decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - else + + if ( ( error = validateOutputAudioConfig( outConfig ) ) != IVAS_ERR_OK ) { - decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC_ROOM; + return error; } - decDummy->ivas_format = MASA_FORMAT; - decDummy->transport_config = IVAS_AUDIO_CONFIG_INVALID; - if ( ( error = ivas_dirac_dec_config( decDummy, DIRAC_OPEN ) ) != IVAS_ERR_OK ) + if ( ( error = validateOutputSampleRate( outputSampleRate, outConfig ) ) != IVAS_ERR_OK ) { return error; } - decDummy->hSpatParamRendCom->dirac_bs_md_write_idx = 0; - if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &decDummy->hHrtfParambin ) ) != IVAS_ERR_OK ) + *phIvasRend = (IVAS_REND_HANDLE) malloc( sizeof( struct IVAS_REND ) ); + if ( *phIvasRend == NULL ) { - return error; + return IVAS_ERR_FAILED_ALLOC; } - if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy, decDummy->hHrtfParambin ) ) != IVAS_ERR_OK ) + hIvasRend = *phIvasRend; + hIvasRend->sampleRateOut = outputSampleRate; + hIvasRend->outputConfig = outConfig; + hIvasRend->customLsOut = defaultCustomLs(); + hIvasRend->hLimiter = NULL; + hIvasRend->efapOutWrapper.hEfap = NULL; + hIvasRend->efapOutWrapper.pCustomLsSetup = NULL; +#ifdef DEBUGGING + hIvasRend->numClipping = 0; +#endif + hIvasRend->num_subframes = num_subframes; + + /* Initialize limiter */ + if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { return error; } - for ( i = 0; i < BINAURAL_CHANNELS; i++ ) + if ( ( error = initLimiter( &hIvasRend->hLimiter, numOutChannels, outputSampleRate ) ) != IVAS_ERR_OK ) { - if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) - { - return error; - } - if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) - { - return error; - } + return error; } - for ( ; i < MAX_INTERN_CHANNELS; i++ ) + + /* Initialize headrotation data */ + if ( ( error = initHeadRotation( hIvasRend ) ) != IVAS_ERR_OK ) { - decDummy->cldfbAnaDec[i] = NULL; - decDummy->cldfbSynDec[i] = NULL; + return error; } - return IVAS_ERR_OK; -} - - -static ivas_error updateMasaDummyDec( - input_masa *inputMasa, - const AUDIO_CONFIG outConfig ) -{ - ivas_error error; - - switch ( getAudioConfigType( outConfig ) ) + /* Initialize external orientation data */ + if ( ( error = ivas_external_orientation_open( &( hIvasRend->hExternalOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) { - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - error = initMasaDummyDecForMcOut( inputMasa, outConfig ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - error = initMasaDummyDecForSbaOut( inputMasa, outConfig ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - error = initMasaDummyDecForBinauralOut( inputMasa, outConfig ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; + return error; } - return error; -} + /* Initilize combined orientation data */ + if ( ( error = ivas_combined_orientation_open( &( hIvasRend->hCombinedOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) + { + return error; + } + /* Initialize EFAP */ + if ( ( error = initEfap( &hIvasRend->efapOutWrapper, outConfig, &hIvasRend->customLsOut ) ) != IVAS_ERR_OK ) + { + return error; + } -static DecoderDummy *initDecoderDummy( - const int32_t sampleRate, - const int16_t numTransChannels, - const AUDIO_CONFIG outConfig, + /* Initialize inputs */ #ifdef SPLIT_REND_WITH_HEAD_ROT - const uint8_t enableRenderConfig, - const SPLIT_REND_WRAPPER *pSplitRendWrapper -#else - const uint8_t enableRenderConfig + ivas_init_split_rend_handles( &hIvasRend->splitRendWrapper ); + hIvasRend->splitRendEncBuffer.data = NULL; #endif -) -{ - ivas_error error; - int16_t i; - int16_t numOutChannels; - DecoderDummy *decDummy; - int16_t num_subframes; - num_subframes = MAX_PARAM_SPATIAL_SUBFRAMES; - if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) { - /* Checking error with assert is enough, this function is only temporary anyway */ - assert( error == IVAS_ERR_OK ); - } + initRendInputBase( &hIvasRend->inputsIsm[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - decDummy = malloc( sizeof( DecoderDummy ) ); - decDummy->hDecoderConfig = malloc( sizeof( DECODER_CONFIG ) ); - decDummy->hDecoderConfig->output_Fs = sampleRate; - decDummy->hDecoderConfig->nchan_out = numOutChannels; - decDummy->hDecoderConfig->Opt_Headrotation = 0; - decDummy->hDecoderConfig->Opt_tsm = 0; - decDummy->hDecoderConfig->Opt_5ms = 0; - decDummy->hBinRenderer = NULL; + hIvasRend->inputsIsm[i].crendWrapper = NULL; + hIvasRend->inputsIsm[i].hReverb = NULL; + hIvasRend->inputsIsm[i].tdRendWrapper.hBinRendererTd = NULL; #ifdef SPLIT_REND_WITH_HEAD_ROT - decDummy->hSplitBinRend.hSplitRendBits = NULL; - decDummy->hSplitBinRend.hMultiBinCldfbData = NULL; - ivas_init_split_rend_handles( &decDummy->hSplitBinRend.splitrend ); - decDummy->hSplitBinRend.splitrend = *pSplitRendWrapper; - - if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) - { - if ( ( decDummy->hSplitBinRend.hMultiBinCldfbData = (IVAS_DEC_SPLIT_REND_MULTI_BIN_CLDFB_DATA_HANDLE) malloc( sizeof( IVAS_DEC_SPLIT_REND_MULTI_BIN_CLDFB_DATA ) ) ) == NULL ) + for ( j = 0; j < (int16_t) ( sizeof( hIvasRend->inputsIsm[i].splitTdRendWrappers ) / sizeof( *hIvasRend->inputsIsm[i].splitTdRendWrappers ) ); ++j ) { - assert( error == IVAS_ERR_OK ); + hIvasRend->inputsIsm[i].splitTdRendWrappers[j].hBinRendererTd = NULL; + hIvasRend->inputsIsm[i].splitTdRendWrappers[j].hHrtfTD = NULL; } - } #endif - - decDummy->hEFAPdata = NULL; - decDummy->hCrendWrapper = NULL; - decDummy->hHrtfTD = NULL; - decDummy->hSpar = NULL; - decDummy->hoa_dec_mtx = NULL; - decDummy->hVBAPdata = NULL; // note: not used at the moment - decDummy->hMasa = NULL; - -#ifdef SPLIT_REND_WITH_HEAD_ROT - for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) - { - decDummy->hDiracDecBin[i] = NULL; + hIvasRend->inputsIsm[i].bufferData = NULL; + hIvasRend->inputsIsm[i].nonDiegeticPan = nonDiegeticPan; + hIvasRend->inputsIsm[i].nonDiegeticPanGain = nonDiegeticPanGain; + hIvasRend->inputsIsm[i].hOMasa = NULL; } -#else - decDummy->hDiracDecBin = NULL; -#endif - - decDummy->hDirACRend = NULL; - decDummy->hSpatParamRendCom = NULL; - decDummy->hQMetaData = NULL; - decDummy->hHrtfParambin = NULL; - decDummy->hHeadTrackData = NULL; - decDummy->hDirAC = NULL; - decDummy->hTcBuffer = NULL; - decDummy->hDecoderConfig->output_config = outConfig; - decDummy->nchan_transport = numTransChannels; - if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) { - decDummy->hHeadTrackData = malloc( sizeof( HEAD_TRACK_DATA ) ); - /* Initialise Rmat_prev to I, Rmat will be computed later */ - for ( i = 0; i < 3; i++ ) - { - set_zero( decDummy->hHeadTrackData->Rmat_prev[i], 3 ); - decDummy->hHeadTrackData->Rmat_prev[i][i] = 1.0f; - } - - set_zero( decDummy->hHeadTrackData->chEneIIR[0], MASA_FREQUENCY_BANDS ); - set_zero( decDummy->hHeadTrackData->chEneIIR[1], MASA_FREQUENCY_BANDS ); - set_zero( decDummy->hHeadTrackData->procChEneIIR[0], MASA_FREQUENCY_BANDS ); - set_zero( decDummy->hHeadTrackData->procChEneIIR[1], MASA_FREQUENCY_BANDS ); - for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) - { - decDummy->hHeadTrackData->Quaternions[i].w = 1.0f; - decDummy->hHeadTrackData->Quaternions[i].x = 0.0f; - decDummy->hHeadTrackData->Quaternions[i].y = 0.0f; - decDummy->hHeadTrackData->Quaternions[i].z = 0.0f; - } - decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; - decDummy->hHeadTrackData->lrSwitchedCurrent = 0; - decDummy->hHeadTrackData->lrSwitchedNext = 0; - decDummy->hHeadTrackData->OrientationTracker = (ivas_orient_trk_state_t *) malloc( sizeof( ivas_orient_trk_state_t ) ); - ivas_orient_trk_Init( decDummy->hHeadTrackData->OrientationTracker ); + initRendInputBase( &hIvasRend->inputsMc[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - /* External orientations */ - if ( ( error = ivas_external_orientation_open( &( decDummy->hExtOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) - { - assert( error == IVAS_ERR_OK ); - } - - /* Combined orientations */ - if ( ( error = ivas_combined_orientation_open( &( decDummy->hCombinedOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) + hIvasRend->inputsMc[i].efapInWrapper.hEfap = NULL; + hIvasRend->inputsMc[i].crendWrapper = NULL; + hIvasRend->inputsMc[i].hReverb = NULL; + hIvasRend->inputsMc[i].tdRendWrapper.hBinRendererTd = NULL; + hIvasRend->inputsMc[i].bufferData = NULL; + hIvasRend->inputsMc[i].lfeDelayBuffer = NULL; + hIvasRend->inputsMc[i].nonDiegeticPan = nonDiegeticPan; + hIvasRend->inputsMc[i].nonDiegeticPanGain = nonDiegeticPanGain; + hIvasRend->inputsMc[i].hMcMasa = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( j = 0; j < (int16_t) ( sizeof( hIvasRend->inputsMc[i].splitTdRendWrappers ) / sizeof( *hIvasRend->inputsMc[i].splitTdRendWrappers ) ); ++j ) { - assert( error == IVAS_ERR_OK ); + hIvasRend->inputsMc[i].splitTdRendWrappers[j].hBinRendererTd = NULL; + hIvasRend->inputsMc[i].splitTdRendWrappers[j].hHrtfTD = NULL; } +#endif } - else + + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) { - decDummy->hHeadTrackData = NULL; - decDummy->hExtOrientationData = NULL; - decDummy->hCombinedOrientationData = NULL; + initRendInputBase( &hIvasRend->inputsSba[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); + + hIvasRend->inputsSba[i].crendWrapper = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT + hIvasRend->inputsSba[i].cldfbRendWrapper.hCldfbRend = NULL; + hIvasRend->inputsSba[i].cldfbRendWrapper.hHrtfFastConv = NULL; +#endif + hIvasRend->inputsSba[i].bufferData = NULL; + hIvasRend->inputsSba[i].hDirAC = NULL; } - if ( enableRenderConfig ) + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) { - ivas_render_config_open( &decDummy->hRenderConfig ); + initRendInputBase( &hIvasRend->inputsMasa[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); + + hIvasRend->inputsMasa[i].metadataHasBeenFed = false; + hIvasRend->inputsMasa[i].bufferData = NULL; + hIvasRend->inputsMasa[i].hMasaPrerend = NULL; + hIvasRend->inputsMasa[i].hMasaExtRend = NULL; } - else + +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( i = 0; i < RENDERER_MAX_BIN_INPUTS; ++i ) { - decDummy->hRenderConfig = NULL; - } + initRendInputBase( &hIvasRend->inputsSplitPost[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - /* get a default TC buffer, needed for some renderers */ - ivas_jbm_dec_tc_buffer_open( decDummy, TC_BUFFER_MODE_NONE, 0, 0, 0, 1 ); + ivas_init_split_post_rend_handles( &hIvasRend->inputsSplitPost[i].splitPostRendWrapper ); - decDummy->renderer_type = RENDERER_DISABLE; +#ifdef SPLIT_REND_WITH_HEAD_ROT + hIvasRend->splitRendBFI = 0; +#endif + hIvasRend->inputsSplitPost[i].bufferData = NULL; + } +#endif - return decDummy; + return IVAS_ERR_OK; } -static ivas_error setRendInputActiveMasa( - void *input, - const AUDIO_CONFIG inConfig, - const IVAS_REND_InputId id, - RENDER_CONFIG_DATA *hRendCfg ) /* Todo: This is not used at all within MASA. Support might be better to do after refactoring. */ +static LSSETUP_CUSTOM_STRUCT makeCustomLsSetup( + const IVAS_CUSTOM_LS_DATA rendCustomLsLayout ) { - ivas_error error; - rendering_context rendCtx; - AUDIO_CONFIG outConfig; - input_masa *inputMasa; - int16_t numInChannels; + int16_t i; + LSSETUP_CUSTOM_STRUCT customLs; - inputMasa = (input_masa *) input; - rendCtx = inputMasa->base.ctx; - outConfig = *rendCtx.pOutConfig; - (void) hRendCfg; /* Suppress warning */ + /* Copy layout description */ + customLs.num_spk = rendCustomLsLayout.num_spk; + mvr2r( rendCustomLsLayout.azimuth, customLs.ls_azimuth, rendCustomLsLayout.num_spk ); + mvr2r( rendCustomLsLayout.elevation, customLs.ls_elevation, rendCustomLsLayout.num_spk ); - if ( !isIoConfigPairSupported( inConfig, outConfig ) ) + customLs.is_planar_setup = 1; + for ( i = 0; i < rendCustomLsLayout.num_spk; ++i ) { - return IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED; + if ( fabsf( rendCustomLsLayout.elevation[i] ) > EPSILON ) + { + customLs.is_planar_setup = 0; + break; + } } - if ( ( error = allocateInputBaseBufferData( &inputMasa->bufferData, MAX_BUFFER_LENGTH ) ) != IVAS_ERR_OK ) - { - return error; - } - initRendInputBase( &inputMasa->base, inConfig, id, rendCtx, inputMasa->bufferData, MAX_BUFFER_LENGTH ); + customLs.num_lfe = rendCustomLsLayout.num_lfe; + mvs2s( rendCustomLsLayout.lfe_idx, customLs.lfe_idx, rendCustomLsLayout.num_lfe ); - if ( ( error = getAudioConfigNumChannels( inConfig, &numInChannels ) ) != IVAS_ERR_OK ) + return customLs; +} + + +static ivas_error validateCustomLsLayout( + const IVAS_CUSTOM_LS_DATA layout ) +{ + int16_t i; + + /* Negative number of speakers or LFEs makes no sense */ + if ( layout.num_spk < 0 || layout.num_lfe < 0 ) { - return error; + return IVAS_ERR_INVALID_CUSTOM_LS_LAYOUT; } - if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + /* There must be at least one speaker or LFE in the layout */ + if ( layout.num_spk + layout.num_lfe <= 0 ) { - inputMasa->metadataHasBeenFed = false; - if ( ( error = masaPrerendOpen( &inputMasa->hMasaPrerend, inputMasa->base.inConfig == IVAS_AUDIO_CONFIG_MASA1 ? 1 : 2, *( inputMasa->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERR_INVALID_CUSTOM_LS_LAYOUT; } - else - { -#ifdef SPLIT_REND_WITH_HEAD_ROT - inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, numInChannels, outConfig, 0, rendCtx.pSplitRendWrapper ); -#else - inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, numInChannels, outConfig, 0 ); -#endif - - inputMasa->metadataHasBeenFed = false; - if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) + /* LFE indices must be positive */ + for ( i = 0; i < layout.num_lfe; ++i ) + { + if ( layout.lfe_idx[i] < 0 ) { - return error; + return IVAS_ERR_INVALID_CUSTOM_LS_LAYOUT; } } @@ -3376,419 +3309,490 @@ static ivas_error setRendInputActiveMasa( } -static void freeDecoderDummy( - DecoderDummy **ppDecDummy ) +/*-------------------------------------------------------------------* + * IVAS_REND_ConfigureCustomOutputLoudspeakerLayout() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( + IVAS_REND_HANDLE hIvasRend, + const IVAS_CUSTOM_LS_DATA layout ) { - int16_t i; - DecoderDummy *pDecDummy; + int16_t i, numOutChannels; + ivas_error error; + input_mc *inputMc; + input_sba *inputSba; - if ( ppDecDummy == NULL || *ppDecDummy == NULL ) + /* Validate function arguments */ + if ( hIvasRend == NULL ) { - return; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - pDecDummy = *ppDecDummy; - if ( pDecDummy->hDecoderConfig != NULL ) + if ( hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) { - free( pDecDummy->hDecoderConfig ); + /* Specifying details of custom speaker layout only makes sense if output config is set to custom speaker layout */ + return IVAS_ERR_INVALID_OUTPUT_FORMAT; } - if ( pDecDummy->hHeadTrackData != NULL ) + if ( ( error = validateCustomLsLayout( layout ) ) != IVAS_ERR_OK ) { - if ( pDecDummy->hHeadTrackData->OrientationTracker != NULL ) - { - free( pDecDummy->hHeadTrackData->OrientationTracker ); - } - free( pDecDummy->hHeadTrackData ); + return error; } - if ( pDecDummy->hExtOrientationData != NULL ) + hIvasRend->customLsOut = makeCustomLsSetup( layout ); + + /* Re-initialize limiter - number of output channels may have changed */ + if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { - free( pDecDummy->hExtOrientationData ); + return error; } - if ( pDecDummy->hCombinedOrientationData != NULL ) + if ( ( error = initLimiter( &hIvasRend->hLimiter, numOutChannels, hIvasRend->sampleRateOut ) ) != IVAS_ERR_OK ) { - free( pDecDummy->hCombinedOrientationData ); + return error; } - ivas_render_config_close( &pDecDummy->hRenderConfig ); + /* Re-initialize EFAP - output layout has changed or has been fully defined for the first time */ + if ( ( error = initEfap( &hIvasRend->efapOutWrapper, hIvasRend->outputConfig, &hIvasRend->customLsOut ) ) != IVAS_ERR_OK ) + { + return error; + } - /* CLDFB handles */ - for ( i = 0; i < MAX_INTERN_CHANNELS; i++ ) + /* Re-initialize panning gains for each active MC input, This includes re-initializing + * LFE handling for the new output layout, which means custom LFE handling is overwritten, + * if previously set for any MC input. */ + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) { - if ( pDecDummy->cldfbAnaDec[i] != NULL ) + inputMc = &hIvasRend->inputsMc[i]; + if ( inputMc->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + /* Input inactive, skip. */ + continue; + } + + inputMc->lfeRouting = defaultLfeRouting( inputMc->base.inConfig, inputMc->customLsInput, hIvasRend->outputConfig, *inputMc->base.ctx.pCustomLsOut ); + + if ( ( error = updateMcPanGains( inputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) { - deleteCldfb( &( pDecDummy->cldfbAnaDec[i] ) ); + return error; } } - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + /* Re-initialize panning gains for each active SBA input */ + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) { - if ( pDecDummy->cldfbSynDec[i] != NULL ) + inputSba = &hIvasRend->inputsSba[i]; + + if ( inputSba->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + /* Input inactive, skip. */ + continue; + } + + if ( ( error = updateSbaPanGains( inputSba, hIvasRend->outputConfig, hIvasRend->hRendererConfig ) ) != IVAS_ERR_OK ) { - deleteCldfb( &( pDecDummy->cldfbSynDec[i] ) ); + return error; } } - /* DirAC handle */ - ivas_dirac_rend_close( &( pDecDummy->hDirACRend ) ); - ivas_spat_hSpatParamRendCom_close( &( pDecDummy->hSpatParamRendCom ) ); - ivas_dirac_dec_close( &( pDecDummy->hDirAC ) ); + return IVAS_ERR_OK; +} - /* Qmetadata handle */ - ivas_qmetadata_close( &pDecDummy->hQMetaData ); - /* VBAP handle */ - vbap_free_data( &( pDecDummy->hVBAPdata ) ); +/*-------------------------------------------------------------------* + * IVAS_REND_NumOutChannels() + * + * + *-------------------------------------------------------------------*/ - /* HOA decoder matrix */ - if ( pDecDummy->hoa_dec_mtx != NULL ) - { - free( pDecDummy->hoa_dec_mtx ); - pDecDummy->hoa_dec_mtx = NULL; - } +ivas_error IVAS_REND_NumOutChannels( + IVAS_REND_CONST_HANDLE hIvasRend, + int16_t *numOutChannels ) +{ + ivas_error error; - /* Parametric binaural renderer HRTF structure */ - if ( pDecDummy->hHrtfParambin != NULL ) + /* Validate function arguments */ + if ( hIvasRend == NULL || numOutChannels == NULL ) { - free( pDecDummy->hHrtfParambin ); - pDecDummy->hHrtfParambin = NULL; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } -#ifdef SPLIT_REND_WITH_HEAD_ROT - /* Split binaural renderr handle */ - if ( pDecDummy->hSplitBinRend.hMultiBinCldfbData != NULL ) + /* Handle special cases where additional info is needed from the renderer, otherwise use getAudioConfigNumChannels() */ + switch ( hIvasRend->outputConfig ) { - free( pDecDummy->hSplitBinRend.hMultiBinCldfbData ); - pDecDummy->hSplitBinRend.hMultiBinCldfbData = NULL; + case IVAS_AUDIO_CONFIG_LS_CUSTOM: + *numOutChannels = hIvasRend->customLsOut.num_spk + hIvasRend->customLsOut.num_lfe; + break; + default: + if ( ( error = getAudioConfigNumChannels( hIvasRend->outputConfig, numOutChannels ) ) != IVAS_ERR_OK ) + { + return error; + } + break; } -#endif - - /* Parametric binaural renderer handle */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - ivas_dirac_dec_close_binaural_data( pDecDummy->hDiracDecBin ); -#else - ivas_dirac_dec_close_binaural_data( &pDecDummy->hDiracDecBin ); -#endif - - /* TC buffer */ - ivas_jbm_dec_tc_buffer_close( &pDecDummy->hTcBuffer ); - free( pDecDummy ); - pDecDummy = NULL; - - return; + return IVAS_ERR_OK; } -static void clearInputMasa( - input_masa *inputMasa ) +static IVAS_REND_InputId makeInputId( + AUDIO_CONFIG config, + const int32_t inputIndex ) { - rendering_context rendCtx; - - rendCtx = inputMasa->base.ctx; - - freeInputBaseBufferData( &inputMasa->bufferData ); - - masaPrerendClose( &inputMasa->hMasaPrerend ); + /* Put config type in second byte (from LSB), put index + 1 in first byte + * + * Index is incremented here so that a valid ID can never be 0. */ + return (IVAS_REND_InputId) ( ( ( (uint32_t) getAudioConfigType( config ) ) << 8 ) | ( inputIndex + 1 ) ); +} - initRendInputBase( &inputMasa->base, IVAS_AUDIO_CONFIG_INVALID, 0, rendCtx, NULL, 0 ); - freeDecoderDummy( &inputMasa->decDummy ); - return; -} +static ivas_error getInputById( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_InputId inputId, + void **ppInput ) +{ + int32_t inputIndex; + IVAS_REND_AudioConfigType configType; + input_base *pInputBase; + /* Reverse makeInputId() */ + inputIndex = ( inputId & 0xFF ) - 1; + configType = ( inputId & 0xFF00 ) >> 8; -#ifdef SPLIT_REND_WITH_HEAD_ROT -static ivas_error initSplitRend( - SPLIT_REND_WRAPPER *pSplitRendWrapper, - IVAS_REND_AudioBuffer *pSplitRendEncBuffer, - const IVAS_SPLIT_REND_CONFIG_DATA *pSplit_rend_config, - IVAS_REND_HeadRotData headRotData, - const int32_t outputSampleRate, - const AUDIO_CONFIG outConfig, - const int16_t cldfb_in_flag, - const int16_t is_5ms_frame ) -{ - ivas_error error; - IVAS_REND_AudioBufferConfig bufConfig; - - if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + /* Validate values derived from input ID */ + if ( inputIndex < 0 ) { - if ( pSplit_rend_config->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) - { - ivas_renderSplitGetMultiBinPoseData( pSplit_rend_config, &pSplitRendWrapper->multiBinPoseData, headRotData.sr_pose_pred_axis ); - } - else if ( pSplit_rend_config->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) - { - ivas_renderSplitUpdateNoCorrectionPoseData( pSplit_rend_config, &pSplitRendWrapper->multiBinPoseData ); - } - - if ( ( error = ivas_split_renderer_open( pSplitRendWrapper, pSplit_rend_config, outputSampleRate, cldfb_in_flag, outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM, is_5ms_frame ) ) != IVAS_ERR_OK ) - { - return error; - } - - /*allocate for CLDFB in and change to TD during process if needed*/ - bufConfig.numSamplesPerChannel = MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL; - bufConfig.numChannels = BINAURAL_CHANNELS * pSplitRendWrapper->multiBinPoseData.num_poses; - bufConfig.is_cldfb = 1; - pSplitRendEncBuffer->config = bufConfig; - - if ( ( pSplitRendEncBuffer->data = malloc( bufConfig.numChannels * bufConfig.numSamplesPerChannel * sizeof( float ) ) ) == NULL ) - { - return IVAS_ERR_FAILED_ALLOC; - } + return IVAS_ERR_INVALID_INPUT_ID; } - else + switch ( configType ) { - IVAS_REND_AudioBufferConfig bufConfig2; + case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: + if ( inputIndex > RENDERER_MAX_ISM_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsIsm[inputIndex].base; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + if ( inputIndex > RENDERER_MAX_MC_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMc[inputIndex].base; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + if ( inputIndex > RENDERER_MAX_SBA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsSba[inputIndex].base; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + break; +#ifdef SPLIT_REND_WITH_HEAD_ROT + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + if ( inputIndex > RENDERER_MAX_BIN_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsSplitPost[inputIndex].base; + break; +#endif + default: + return IVAS_ERR_INVALID_INPUT_ID; + } - bufConfig2.numSamplesPerChannel = 0; - bufConfig2.numChannels = 0; - bufConfig2.is_cldfb = 0; - pSplitRendEncBuffer->config = bufConfig2; - pSplitRendEncBuffer->data = NULL; + /* Ensure input ID matches and that input is active */ + if ( pInputBase->id != inputId || pInputBase->inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + return IVAS_ERR_INVALID_INPUT_ID; } + /* Validation done, set value via output parameter */ + *ppInput = pInputBase; + return IVAS_ERR_OK; } -#endif -/*------------------------------------------------------------------------- - * IVAS_REND_Open() - * - * - *------------------------------------------------------------------------*/ - -ivas_error IVAS_REND_Open( - IVAS_REND_HANDLE *phIvasRend, - const int32_t outputSampleRate, - const AUDIO_CONFIG outConfig, - const int16_t nonDiegeticPan, - const float nonDiegeticPanGain, - const int16_t num_subframes ) +static ivas_error getConstInputById( + IVAS_REND_CONST_HANDLE hIvasRend, + const IVAS_REND_InputId inputId, + const void **ppInput ) { - int16_t i; -#ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t j; -#endif - IVAS_REND_HANDLE hIvasRend; - ivas_error error; - int16_t numOutChannels; + int32_t inputIndex; + IVAS_REND_AudioConfigType configType; + const input_base *pInputBase; - /* Validate function arguments */ - if ( phIvasRend == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + /* Reverse makeInputId() */ + inputIndex = ( inputId & 0xFF ) - 1; + configType = ( inputId & 0xFF00 ) >> 8; - if ( ( error = validateOutputAudioConfig( outConfig ) ) != IVAS_ERR_OK ) + /* Validate values derived from input ID */ + if ( inputIndex < 0 ) { - return error; + return IVAS_ERR_INVALID_INPUT_ID; } - - if ( ( error = validateOutputSampleRate( outputSampleRate, outConfig ) ) != IVAS_ERR_OK ) + switch ( configType ) { - return error; + case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: + if ( inputIndex > RENDERER_MAX_ISM_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsIsm[inputIndex].base; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + if ( inputIndex > RENDERER_MAX_MC_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMc[inputIndex].base; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + if ( inputIndex > RENDERER_MAX_SBA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsSba[inputIndex].base; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + break; +#ifdef SPLIT_REND_WITH_HEAD_ROT + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + if ( inputIndex > RENDERER_MAX_BIN_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsSplitPost[inputIndex].base; + break; +#endif + default: + return IVAS_ERR_INVALID_INPUT_ID; } - *phIvasRend = (IVAS_REND_HANDLE) malloc( sizeof( struct IVAS_REND ) ); - if ( *phIvasRend == NULL ) + /* Ensure input ID matches and that input is active */ + if ( pInputBase->id != inputId || pInputBase->inConfig == IVAS_AUDIO_CONFIG_INVALID ) { - return IVAS_ERR_FAILED_ALLOC; + return IVAS_ERR_INVALID_INPUT_ID; } - hIvasRend = *phIvasRend; - hIvasRend->sampleRateOut = outputSampleRate; - hIvasRend->outputConfig = outConfig; - hIvasRend->customLsOut = defaultCustomLs(); - hIvasRend->hLimiter = NULL; - hIvasRend->efapOutWrapper.hEfap = NULL; - hIvasRend->efapOutWrapper.pCustomLsSetup = NULL; -#ifdef DEBUGGING - hIvasRend->numClipping = 0; -#endif - hIvasRend->num_subframes = num_subframes; + /* Validation done, set value via output parameter */ + *ppInput = pInputBase; - /* Initialize limiter */ - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERR_OK; +} - if ( ( error = initLimiter( &hIvasRend->hLimiter, numOutChannels, outputSampleRate ) ) != IVAS_ERR_OK ) - { - return error; - } - /* Initialize headrotation data */ - if ( ( error = initHeadRotation( hIvasRend ) ) != IVAS_ERR_OK ) - { - return error; - } +static ivas_error findFreeInputSlot( + const void *inputs, + const int32_t inputStructSize, + const int32_t maxInputs, + int32_t *inputIndex ) +{ + /* Using a void pointer and a separately provided size is a hack for this function + to be reusable for arrays of any input type (input_ism, input_mc, input_sba, input_masa). + Assumptions: + - input_base is always the first member in the input struct + - provided size is correct + */ - /* Initialize external orientation data */ - if ( ( error = ivas_external_orientation_open( &( hIvasRend->hExternalOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) + int32_t i; + bool canAddInput; + const uint8_t *pByte; + const input_base *pInputBase; + + canAddInput = false; + + /* Find first unused input in array */ + for ( i = 0, pByte = inputs; i < maxInputs; ++i, pByte += inputStructSize ) { - return error; + pInputBase = (const input_base *) pByte; + + if ( pInputBase->inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + *inputIndex = i; + canAddInput = true; + break; + } } - /* Initilize combined orientation data */ - if ( ( error = ivas_combined_orientation_open( &( hIvasRend->hCombinedOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) + if ( !canAddInput ) { - return error; + return IVAS_ERR_TOO_MANY_INPUTS; } - /* Initialize EFAP */ - if ( ( error = initEfap( &hIvasRend->efapOutWrapper, outConfig, &hIvasRend->customLsOut ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERR_OK; +} + - /* Initialize inputs */ #ifdef SPLIT_REND_WITH_HEAD_ROT - ivas_init_split_rend_handles( &hIvasRend->splitRendWrapper ); - hIvasRend->splitRendEncBuffer.data = NULL; -#endif +static int16_t getCldfbRendFlag( + IVAS_REND_HANDLE hIvasRend, /* i : Renderer handle */ + const IVAS_REND_AudioConfigType new_configType ) +{ + int16_t i; + int16_t numMasaInputs = 0, numSbaInputs = 0, numIsmInputs = 0, numMcInputs = 0; + int16_t isCldfbRend; - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) + isCldfbRend = 0; + if ( hIvasRend->hRendererConfig != NULL ) { - initRendInputBase( &hIvasRend->inputsIsm[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - - hIvasRend->inputsIsm[i].crendWrapper = NULL; - hIvasRend->inputsIsm[i].hReverb = NULL; - hIvasRend->inputsIsm[i].tdRendWrapper.hBinRendererTd = NULL; -#ifdef SPLIT_REND_WITH_HEAD_ROT - for ( j = 0; j < (int16_t) ( sizeof( hIvasRend->inputsIsm[i].splitTdRendWrappers ) / sizeof( *hIvasRend->inputsIsm[i].splitTdRendWrappers ) ); ++j ) + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) { - hIvasRend->inputsIsm[i].splitTdRendWrappers[j].hBinRendererTd = NULL; - hIvasRend->inputsIsm[i].splitTdRendWrappers[j].hHrtfTD = NULL; + numMasaInputs += ( hIvasRend->inputsMasa[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) ? 0 : 1; + } + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) + { + numSbaInputs += ( hIvasRend->inputsSba[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) ? 0 : 1; + } + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) + { + numIsmInputs += ( hIvasRend->inputsIsm[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) ? 0 : 1; + } + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) + { + numMcInputs += ( hIvasRend->inputsMc[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? 0 : 1; } -#endif - hIvasRend->inputsIsm[i].bufferData = NULL; - hIvasRend->inputsIsm[i].nonDiegeticPan = nonDiegeticPan; - hIvasRend->inputsIsm[i].nonDiegeticPanGain = nonDiegeticPanGain; - hIvasRend->inputsIsm[i].hOMasa = NULL; - } - - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) - { - initRendInputBase( &hIvasRend->inputsMc[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - hIvasRend->inputsMc[i].efapInWrapper.hEfap = NULL; - hIvasRend->inputsMc[i].crendWrapper = NULL; - hIvasRend->inputsMc[i].hReverb = NULL; - hIvasRend->inputsMc[i].tdRendWrapper.hBinRendererTd = NULL; - hIvasRend->inputsMc[i].bufferData = NULL; - hIvasRend->inputsMc[i].lfeDelayBuffer = NULL; - hIvasRend->inputsMc[i].nonDiegeticPan = nonDiegeticPan; - hIvasRend->inputsMc[i].nonDiegeticPanGain = nonDiegeticPanGain; - hIvasRend->inputsMc[i].hMcMasa = NULL; -#ifdef SPLIT_REND_WITH_HEAD_ROT - for ( j = 0; j < (int16_t) ( sizeof( hIvasRend->inputsMc[i].splitTdRendWrappers ) / sizeof( *hIvasRend->inputsMc[i].splitTdRendWrappers ) ); ++j ) + if ( numIsmInputs > 0 || numMcInputs > 0 ) { - hIvasRend->inputsMc[i].splitTdRendWrappers[j].hBinRendererTd = NULL; - hIvasRend->inputsMc[i].splitTdRendWrappers[j].hHrtfTD = NULL; + isCldfbRend = 0; + } + else if ( ( numMasaInputs > 0 ) || ( numSbaInputs > 0 && hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) ) + { + isCldfbRend = 1; } -#endif } - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) - { - initRendInputBase( &hIvasRend->inputsSba[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - - hIvasRend->inputsSba[i].crendWrapper = NULL; -#ifdef SPLIT_REND_WITH_HEAD_ROT - hIvasRend->inputsSba[i].cldfbRendWrapper.hCldfbRend = NULL; - hIvasRend->inputsSba[i].cldfbRendWrapper.hHrtfFastConv = NULL; -#endif - hIvasRend->inputsSba[i].bufferData = NULL; - hIvasRend->inputsSba[i].hDirAC = NULL; - } + return isCldfbRend; +} - for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) - { - initRendInputBase( &hIvasRend->inputsMasa[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); - hIvasRend->inputsMasa[i].decDummy = NULL; - hIvasRend->inputsMasa[i].metadataHasBeenFed = false; - hIvasRend->inputsMasa[i].bufferData = NULL; - hIvasRend->inputsMasa[i].hMasaPrerend = NULL; - } +static void closeSplitRend( + SPLIT_REND_WRAPPER *pSplitRendWrapper, + IVAS_REND_AudioBuffer *pSplitRendEncBuffer ) +{ + ivas_split_renderer_close( pSplitRendWrapper ); -#ifdef SPLIT_REND_WITH_HEAD_ROT - for ( i = 0; i < RENDERER_MAX_BIN_INPUTS; ++i ) + if ( pSplitRendEncBuffer->data != NULL ) { - initRendInputBase( &hIvasRend->inputsSplitPost[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); + free( pSplitRendEncBuffer->data ); + pSplitRendEncBuffer->data = NULL; + } - ivas_init_split_post_rend_handles( &hIvasRend->inputsSplitPost[i].splitPostRendWrapper ); + pSplitRendEncBuffer->config.numChannels = 0; + pSplitRendEncBuffer->config.numSamplesPerChannel = 0; -#ifdef SPLIT_REND_WITH_HEAD_ROT - hIvasRend->splitRendBFI = 0; -#endif - hIvasRend->inputsSplitPost[i].bufferData = NULL; - } + return; +} #endif - return IVAS_ERR_OK; -} +/*-------------------------------------------------------------------* + * IVAS_REND_AddInput() + * + * + *-------------------------------------------------------------------*/ -static LSSETUP_CUSTOM_STRUCT makeCustomLsSetup( - const IVAS_CUSTOM_LS_DATA rendCustomLsLayout ) +ivas_error IVAS_REND_AddInput( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const AUDIO_CONFIG inConfig, /* i : audio config for a new input */ + IVAS_REND_InputId *inputId /* o : ID of the new input */ +) { - int16_t i; - LSSETUP_CUSTOM_STRUCT customLs; - - /* Copy layout description */ - customLs.num_spk = rendCustomLsLayout.num_spk; - mvr2r( rendCustomLsLayout.azimuth, customLs.ls_azimuth, rendCustomLsLayout.num_spk ); - mvr2r( rendCustomLsLayout.elevation, customLs.ls_elevation, rendCustomLsLayout.num_spk ); + ivas_error error; + int32_t maxNumInputsOfType; + void *inputsArray; + int32_t inputStructSize; + ivas_error ( *activateInput )( void *, AUDIO_CONFIG, IVAS_REND_InputId, RENDER_CONFIG_DATA * ); + int32_t inputIndex; - customLs.is_planar_setup = 1; - for ( i = 0; i < rendCustomLsLayout.num_spk; ++i ) + /* Validate function arguments */ + if ( hIvasRend == NULL || inputId == NULL ) { - if ( fabsf( rendCustomLsLayout.elevation[i] ) > EPSILON ) - { - customLs.is_planar_setup = 0; - break; - } + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - customLs.num_lfe = rendCustomLsLayout.num_lfe; - mvs2s( rendCustomLsLayout.lfe_idx, customLs.lfe_idx, rendCustomLsLayout.num_lfe ); - - return customLs; -} +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) && hIvasRend->splitRendEncBuffer.data == NULL && hIvasRend->hRendererConfig != NULL ) + { + int16_t cldfb_in_flag; + cldfb_in_flag = getCldfbRendFlag( hIvasRend, getAudioConfigType( inConfig ) ); + if ( ( error = ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, &hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, cldfb_in_flag, hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) != IVAS_ERR_OK ) + { + return error; + } -static ivas_error validateCustomLsLayout( - const IVAS_CUSTOM_LS_DATA layout ) -{ - int16_t i; + if ( ( error = initSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, ( hIvasRend->num_subframes == 1 ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif - /* Negative number of speakers or LFEs makes no sense */ - if ( layout.num_spk < 0 || layout.num_lfe < 0 ) + switch ( getAudioConfigType( inConfig ) ) { - return IVAS_ERR_INVALID_CUSTOM_LS_LAYOUT; + case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: + maxNumInputsOfType = RENDERER_MAX_ISM_INPUTS; + inputsArray = hIvasRend->inputsIsm; + inputStructSize = sizeof( *hIvasRend->inputsIsm ); + activateInput = setRendInputActiveIsm; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + maxNumInputsOfType = RENDERER_MAX_MC_INPUTS; + inputsArray = hIvasRend->inputsMc; + inputStructSize = sizeof( *hIvasRend->inputsMc ); + activateInput = setRendInputActiveMc; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + maxNumInputsOfType = RENDERER_MAX_SBA_INPUTS; + inputsArray = hIvasRend->inputsSba; + inputStructSize = sizeof( *hIvasRend->inputsSba ); + activateInput = setRendInputActiveSba; + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + maxNumInputsOfType = RENDERER_MAX_MASA_INPUTS; + inputsArray = hIvasRend->inputsMasa; + inputStructSize = sizeof( *hIvasRend->inputsMasa ); + activateInput = setRendInputActiveMasa; + break; +#ifdef SPLIT_REND_WITH_HEAD_ROT + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + maxNumInputsOfType = RENDERER_MAX_BIN_INPUTS; + inputsArray = hIvasRend->inputsSplitPost; + inputStructSize = sizeof( *hIvasRend->inputsSplitPost ); + activateInput = setRendInputActiveSplitPostRend; + break; +#endif + default: + return IVAS_ERR_INVALID_INPUT_FORMAT; } - /* There must be at least one speaker or LFE in the layout */ - if ( layout.num_spk + layout.num_lfe <= 0 ) + /* Find first free input in array corresponding to input type */ + if ( ( error = findFreeInputSlot( inputsArray, inputStructSize, maxNumInputsOfType, &inputIndex ) ) != IVAS_ERR_OK ) { - return IVAS_ERR_INVALID_CUSTOM_LS_LAYOUT; + return error; } - /* LFE indices must be positive */ - for ( i = 0; i < layout.num_lfe; ++i ) + *inputId = makeInputId( inConfig, inputIndex ); + + if ( ( error = activateInput( (uint8_t *) inputsArray + inputStructSize * inputIndex, inConfig, *inputId, hIvasRend->hRendererConfig ) ) != IVAS_ERR_OK ) { - if ( layout.lfe_idx[i] < 0 ) - { - return IVAS_ERR_INVALID_CUSTOM_LS_LAYOUT; - } + return error; } return IVAS_ERR_OK; @@ -3796,19 +3800,20 @@ static ivas_error validateCustomLsLayout( /*-------------------------------------------------------------------* - * IVAS_REND_ConfigureCustomOutputLoudspeakerLayout() + * IVAS_REND_ConfigureCustomInputLoudspeakerLayout() * * + * Note: this will reset any custom LFE routing set for the input *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( - IVAS_REND_HANDLE hIvasRend, - const IVAS_CUSTOM_LS_DATA layout ) +ivas_error IVAS_REND_ConfigureCustomInputLoudspeakerLayout( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_CUSTOM_LS_DATA layout /* i : custom loudspeaker layout for input */ +) { - int16_t i, numOutChannels; - ivas_error error; input_mc *inputMc; - input_sba *inputSba; + ivas_error error; /* Validate function arguments */ if ( hIvasRend == NULL ) @@ -3816,559 +3821,520 @@ ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) - { - /* Specifying details of custom speaker layout only makes sense if output config is set to custom speaker layout */ - return IVAS_ERR_INVALID_OUTPUT_FORMAT; - } - if ( ( error = validateCustomLsLayout( layout ) ) != IVAS_ERR_OK ) { return error; } - hIvasRend->customLsOut = makeCustomLsSetup( layout ); - - /* Re-initialize limiter - number of output channels may have changed */ - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputMc ) ) != IVAS_ERR_OK ) { return error; } - if ( ( error = initLimiter( &hIvasRend->hLimiter, numOutChannels, hIvasRend->sampleRateOut ) ) != IVAS_ERR_OK ) + if ( inputMc->base.inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) { - return error; + /* Specifying details of custom speaker layout only makes sense if input config is set to custom speaker layout */ + return IVAS_ERR_INVALID_INPUT_FORMAT; } - /* Re-initialize EFAP - output layout has changed or has been fully defined for the first time */ - if ( ( error = initEfap( &hIvasRend->efapOutWrapper, hIvasRend->outputConfig, &hIvasRend->customLsOut ) ) != IVAS_ERR_OK ) + /* Re-initialize panning gains for the MC input, This includes re-initializing LFE handling + * for the new input layout, which means custom LFE handling is overwritten, if previously + * set for the MC input. */ + inputMc->customLsInput = makeCustomLsSetup( layout ); + + inputMc->lfeRouting = defaultLfeRouting( inputMc->base.inConfig, inputMc->customLsInput, hIvasRend->outputConfig, *inputMc->base.ctx.pCustomLsOut ); + + if ( ( error = initEfap( &inputMc->efapInWrapper, inputMc->base.inConfig, &inputMc->customLsInput ) ) != IVAS_ERR_OK ) { return error; } - /* Re-initialize panning gains for each active MC input, This includes re-initializing - * LFE handling for the new output layout, which means custom LFE handling is overwritten, - * if previously set for any MC input. */ - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) +#else + if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) +#endif { - inputMc = &hIvasRend->inputsMc[i]; - if ( inputMc->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - /* Input inactive, skip. */ - continue; - } - - inputMc->lfeRouting = defaultLfeRouting( inputMc->base.inConfig, inputMc->customLsInput, hIvasRend->outputConfig, *inputMc->base.ctx.pCustomLsOut ); - - if ( ( error = updateMcPanGains( inputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) + if ( ( error = initMcBinauralRendering( inputMc, inputMc->base.inConfig, hIvasRend->outputConfig, hIvasRend->hRendererConfig, FALSE ) ) != IVAS_ERR_OK ) { return error; } } - /* Re-initialize panning gains for each active SBA input */ - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) + if ( ( error = updateMcPanGains( inputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) { - inputSba = &hIvasRend->inputsSba[i]; + return error; + } - if ( inputSba->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - /* Input inactive, skip. */ - continue; - } + return IVAS_ERR_OK; +} - if ( ( error = updateSbaPanGains( inputSba, hIvasRend->outputConfig, hIvasRend->hRendererConfig ) ) != IVAS_ERR_OK ) - { - return error; - } + +/*-------------------------------------------------------------------* + * IVAS_REND_SetInputGain() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_SetInputGain( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const float gain /* i : linear gain (not in dB) */ +) +{ + input_base *inputBase; + ivas_error error; + + /* Validate function arguments */ + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + { + return error; + } + + inputBase->gain = gain; + return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_NumOutChannels() + * IVAS_REND_SetInputLfeMtx() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_NumOutChannels( - IVAS_REND_CONST_HANDLE hIvasRend, - int16_t *numOutChannels ) +ivas_error IVAS_REND_SetInputLfeMtx( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_REND_LfePanMtx *lfePanMtx /* i : LFE panning matrix */ +) { + int16_t i; + input_base *pInputBase; + input_mc *pInputMc; ivas_error error; /* Validate function arguments */ - if ( hIvasRend == NULL || numOutChannels == NULL ) + if ( hIvasRend == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - /* Handle special cases where additional info is needed from the renderer, otherwise use getAudioConfigNumChannels() */ - switch ( hIvasRend->outputConfig ) + if ( ( error = getInputById( hIvasRend, inputId, (void **) &pInputBase ) ) != IVAS_ERR_OK ) { - case IVAS_AUDIO_CONFIG_LS_CUSTOM: - *numOutChannels = hIvasRend->customLsOut.num_spk + hIvasRend->customLsOut.num_lfe; - break; - default: - if ( ( error = getAudioConfigNumChannels( hIvasRend->outputConfig, numOutChannels ) ) != IVAS_ERR_OK ) - { - return error; - } - break; + return error; + } + + if ( getAudioConfigType( pInputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + /* Custom LFE panning matrix only makes sense with channel-based input */ + return IVAS_ERR_INVALID_INPUT_FORMAT; + } + pInputMc = (input_mc *) pInputBase; + + /* copy LFE panning matrix */ + for ( i = 0; i < IVAS_MAX_INPUT_LFE_CHANNELS; i++ ) + { + mvr2r( ( *lfePanMtx )[i], pInputMc->lfeRouting.lfePanMtx[i], IVAS_MAX_OUTPUT_CHANNELS ); + } + + if ( ( error = updateMcPanGains( pInputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) + { + return error; } return IVAS_ERR_OK; } -static IVAS_REND_InputId makeInputId( - AUDIO_CONFIG config, - const int32_t inputIndex ) +/*-------------------------------------------------------------------* + * IVAS_REND_SetInputLfePos() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_SetInputLfePos( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const float inputGain, /* i : Input gain to be applied to the LFE channel(s) */ + const float outputAzimuth, /* i : Output azimuth position */ + const float outputElevation /* i : Output elevation position */ +) { - /* Put config type in second byte (from LSB), put index + 1 in first byte - * - * Index is incremented here so that a valid ID can never be 0. */ - return (IVAS_REND_InputId) ( ( ( (uint32_t) getAudioConfigType( config ) ) << 8 ) | ( inputIndex + 1 ) ); + input_base *pInputBase; + input_mc *pInputMc; + ivas_error error; + + /* Validate function arguments */ + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + if ( ( error = getInputById( hIvasRend, inputId, (void **) &pInputBase ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( getAudioConfigType( pInputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + /* Custom LFE routing only makes sense with channel-based input */ + return IVAS_ERR_INVALID_INPUT_FORMAT; + } + pInputMc = (input_mc *) pInputBase; + + pInputMc->lfeRouting.pan_lfe = true; + pInputMc->lfeRouting.lfeInputGain = inputGain; + pInputMc->lfeRouting.lfeOutputAzimuth = outputAzimuth; + pInputMc->lfeRouting.lfeOutputElevation = outputElevation; + + if ( ( error = updateMcPanGains( pInputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; } -static ivas_error getInputById( - IVAS_REND_HANDLE hIvasRend, - IVAS_REND_InputId inputId, - void **ppInput ) +/*-------------------------------------------------------------------* + * IVAS_REND_RemoveInput() + * + * + *-------------------------------------------------------------------*/ +/* ToDo; unused function */ +ivas_error IVAS_REND_RemoveInput( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId /* i : ID of the input */ +) { - int32_t inputIndex; - IVAS_REND_AudioConfigType configType; - input_base *pInputBase; + ivas_error error; + input_base *inputBase; - /* Reverse makeInputId() */ - inputIndex = ( inputId & 0xFF ) - 1; - configType = ( inputId & 0xFF00 ) >> 8; + /* Validate function arguments */ + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } - /* Validate values derived from input ID */ - if ( inputIndex < 0 ) + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) { - return IVAS_ERR_INVALID_INPUT_ID; + return error; } - switch ( configType ) + + switch ( getAudioConfigType( inputBase->inConfig ) ) { case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: - if ( inputIndex > RENDERER_MAX_ISM_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsIsm[inputIndex].base; + clearInputIsm( (input_ism *) inputBase ); break; case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - if ( inputIndex > RENDERER_MAX_MC_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsMc[inputIndex].base; + clearInputMc( (input_mc *) inputBase ); break; case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - if ( inputIndex > RENDERER_MAX_SBA_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsSba[inputIndex].base; + clearInputSba( (input_sba *) inputBase ); break; case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + clearInputMasa( (input_masa *) inputBase ); break; #ifdef SPLIT_REND_WITH_HEAD_ROT case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - if ( inputIndex > RENDERER_MAX_BIN_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsSplitPost[inputIndex].base; + clearInputSplitRend( (input_split_post_rend *) inputBase ); break; #endif default: - return IVAS_ERR_INVALID_INPUT_ID; - } - - /* Ensure input ID matches and that input is active */ - if ( pInputBase->id != inputId || pInputBase->inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - return IVAS_ERR_INVALID_INPUT_ID; + return IVAS_ERR_INVALID_INPUT_FORMAT; } - /* Validation done, set value via output parameter */ - *ppInput = pInputBase; - return IVAS_ERR_OK; } -static ivas_error getConstInputById( - IVAS_REND_CONST_HANDLE hIvasRend, - const IVAS_REND_InputId inputId, - const void **ppInput ) -{ - int32_t inputIndex; - IVAS_REND_AudioConfigType configType; - const input_base *pInputBase; - - /* Reverse makeInputId() */ - inputIndex = ( inputId & 0xFF ) - 1; - configType = ( inputId & 0xFF00 ) >> 8; +/*-------------------------------------------------------------------* + * IVAS_REND_GetInputNumChannels() + * + * + *-------------------------------------------------------------------*/ - /* Validate values derived from input ID */ - if ( inputIndex < 0 ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - switch ( configType ) - { - case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: - if ( inputIndex > RENDERER_MAX_ISM_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsIsm[inputIndex].base; - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - if ( inputIndex > RENDERER_MAX_MC_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsMc[inputIndex].base; - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - if ( inputIndex > RENDERER_MAX_SBA_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsSba[inputIndex].base; - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsMasa[inputIndex].base; - break; -#ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - if ( inputIndex > RENDERER_MAX_BIN_INPUTS ) - { - return IVAS_ERR_INVALID_INPUT_ID; - } - pInputBase = &hIvasRend->inputsSplitPost[inputIndex].base; - break; -#endif - default: - return IVAS_ERR_INVALID_INPUT_ID; - } +ivas_error IVAS_REND_GetInputNumChannels( + IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + int16_t *numChannels /* o : number of channels of the input */ +) +{ + ivas_error error; + const input_base *pInput; - /* Ensure input ID matches and that input is active */ - if ( pInputBase->id != inputId || pInputBase->inConfig == IVAS_AUDIO_CONFIG_INVALID ) + /* Validate function arguments */ + if ( hIvasRend == NULL || numChannels == NULL ) { - return IVAS_ERR_INVALID_INPUT_ID; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - /* Validation done, set value via output parameter */ - *ppInput = pInputBase; - - return IVAS_ERR_OK; -} - - -static ivas_error findFreeInputSlot( - const void *inputs, - const int32_t inputStructSize, - const int32_t maxInputs, - int32_t *inputIndex ) -{ - /* Using a void pointer and a separately provided size is a hack for this function - to be reusable for arrays of any input type (input_ism, input_mc, input_sba, input_masa). - Assumptions: - - input_base is always the first member in the input struct - - provided size is correct - */ - - int32_t i; - bool canAddInput; - const uint8_t *pByte; - const input_base *pInputBase; - - canAddInput = false; - - /* Find first unused input in array */ - for ( i = 0, pByte = inputs; i < maxInputs; ++i, pByte += inputStructSize ) + if ( ( error = getConstInputById( hIvasRend, inputId, (const void **) &pInput ) ) != IVAS_ERR_OK ) { - pInputBase = (const input_base *) pByte; - - if ( pInputBase->inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - *inputIndex = i; - canAddInput = true; - break; - } + return error; } - if ( !canAddInput ) + if ( ( error = getRendInputNumChannels( pInput, numChannels ) ) != IVAS_ERR_OK ) { - return IVAS_ERR_TOO_MANY_INPUTS; + return error; } return IVAS_ERR_OK; } -#ifdef SPLIT_REND_WITH_HEAD_ROT -static int16_t getCldfbRendFlag( - IVAS_REND_HANDLE hIvasRend, /* i : Renderer handle */ - const IVAS_REND_AudioConfigType new_configType ) -{ - int16_t i; - int16_t numMasaInputs = 0, numSbaInputs = 0, numIsmInputs = 0, numMcInputs = 0; - int16_t isCldfbRend; +/*-------------------------------------------------------------------* + * IVAS_REND_GetNumAllObjects() + * + * + *-------------------------------------------------------------------*/ - isCldfbRend = 0; - if ( hIvasRend->hRendererConfig != NULL ) +ivas_error IVAS_REND_GetNumAllObjects( + IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer handle */ + int16_t *numChannels /* o : number of all objects */ +) +{ + if ( hIvasRend == NULL || numChannels == NULL ) { - for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) - { - numMasaInputs += ( hIvasRend->inputsMasa[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) ? 0 : 1; - } - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) - { - numSbaInputs += ( hIvasRend->inputsSba[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) ? 0 : 1; - } - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) - { - numIsmInputs += ( hIvasRend->inputsIsm[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) ? 0 : 1; - } - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) - { - numMcInputs += ( hIvasRend->inputsMc[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? 0 : 1; - } - - if ( numIsmInputs > 0 || numMcInputs > 0 ) - { - isCldfbRend = 0; - } - else if ( ( numMasaInputs > 0 ) || ( numSbaInputs > 0 && hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) ) - { - isCldfbRend = 1; - } + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - return isCldfbRend; -} - - -static void closeSplitRend( - SPLIT_REND_WRAPPER *pSplitRendWrapper, - IVAS_REND_AudioBuffer *pSplitRendEncBuffer ) -{ - ivas_split_renderer_close( pSplitRendWrapper ); - - if ( pSplitRendEncBuffer->data != NULL ) + if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA1 || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA2 ) { - free( pSplitRendEncBuffer->data ); - pSplitRendEncBuffer->data = NULL; + *numChannels = (int16_t) hIvasRend->inputsIsm[0].total_num_objects; } - pSplitRendEncBuffer->config.numChannels = 0; - pSplitRendEncBuffer->config.numSamplesPerChannel = 0; - - return; + return IVAS_ERR_OK; } -#endif /*-------------------------------------------------------------------* - * IVAS_REND_AddInput() + * IVAS_REND_GetDelay() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_AddInput( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const AUDIO_CONFIG inConfig, /* i : audio config for a new input */ - IVAS_REND_InputId *inputId /* o : ID of the new input */ +ivas_error IVAS_REND_GetDelay( + IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer state */ + int16_t *nSamples, /* o : Renderer delay in samples */ + int32_t *timeScale /* o : Time scale of the delay, equal to renderer output sampling rate */ ) { - ivas_error error; - int32_t maxNumInputsOfType; - void *inputsArray; - int32_t inputStructSize; - ivas_error ( *activateInput )( void *, AUDIO_CONFIG, IVAS_REND_InputId, RENDER_CONFIG_DATA * ); - int32_t inputIndex; + /* TODO tmu : this function only returns the maximum delay across all inputs + * Ideally each input has its own delay buffer and everything is aligned (binaural and LFE filtering delays are nonuniform) + */ + int16_t i; + int32_t latency_ns; + int32_t max_latency_ns; /* Validate function arguments */ - if ( hIvasRend == NULL || inputId == NULL ) + if ( hIvasRend == NULL || nSamples == NULL || timeScale == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) && hIvasRend->splitRendEncBuffer.data == NULL && hIvasRend->hRendererConfig != NULL ) - { - int16_t cldfb_in_flag; - cldfb_in_flag = getCldfbRendFlag( hIvasRend, getAudioConfigType( inConfig ) ); + *timeScale = hIvasRend->sampleRateOut; + *nSamples = 0; + max_latency_ns = 0; - if ( ( error = ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, &hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, cldfb_in_flag, hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) != IVAS_ERR_OK ) + /* Compute the maximum delay across all inputs */ + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; i++ ) + { + if ( hIvasRend->inputsIsm[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) { - return error; + latency_ns = max( ( hIvasRend->inputsIsm[i].crendWrapper != NULL ) ? hIvasRend->inputsIsm[i].crendWrapper->binaural_latency_ns : 0, + hIvasRend->inputsIsm[i].tdRendWrapper.binaural_latency_ns ); + max_latency_ns = max( max_latency_ns, latency_ns ); } + } - if ( ( error = initSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, ( hIvasRend->num_subframes == 1 ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; i++ ) + { + if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) { - return error; + latency_ns = max( ( hIvasRend->inputsMc[i].crendWrapper != NULL ) ? hIvasRend->inputsMc[i].crendWrapper->binaural_latency_ns : 0, + hIvasRend->inputsMc[i].tdRendWrapper.binaural_latency_ns ); + max_latency_ns = max( max_latency_ns, latency_ns ); } } -#endif - switch ( getAudioConfigType( inConfig ) ) + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; i++ ) { - case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: - maxNumInputsOfType = RENDERER_MAX_ISM_INPUTS; - inputsArray = hIvasRend->inputsIsm; - inputStructSize = sizeof( *hIvasRend->inputsIsm ); - activateInput = setRendInputActiveIsm; - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - maxNumInputsOfType = RENDERER_MAX_MC_INPUTS; - inputsArray = hIvasRend->inputsMc; - inputStructSize = sizeof( *hIvasRend->inputsMc ); - activateInput = setRendInputActiveMc; - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - maxNumInputsOfType = RENDERER_MAX_SBA_INPUTS; - inputsArray = hIvasRend->inputsSba; - inputStructSize = sizeof( *hIvasRend->inputsSba ); - activateInput = setRendInputActiveSba; - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - maxNumInputsOfType = RENDERER_MAX_MASA_INPUTS; - inputsArray = hIvasRend->inputsMasa; - inputStructSize = sizeof( *hIvasRend->inputsMasa ); - activateInput = setRendInputActiveMasa; - break; + if ( hIvasRend->inputsSba[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { #ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - maxNumInputsOfType = RENDERER_MAX_BIN_INPUTS; - inputsArray = hIvasRend->inputsSplitPost; - inputStructSize = sizeof( *hIvasRend->inputsSplitPost ); - activateInput = setRendInputActiveSplitPostRend; - break; + if ( hIvasRend->splitRendWrapper.hBinHrSplitPreRend != NULL ) + { + if ( hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) + { + latency_ns = hIvasRend->inputsSba[i].cldfbRendWrapper.binaural_latency_ns; + } + else + { + latency_ns = ( hIvasRend->inputsSba[i].crendWrapper != NULL ) ? hIvasRend->inputsSba[i].crendWrapper->binaural_latency_ns : 0; + } + max_latency_ns = max( max_latency_ns, latency_ns ); + } + else if ( hIvasRend->inputsSba[i].cldfbRendWrapper.hCldfbRend != NULL ) + { + latency_ns = hIvasRend->inputsSba[i].cldfbRendWrapper.binaural_latency_ns; + latency_ns += IVAS_FB_DEC_DELAY_NS; + max_latency_ns = max( max_latency_ns, latency_ns ); + } + else #endif - default: - return IVAS_ERR_INVALID_INPUT_FORMAT; + { + latency_ns = ( hIvasRend->inputsSba[i].crendWrapper != NULL ) ? hIvasRend->inputsSba[i].crendWrapper->binaural_latency_ns : 0; + max_latency_ns = max( max_latency_ns, latency_ns ); + } + } } - /* Find first free input in array corresponding to input type */ - if ( ( error = findFreeInputSlot( inputsArray, inputStructSize, maxNumInputsOfType, &inputIndex ) ) != IVAS_ERR_OK ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( i = 0; i < RENDERER_MAX_BIN_INPUTS; i++ ) { - return error; + if ( hIvasRend->inputsSplitPost[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + latency_ns = 0; + if ( hIvasRend->inputsSplitPost[i].splitPostRendWrapper.hLc3plusDec != NULL ) + { + int32_t lc3plusDelaySamples; + IVAS_LC3PLUS_DEC_GetDelay( hIvasRend->inputsSplitPost[i].splitPostRendWrapper.hLc3plusDec, &lc3plusDelaySamples ); + latency_ns = (int32_t) roundf( lc3plusDelaySamples * 1000000000.f / *timeScale ); + } + if ( hIvasRend->inputsSplitPost[i].splitPostRendWrapper.multiBinPoseData.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) + { + latency_ns += IVAS_FB_DEC_DELAY_NS; + } + max_latency_ns = max( max_latency_ns, latency_ns ); + } } +#endif - *inputId = makeInputId( inConfig, inputIndex ); - - if ( ( error = activateInput( (uint8_t *) inputsArray + inputStructSize * inputIndex, inConfig, *inputId, hIvasRend->hRendererConfig ) ) != IVAS_ERR_OK ) + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) { - return error; + if ( hIvasRend->inputsMasa[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + latency_ns = (int32_t) ( (float) IVAS_FB_DEC_DELAY_NS + 0.5f ); + max_latency_ns = max( max_latency_ns, latency_ns ); + } } + *nSamples = (int16_t) roundf( (float) max_latency_ns * *timeScale / 1000000000.f ); + return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_ConfigureCustomInputLoudspeakerLayout() + * IVAS_REND_FeedInputAudio() * * - * Note: this will reset any custom LFE routing set for the input *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_ConfigureCustomInputLoudspeakerLayout( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const IVAS_CUSTOM_LS_DATA layout /* i : custom loudspeaker layout for input */ +ivas_error IVAS_REND_FeedInputAudio( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_REND_ReadOnlyAudioBuffer inputAudio /* i : buffer with input audio */ ) { - input_mc *inputMc; ivas_error error; + input_base *inputBase; + int16_t numInputChannels; +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; +#endif /* Validate function arguments */ - if ( hIvasRend == NULL ) + if ( hIvasRend == NULL || inputAudio.data == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( ( error = validateCustomLsLayout( layout ) ) != IVAS_ERR_OK ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + cldfb2tdSampleFact = ( inputAudio.config.is_cldfb ) ? 2 : 1; + + if ( inputAudio.config.numSamplesPerChannel <= 0 || ( MAX_BUFFER_LENGTH_PER_CHANNEL < inputAudio.config.numSamplesPerChannel && inputAudio.config.is_cldfb == 0 ) || + ( ( MAX_BUFFER_LENGTH_PER_CHANNEL * cldfb2tdSampleFact ) < inputAudio.config.numSamplesPerChannel && inputAudio.config.is_cldfb == 1 ) ) +#else + if ( inputAudio.config.numSamplesPerChannel <= 0 || MAX_BUFFER_LENGTH_PER_CHANNEL < inputAudio.config.numSamplesPerChannel ) +#endif { - return error; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Buffer size outside of supported range" ); } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputMc ) ) != IVAS_ERR_OK ) + if ( inputAudio.config.numChannels <= 0 || MAX_INPUT_CHANNELS < inputAudio.config.numChannels ) { - return error; + return IVAS_ERR_WRONG_NUM_CHANNELS; } - if ( inputMc->base.inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && + hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && + hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM && + ( inputAudio.config.numSamplesPerChannel * 1000 / cldfb2tdSampleFact ) != ( BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->num_subframes ) * hIvasRend->sampleRateOut ) +#else + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && + inputAudio.config.numSamplesPerChannel * 1000 != ( BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->num_subframes ) * hIvasRend->sampleRateOut ) +#endif { - /* Specifying details of custom speaker layout only makes sense if input config is set to custom speaker layout */ - return IVAS_ERR_INVALID_INPUT_FORMAT; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Binaural rendering requires specific frame size" ); } - /* Re-initialize panning gains for the MC input, This includes re-initializing LFE handling - * for the new input layout, which means custom LFE handling is overwritten, if previously - * set for the MC input. */ - inputMc->customLsInput = makeCustomLsSetup( layout ); - - inputMc->lfeRouting = defaultLfeRouting( inputMc->base.inConfig, inputMc->customLsInput, hIvasRend->outputConfig, *inputMc->base.ctx.pCustomLsOut ); + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + { + return error; + } - if ( ( error = initEfap( &inputMc->efapInWrapper, inputMc->base.inConfig, &inputMc->customLsInput ) ) != IVAS_ERR_OK ) + if ( ( error = getRendInputNumChannels( inputBase, &numInputChannels ) ) != IVAS_ERR_OK ) { return error; } -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) -#else - if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) -#endif + if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA1 || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA2 ) && inputBase->inConfig == IVAS_AUDIO_CONFIG_OBA ) { - if ( ( error = initMcBinauralRendering( inputMc, inputMc->base.inConfig, hIvasRend->outputConfig, hIvasRend->hRendererConfig, FALSE ) ) != IVAS_ERR_OK ) - { - return error; - } + numInputChannels = (int16_t) hIvasRend->inputsIsm[0].total_num_objects; } - if ( ( error = updateMcPanGains( inputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) + if ( numInputChannels != inputAudio.config.numChannels ) { - return error; + return IVAS_ERR_WRONG_NUM_CHANNELS; } + inputBase->inputBuffer.config = inputAudio.config; + + mvr2r( inputAudio.data, inputBase->inputBuffer.data, inputAudio.config.numSamplesPerChannel * inputAudio.config.numChannels ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + inputBase->numNewSamplesPerChannel = inputAudio.config.numSamplesPerChannel / cldfb2tdSampleFact; +#else + inputBase->numNewSamplesPerChannel = inputAudio.config.numSamplesPerChannel; +#endif + return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_SetInputGain() + * IVAS_REND_FeedInputObjectMetadata() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_SetInputGain( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const float gain /* i : linear gain (not in dB) */ +ivas_error IVAS_REND_FeedInputObjectMetadata( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_ISM_METADATA objectPosition /* i : object position struct */ ) { input_base *inputBase; + input_ism *inputIsm; ivas_error error; /* Validate function arguments */ @@ -4382,79 +4348,61 @@ ivas_error IVAS_REND_SetInputGain( return error; } - inputBase->gain = gain; + if ( inputBase->inConfig != IVAS_AUDIO_CONFIG_OBA ) + { + /* Object metadata should only be fed for object inputs */ + return IVAS_ERR_METADATA_NOT_EXPECTED; + } + + inputIsm = (input_ism *) inputBase; + inputIsm->previousPos = inputIsm->currentPos; + inputIsm->currentPos = objectPosition; return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_SetInputLfeMtx() + * IVAS_REND_FeedInputObjectMetadataToOMasa() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_SetInputLfeMtx( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const IVAS_REND_LfePanMtx *lfePanMtx /* i : LFE panning matrix */ +ivas_error IVAS_REND_FeedInputObjectMetadataToOMasa( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const int16_t inputIndex, /* i : Index of the input */ + const IVAS_ISM_METADATA objectPosition /* i : object position struct */ ) { - int16_t i; - input_base *pInputBase; - input_mc *pInputMc; - ivas_error error; - /* Validate function arguments */ if ( hIvasRend == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &pInputBase ) ) != IVAS_ERR_OK ) - { - return error; - } - - if ( getAudioConfigType( pInputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) - { - /* Custom LFE panning matrix only makes sense with channel-based input */ - return IVAS_ERR_INVALID_INPUT_FORMAT; - } - pInputMc = (input_mc *) pInputBase; - - /* copy LFE panning matrix */ - for ( i = 0; i < IVAS_MAX_INPUT_LFE_CHANNELS; i++ ) - { - mvr2r( ( *lfePanMtx )[i], pInputMc->lfeRouting.lfePanMtx[i], IVAS_MAX_OUTPUT_CHANNELS ); - } - - if ( ( error = updateMcPanGains( pInputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) - { - return error; - } + /* Set position to OMasa struct */ + hIvasRend->inputsIsm->hOMasa->ism_azimuth[inputIndex] = objectPosition.azimuth; + hIvasRend->inputsIsm->hOMasa->ism_elevation[inputIndex] = objectPosition.elevation; return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_SetInputLfePos() + * IVAS_REND_FeedInputMasaMetadata() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_SetInputLfePos( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const float inputGain, /* i : Input gain to be applied to the LFE channel(s) */ - const float outputAzimuth, /* i : Output azimuth position */ - const float outputElevation /* i : Output elevation position */ +ivas_error IVAS_REND_FeedInputMasaMetadata( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + IVAS_MASA_METADATA_HANDLE masaMetadata /* i : MASA metadata frame */ ) { - input_base *pInputBase; - input_mc *pInputMc; ivas_error error; + input_base *inputBase; + input_masa *inputMasa; /* Validate function arguments */ if ( hIvasRend == NULL ) @@ -4462,365 +4410,407 @@ ivas_error IVAS_REND_SetInputLfePos( return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &pInputBase ) ) != IVAS_ERR_OK ) + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) { return error; } - if ( getAudioConfigType( pInputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + if ( getAudioConfigType( inputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) { - /* Custom LFE routing only makes sense with channel-based input */ - return IVAS_ERR_INVALID_INPUT_FORMAT; + /* MASA metadata should only be fed for MASA inputs */ + return IVAS_ERR_METADATA_NOT_EXPECTED; } - pInputMc = (input_mc *) pInputBase; - pInputMc->lfeRouting.pan_lfe = true; - pInputMc->lfeRouting.lfeInputGain = inputGain; - pInputMc->lfeRouting.lfeOutputAzimuth = outputAzimuth; - pInputMc->lfeRouting.lfeOutputElevation = outputElevation; - - if ( ( error = updateMcPanGains( pInputMc, hIvasRend->outputConfig ) ) != IVAS_ERR_OK ) - { - return error; - } + inputMasa = (input_masa *) inputBase; + inputMasa->masaMetadata = *masaMetadata; + inputMasa->metadataHasBeenFed = true; return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_RemoveInput() + * IVAS_REND_InitConfig() * * *-------------------------------------------------------------------*/ -/* ToDo; unused function */ -ivas_error IVAS_REND_RemoveInput( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId /* i : ID of the input */ + +ivas_error IVAS_REND_InitConfig( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const AUDIO_CONFIG outAudioConfig /* i : output audioConfig */ ) { ivas_error error; - input_base *inputBase; + bool rendererConfigEnabled; - /* Validate function arguments */ - if ( hIvasRend == NULL ) + rendererConfigEnabled = ( getAudioConfigType( outAudioConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ); + + if ( rendererConfigEnabled ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + hIvasRend->rendererConfigEnabled = 1; + } + else + { + hIvasRend->rendererConfigEnabled = 0; } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + if ( rendererConfigEnabled ) { - return error; + if ( ( error = ivas_render_config_open( &( hIvasRend->hRendererConfig ) ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = ivas_render_config_init_from_rom( &hIvasRend->hRendererConfig ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + hIvasRend->hRendererConfig = NULL; } - switch ( getAudioConfigType( inputBase->inConfig ) ) + return IVAS_ERR_OK; +} + + +/*-------------------------------------------------------------------* + * IVAS_REND_GetRenderConfig() + * + * + *-------------------------------------------------------------------*/ + +int16_t IVAS_REND_GetRenderConfig( + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ + const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render configuration handle */ +) +{ + RENDER_CONFIG_HANDLE hRCin; + + if ( hIvasRend == NULL || hIvasRend->hRendererConfig == NULL || hRCout == NULL ) { - case IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED: - clearInputIsm( (input_ism *) inputBase ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - clearInputMc( (input_mc *) inputBase ); + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + hRCin = hIvasRend->hRendererConfig; +#ifdef DEBUGGING + switch ( hRCin->renderer_type_override ) + { + case IVAS_RENDER_TYPE_OVERRIDE_CREND: + hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_CREND; break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - clearInputSba( (input_sba *) inputBase ); + case IVAS_RENDER_TYPE_OVERRIDE_FASTCONV: + hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_FASTCONV; break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - clearInputMasa( (input_masa *) inputBase ); + default: + hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_NONE; break; + } +#endif + hRCout->roomAcoustics.override = hRCin->roomAcoustics.override; + hRCout->roomAcoustics.nBands = hRCin->roomAcoustics.nBands; + hRCout->roomAcoustics.acousticPreDelay = hRCin->roomAcoustics.acousticPreDelay; + hRCout->roomAcoustics.inputPreDelay = hRCin->roomAcoustics.inputPreDelay; + mvr2r( hRCin->directivity, hRCout->directivity, 3 * MAX_NUM_OBJECTS ); + + mvr2r( hRCin->roomAcoustics.pFc_input, hRCout->roomAcoustics.pFc_input, CLDFB_NO_CHANNELS_MAX ); + mvr2r( hRCin->roomAcoustics.pAcoustic_rt60, hRCout->roomAcoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX ); + mvr2r( hRCin->roomAcoustics.pAcoustic_dsr, hRCout->roomAcoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX ); + #ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - clearInputSplitRend( (input_split_post_rend *) inputBase ); - break; + hRCout->split_rend_config.splitRendBitRate = SPLIT_REND_768k; + hRCout->split_rend_config.dof = 3; + hRCout->split_rend_config.hq_mode = 0; + hRCout->split_rend_config.codec_delay_ms = 0; + hRCout->split_rend_config.codec_frame_size_ms = 0; /* 0 means "use default for selected codec" */ + hRCout->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; + hRCout->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB; + hRCout->split_rend_config.rendererSelection = hRCin->split_rend_config.rendererSelection; #endif - default: - return IVAS_ERR_INVALID_INPUT_FORMAT; - } + + hRCout->roomAcoustics.use_er = hRCin->roomAcoustics.use_er; + hRCout->roomAcoustics.lowComplexity = hRCin->roomAcoustics.lowComplexity; return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_GetInputNumChannels() + * IVAS_REND_FeedRenderConfig() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_GetInputNumChannels( - IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - int16_t *numChannels /* o : number of channels of the input */ +int16_t IVAS_REND_FeedRenderConfig( + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ + const IVAS_RENDER_CONFIG_DATA renderConfig /* i : Render configuration struct */ ) { + RENDER_CONFIG_HANDLE hRenderConfig; +#ifdef SPLIT_REND_WITH_HEAD_ROT ivas_error error; - const input_base *pInput; +#endif - /* Validate function arguments */ - if ( hIvasRend == NULL || numChannels == NULL ) + if ( hIvasRend == NULL || hIvasRend->hRendererConfig == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } + hRenderConfig = hIvasRend->hRendererConfig; +#ifdef DEBUGGING + hRenderConfig->renderer_type_override = renderConfig.renderer_type_override; +#endif + + hRenderConfig->roomAcoustics.override = renderConfig.roomAcoustics.override; + hRenderConfig->roomAcoustics.nBands = renderConfig.roomAcoustics.nBands; + hRenderConfig->roomAcoustics.acousticPreDelay = renderConfig.roomAcoustics.acousticPreDelay; + hRenderConfig->roomAcoustics.inputPreDelay = renderConfig.roomAcoustics.inputPreDelay; + mvr2r( renderConfig.roomAcoustics.pFc_input, hRenderConfig->roomAcoustics.pFc_input, CLDFB_NO_CHANNELS_MAX ); + mvr2r( renderConfig.roomAcoustics.pAcoustic_rt60, hRenderConfig->roomAcoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX ); + mvr2r( renderConfig.roomAcoustics.pAcoustic_dsr, hRenderConfig->roomAcoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX ); + mvr2r( renderConfig.directivity, hRenderConfig->directivity, 3 * MAX_NUM_OBJECTS ); + + hRenderConfig->roomAcoustics.use_er = 0; + if ( renderConfig.roomAcoustics.use_er == 1 ) + { + hRenderConfig->roomAcoustics.use_er = renderConfig.roomAcoustics.use_er; + hRenderConfig->roomAcoustics.lowComplexity = renderConfig.roomAcoustics.lowComplexity; + hRenderConfig->roomAcoustics.dimensions = renderConfig.roomAcoustics.dimensions; + hRenderConfig->roomAcoustics.ListenerOrigin = renderConfig.roomAcoustics.ListenerOrigin; + + mvr2r( renderConfig.roomAcoustics.AbsCoeff, hRenderConfig->roomAcoustics.AbsCoeff, ER_ABS_COEFF ); + } - if ( ( error = getConstInputById( hIvasRend, inputId, (const void **) &pInput ) ) != IVAS_ERR_OK ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + hRenderConfig->split_rend_config = renderConfig.split_rend_config; + /* Overwrite any pose correction settings if 0 DOF (no pose correction) was selected */ + if ( hRenderConfig->split_rend_config.dof == 0 ) { - return error; + hRenderConfig->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE; } - if ( ( error = getRendInputNumChannels( pInput, numChannels ) ) != IVAS_ERR_OK ) + hRenderConfig->split_rend_config.codec = renderConfig.split_rend_config.codec; + + if ( ( error = ivas_split_rend_validate_config( &hRenderConfig->split_rend_config, ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ) ) != IVAS_ERR_OK ) { return error; } + /* Must re-initialize split rendering config in case renderer config is updated after adding renderer inputs */ + /* if its not initialized yet then no need to re-initialize, initialization will happen while adding inputs*/ + if ( hIvasRend->splitRendEncBuffer.data != NULL && hIvasRend->hRendererConfig != NULL ) + { + int16_t cldfb_in_flag; + cldfb_in_flag = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); + closeSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer ); + + if ( ( error = ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, &hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, cldfb_in_flag, hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( ( error = initSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, ( hIvasRend->num_subframes == 1 ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif + return IVAS_ERR_OK; } +#ifdef SPLIT_REND_WITH_HEAD_ROT /*-------------------------------------------------------------------* - * IVAS_REND_GetNumAllObjects() + * IVAS_REND_FeedSplitBinauralBitstream() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_GetNumAllObjects( - IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer handle */ - int16_t *numChannels /* o : number of all objects */ +ivas_error IVAS_REND_FeedSplitBinauralBitstream( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + IVAS_REND_BitstreamBuffer *hBits /* i : buffer for input bitstream */ ) { - if ( hIvasRend == NULL || numChannels == NULL ) + ivas_error error; + input_base *inputBase; + input_split_post_rend *inputSplitPostRend; + + /* Validate function arguments */ + if ( hIvasRend == NULL || hBits == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA1 || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA2 ) + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) { - *numChannels = (int16_t) hIvasRend->inputsIsm[0].total_num_objects; + return error; } + inputSplitPostRend = (input_split_post_rend *) inputBase; + inputSplitPostRend->hBits = hBits; + return IVAS_ERR_OK; } +#endif /*-------------------------------------------------------------------* - * IVAS_REND_GetDelay() + * IVAS_REND_SetHeadRotation() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_GetDelay( - IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer state */ - int16_t *nSamples, /* o : Renderer delay in samples */ - int32_t *timeScale /* o : Time scale of the delay, equal to renderer output sampling rate */ +ivas_error IVAS_REND_SetHeadRotation( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_QUATERNION headRot, /* i : head orientations for next rendering call */ + const IVAS_VECTOR3 Pos, /* i : listener positions for next rendering call */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + const IVAS_SPLIT_REND_ROT_AXIS rot_axis, /* i : external control for rotation axis for split rendering */ +#endif + const int16_t sf_idx /* i : subframe index */ ) { - /* TODO tmu : this function only returns the maximum delay across all inputs - * Ideally each input has its own delay buffer and everything is aligned (binaural and LFE filtering delays are nonuniform) - */ int16_t i; - int32_t latency_ns; - int32_t max_latency_ns; + IVAS_QUATERNION rotQuat; + ivas_error error; /* Validate function arguments */ - if ( hIvasRend == NULL || nSamples == NULL || timeScale == NULL ) + if ( hIvasRend == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - *timeScale = hIvasRend->sampleRateOut; - *nSamples = 0; - max_latency_ns = 0; - - /* Compute the maximum delay across all inputs */ - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; i++ ) + if ( getAudioConfigType( hIvasRend->outputConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) { - if ( hIvasRend->inputsIsm[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - latency_ns = max( ( hIvasRend->inputsIsm[i].crendWrapper != NULL ) ? hIvasRend->inputsIsm[i].crendWrapper->binaural_latency_ns : 0, - hIvasRend->inputsIsm[i].tdRendWrapper.binaural_latency_ns ); - max_latency_ns = max( max_latency_ns, latency_ns ); - } + /* Head rotation can be set only with binaural output */ + return IVAS_ERR_INVALID_OUTPUT_FORMAT; } - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; i++ ) + hIvasRend->headRotData.headRotEnabled = 1; + + /* reconfigure binaural rendering to allocate the necessary renderers and free unused ones */ + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) { if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) { - latency_ns = max( ( hIvasRend->inputsMc[i].crendWrapper != NULL ) ? hIvasRend->inputsMc[i].crendWrapper->binaural_latency_ns : 0, - hIvasRend->inputsMc[i].tdRendWrapper.binaural_latency_ns ); - max_latency_ns = max( max_latency_ns, latency_ns ); + if ( ( error = initMcBinauralRendering( &hIvasRend->inputsMc[i], hIvasRend->inputsMc[i].base.inConfig, hIvasRend->outputConfig, hIvasRend->hRendererConfig, TRUE ) ) != IVAS_ERR_OK ) + { + return error; + } } } - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; i++ ) + /* check for Euler angle signaling */ + if ( headRot.w == -3.0f ) { - if ( hIvasRend->inputsSba[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( hIvasRend->splitRendWrapper.hBinHrSplitPreRend != NULL ) - { - if ( hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) - { - latency_ns = hIvasRend->inputsSba[i].cldfbRendWrapper.binaural_latency_ns; - } - else - { - latency_ns = ( hIvasRend->inputsSba[i].crendWrapper != NULL ) ? hIvasRend->inputsSba[i].crendWrapper->binaural_latency_ns : 0; - } - max_latency_ns = max( max_latency_ns, latency_ns ); - } - else if ( hIvasRend->inputsSba[i].cldfbRendWrapper.hCldfbRend != NULL ) - { - latency_ns = hIvasRend->inputsSba[i].cldfbRendWrapper.binaural_latency_ns; - latency_ns += IVAS_FB_DEC_DELAY_NS; - max_latency_ns = max( max_latency_ns, latency_ns ); - } - else -#endif - { - latency_ns = ( hIvasRend->inputsSba[i].crendWrapper != NULL ) ? hIvasRend->inputsSba[i].crendWrapper->binaural_latency_ns : 0; - max_latency_ns = max( max_latency_ns, latency_ns ); - } - } + Euler2Quat( deg2rad( headRot.x ), deg2rad( headRot.y ), deg2rad( headRot.z ), &rotQuat ); } - -#ifdef SPLIT_REND_WITH_HEAD_ROT - for ( i = 0; i < RENDERER_MAX_BIN_INPUTS; i++ ) + else { - if ( hIvasRend->inputsSplitPost[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - latency_ns = 0; - if ( hIvasRend->inputsSplitPost[i].splitPostRendWrapper.hLc3plusDec != NULL ) - { - int32_t lc3plusDelaySamples; - IVAS_LC3PLUS_DEC_GetDelay( hIvasRend->inputsSplitPost[i].splitPostRendWrapper.hLc3plusDec, &lc3plusDelaySamples ); - latency_ns = (int32_t) roundf( lc3plusDelaySamples * 1000000000.f / *timeScale ); - } - if ( hIvasRend->inputsSplitPost[i].splitPostRendWrapper.multiBinPoseData.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) - { - latency_ns += IVAS_FB_DEC_DELAY_NS; - } - max_latency_ns = max( max_latency_ns, latency_ns ); - } + rotQuat = headRot; } -#endif - for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) + if ( ( error = ivas_orient_trk_Process( hIvasRend->headRotData.hOrientationTracker, rotQuat, FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES, &hIvasRend->headRotData.headPositions[sf_idx] ) ) != IVAS_ERR_OK ) { - if ( hIvasRend->inputsMasa[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - latency_ns = (int32_t) ( (float) IVAS_FB_DEC_DELAY_NS + 0.5f ); - max_latency_ns = max( max_latency_ns, latency_ns ); - } + return error; } - *nSamples = (int16_t) roundf( (float) max_latency_ns * *timeScale / 1000000000.f ); + hIvasRend->headRotData.Pos[sf_idx] = Pos; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + hIvasRend->headRotData.sr_pose_pred_axis = rot_axis; +#endif return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_FeedInputAudio() + * IVAS_REND_DisableHeadRotation() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_FeedInputAudio( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const IVAS_REND_ReadOnlyAudioBuffer inputAudio /* i : buffer with input audio */ +ivas_error IVAS_REND_DisableHeadRotation( + IVAS_REND_HANDLE hIvasRend /* i/o: Renderer handle */ ) { + int16_t i; ivas_error error; - input_base *inputBase; - int16_t numInputChannels; -#ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t cldfb2tdSampleFact; -#endif /* Validate function arguments */ - if ( hIvasRend == NULL || inputAudio.data == NULL ) + if ( hIvasRend == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } -#ifdef SPLIT_REND_WITH_HEAD_ROT - cldfb2tdSampleFact = ( inputAudio.config.is_cldfb ) ? 2 : 1; - - if ( inputAudio.config.numSamplesPerChannel <= 0 || ( MAX_BUFFER_LENGTH_PER_CHANNEL < inputAudio.config.numSamplesPerChannel && inputAudio.config.is_cldfb == 0 ) || - ( ( MAX_BUFFER_LENGTH_PER_CHANNEL * cldfb2tdSampleFact ) < inputAudio.config.numSamplesPerChannel && inputAudio.config.is_cldfb == 1 ) ) -#else - if ( inputAudio.config.numSamplesPerChannel <= 0 || MAX_BUFFER_LENGTH_PER_CHANNEL < inputAudio.config.numSamplesPerChannel ) -#endif - { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Buffer size outside of supported range" ); - } + hIvasRend->headRotData.headRotEnabled = 0; - if ( inputAudio.config.numChannels <= 0 || MAX_INPUT_CHANNELS < inputAudio.config.numChannels ) + /* reconfigure binaural rendering to allocate the necessary renderers and free unused ones */ + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) { - return IVAS_ERR_WRONG_NUM_CHANNELS; - } + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) + { + if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + if ( ( error = initMcBinauralRendering( &hIvasRend->inputsMc[i], hIvasRend->inputsMc[i].base.inConfig, hIvasRend->outputConfig, hIvasRend->hRendererConfig, TRUE ) ) != IVAS_ERR_OK ) + { -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && - hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && - hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM && - ( inputAudio.config.numSamplesPerChannel * 1000 / cldfb2tdSampleFact ) != ( BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->num_subframes ) * hIvasRend->sampleRateOut ) -#else - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && - inputAudio.config.numSamplesPerChannel * 1000 != ( BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->num_subframes ) * hIvasRend->sampleRateOut ) -#endif - { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Binaural rendering requires specific frame size" ); + return error; + } + } + } } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERR_OK; +} - if ( ( error = getRendInputNumChannels( inputBase, &numInputChannels ) ) != IVAS_ERR_OK ) - { - return error; - } - if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA1 || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA2 ) && inputBase->inConfig == IVAS_AUDIO_CONFIG_OBA ) - { - numInputChannels = (int16_t) hIvasRend->inputsIsm[0].total_num_objects; - } +#ifdef SPLIT_REND_WITH_HEAD_ROT +/*-------------------------------------------------------------------* + * IVAS_REND_SetSplitRendBFI() + * + * + *-------------------------------------------------------------------*/ - if ( numInputChannels != inputAudio.config.numChannels ) - { - return IVAS_ERR_WRONG_NUM_CHANNELS; - } +ivas_error IVAS_REND_SetSplitRendBFI( + IVAS_REND_HANDLE hIvasRend, + const int16_t bfi ) +{ + hIvasRend->splitRendBFI = bfi; - inputBase->inputBuffer.config = inputAudio.config; + return IVAS_ERR_OK; +} +#endif - mvr2r( inputAudio.data, inputBase->inputBuffer.data, inputAudio.config.numSamplesPerChannel * inputAudio.config.numChannels ); -#ifdef SPLIT_REND_WITH_HEAD_ROT - inputBase->numNewSamplesPerChannel = inputAudio.config.numSamplesPerChannel / cldfb2tdSampleFact; -#else - inputBase->numNewSamplesPerChannel = inputAudio.config.numSamplesPerChannel; -#endif +/*-------------------------------------------------------------------* + * IVAS_REND_SetOrientationTrackingMode() + * + * + *-------------------------------------------------------------------*/ - return IVAS_ERR_OK; +ivas_error IVAS_REND_SetOrientationTrackingMode( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_HEAD_ORIENT_TRK_T orientation_tracking /* i : Head orientation tracking type */ +) +{ + return ivas_orient_trk_SetTrackingType( hIvasRend->headRotData.hOrientationTracker, orientation_tracking ); } /*-------------------------------------------------------------------* - * IVAS_REND_FeedInputObjectMetadata() + * IVAS_REND_SetReferenceRotation() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_FeedInputObjectMetadata( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const IVAS_ISM_METADATA objectPosition /* i : object position struct */ +ivas_error IVAS_REND_SetReferenceRotation( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_QUATERNION refRot /* i : Reference rotation */ ) { - input_base *inputBase; - input_ism *inputIsm; ivas_error error; /* Validate function arguments */ @@ -4829,2130 +4819,2232 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_orient_trk_SetReferenceRotation( hIvasRend->headRotData.hOrientationTracker, refRot ) ) != IVAS_ERR_OK ) { return error; } - if ( inputBase->inConfig != IVAS_AUDIO_CONFIG_OBA ) - { - /* Object metadata should only be fed for object inputs */ - return IVAS_ERR_METADATA_NOT_EXPECTED; - } - - inputIsm = (input_ism *) inputBase; - inputIsm->previousPos = inputIsm->currentPos; - inputIsm->currentPos = objectPosition; - return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_FeedInputObjectMetadataToOMasa() + * IVAS_REND_GetMainOrientation() * * *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_FeedInputObjectMetadataToOMasa( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const int16_t inputIndex, /* i : Index of the input */ - const IVAS_ISM_METADATA objectPosition /* i : object position struct */ +// ToDo: not used +ivas_error IVAS_REND_GetMainOrientation( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_QUATERNION *pOrientation /* i/o: Quaternion pointer for main orientation */ ) { - /* Validate function arguments */ - if ( hIvasRend == NULL ) + ivas_error error; + + if ( hIvasRend == NULL || pOrientation == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - /* Set position to OMasa struct */ - hIvasRend->inputsIsm->hOMasa->ism_azimuth[inputIndex] = objectPosition.azimuth; - hIvasRend->inputsIsm->hOMasa->ism_elevation[inputIndex] = objectPosition.elevation; + if ( ( error = ivas_orient_trk_GetMainOrientation( hIvasRend->headRotData.hOrientationTracker, pOrientation ) ) != IVAS_ERR_OK ) + { + return error; + } return IVAS_ERR_OK; } /*-------------------------------------------------------------------* - * IVAS_REND_FeedInputMasaMetadata() + * IVAS_REND_GetTrackedRotation() * * *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_FeedInputMasaMetadata( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - IVAS_MASA_METADATA_HANDLE masaMetadata /* i : MASA metadata frame */ +// ToDo: not used +ivas_error IVAS_REND_GetTrackedRotation( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_QUATERNION *pRotation /* i/o: Quaternion pointer processed rotation */ ) { ivas_error error; - input_base *inputBase; - input_masa *inputMasa; - /* Validate function arguments */ - if ( hIvasRend == NULL ) + if ( hIvasRend == NULL || pRotation == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_orient_trk_GetTrackedRotation( hIvasRend->headRotData.hOrientationTracker, pRotation ) ) != IVAS_ERR_OK ) { return error; } - if ( getAudioConfigType( inputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) - { - /* MASA metadata should only be fed for MASA inputs */ - return IVAS_ERR_METADATA_NOT_EXPECTED; - } - - inputMasa = (input_masa *) inputBase; - inputMasa->masaMetadata = *masaMetadata; - inputMasa->metadataHasBeenFed = true; - return IVAS_ERR_OK; } -/*-------------------------------------------------------------------* - * IVAS_REND_InitConfig() - * +/*---------------------------------------------------------------------* + * IVAS_REND_SetReferenceVector( ) * - *-------------------------------------------------------------------*/ + * Sets a reference vector spanning from listenerPos to refPos. Only + * available in OTR_TRACKING_REF_VEC and OTR_TRACKING_REF_VEC_LEV modes. + *---------------------------------------------------------------------*/ -ivas_error IVAS_REND_InitConfig( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const AUDIO_CONFIG outAudioConfig /* i : output audioConfig */ +ivas_error IVAS_REND_SetReferenceVector( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_VECTOR3 listenerPos, /* i : Listener position */ + const IVAS_VECTOR3 refPos /* i : Reference position */ ) { - ivas_error error; - bool rendererConfigEnabled; - - rendererConfigEnabled = ( getAudioConfigType( outAudioConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ); - - if ( rendererConfigEnabled ) - { - hIvasRend->rendererConfigEnabled = 1; - } - else - { - hIvasRend->rendererConfigEnabled = 0; - } - - if ( rendererConfigEnabled ) - { - if ( ( error = ivas_render_config_open( &( hIvasRend->hRendererConfig ) ) ) != IVAS_ERR_OK ) - { - return error; - } - if ( ( error = ivas_render_config_init_from_rom( &hIvasRend->hRendererConfig ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else + if ( hIvasRend == NULL || hIvasRend->headRotData.hOrientationTracker == NULL ) { - hIvasRend->hRendererConfig = NULL; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - return IVAS_ERR_OK; + return ivas_orient_trk_SetReferenceVector( hIvasRend->headRotData.hOrientationTracker, listenerPos, refPos ); } -/*-------------------------------------------------------------------* - * IVAS_REND_GetRenderConfig() +/*---------------------------------------------------------------------* + * IVAS_REND_SetExternalOrientation() * * - *-------------------------------------------------------------------*/ + *---------------------------------------------------------------------*/ -int16_t IVAS_REND_GetRenderConfig( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ - const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render configuration handle */ +ivas_error IVAS_REND_SetExternalOrientation( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_QUATERNION *orientation, /* i : external orientation data */ + int8_t enableHeadRotation, /* i : flag to enable head rotation for this frame */ + int8_t enableExternalOrientation, /* i : flag to enable external orientation for this frame */ + int8_t enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ + int16_t numFramesToTargetOrientation, /* i : number of frames until target orientation is reached */ + const int16_t sf_idx /* i : subframe index */ ) { - RENDER_CONFIG_HANDLE hRCin; - - if ( hIvasRend == NULL || hIvasRend->hRendererConfig == NULL || hRCout == NULL ) + /* Validate function arguments */ + if ( hIvasRend == NULL || hIvasRend->hExternalOrientationData == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - hRCin = hIvasRend->hRendererConfig; -#ifdef DEBUGGING - switch ( hRCin->renderer_type_override ) + if ( orientation == NULL ) { - case IVAS_RENDER_TYPE_OVERRIDE_CREND: - hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_CREND; - break; - case IVAS_RENDER_TYPE_OVERRIDE_FASTCONV: - hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_FASTCONV; - break; - default: - hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_NONE; - break; + hIvasRend->hExternalOrientationData->enableExternalOrientation[sf_idx] = 0; } -#endif - hRCout->roomAcoustics.override = hRCin->roomAcoustics.override; - hRCout->roomAcoustics.nBands = hRCin->roomAcoustics.nBands; - hRCout->roomAcoustics.acousticPreDelay = hRCin->roomAcoustics.acousticPreDelay; - hRCout->roomAcoustics.inputPreDelay = hRCin->roomAcoustics.inputPreDelay; - mvr2r( hRCin->directivity, hRCout->directivity, 3 * MAX_NUM_OBJECTS ); - - mvr2r( hRCin->roomAcoustics.pFc_input, hRCout->roomAcoustics.pFc_input, CLDFB_NO_CHANNELS_MAX ); - mvr2r( hRCin->roomAcoustics.pAcoustic_rt60, hRCout->roomAcoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX ); - mvr2r( hRCin->roomAcoustics.pAcoustic_dsr, hRCout->roomAcoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX ); - -#ifdef SPLIT_REND_WITH_HEAD_ROT - hRCout->split_rend_config.splitRendBitRate = SPLIT_REND_768k; - hRCout->split_rend_config.dof = 3; - hRCout->split_rend_config.hq_mode = 0; - hRCout->split_rend_config.codec_delay_ms = 0; - hRCout->split_rend_config.codec_frame_size_ms = 0; /* 0 means "use default for selected codec" */ - hRCout->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; - hRCout->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB; - hRCout->split_rend_config.rendererSelection = hRCin->split_rend_config.rendererSelection; -#endif + else + { + QuaternionInverse( *orientation, &hIvasRend->hExternalOrientationData->Quaternions[sf_idx] ); - hRCout->roomAcoustics.use_er = hRCin->roomAcoustics.use_er; - hRCout->roomAcoustics.lowComplexity = hRCin->roomAcoustics.lowComplexity; + hIvasRend->hExternalOrientationData->enableHeadRotation[sf_idx] = enableHeadRotation; + hIvasRend->hExternalOrientationData->enableExternalOrientation[sf_idx] = enableExternalOrientation; + hIvasRend->hExternalOrientationData->enableRotationInterpolation[sf_idx] = enableRotationInterpolation; + hIvasRend->hExternalOrientationData->numFramesToTargetOrientation[sf_idx] = numFramesToTargetOrientation; + } return IVAS_ERR_OK; } -/*-------------------------------------------------------------------* - * IVAS_REND_FeedRenderConfig() +/*---------------------------------------------------------------------* + * IVAS_REND_CombineHeadAndExternalOrientation() * * - *-------------------------------------------------------------------*/ + *---------------------------------------------------------------------*/ -int16_t IVAS_REND_FeedRenderConfig( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS decoder handle */ - const IVAS_RENDER_CONFIG_DATA renderConfig /* i : Render configuration struct */ +ivas_error IVAS_REND_CombineHeadAndExternalOrientation( + IVAS_REND_HANDLE hIvasRend /* i/o: Renderer handle */ ) { - RENDER_CONFIG_HANDLE hRenderConfig; -#ifdef SPLIT_REND_WITH_HEAD_ROT - ivas_error error; -#endif - - if ( hIvasRend == NULL || hIvasRend->hRendererConfig == NULL ) + if ( hIvasRend == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - hRenderConfig = hIvasRend->hRendererConfig; -#ifdef DEBUGGING - hRenderConfig->renderer_type_override = renderConfig.renderer_type_override; -#endif - - hRenderConfig->roomAcoustics.override = renderConfig.roomAcoustics.override; - hRenderConfig->roomAcoustics.nBands = renderConfig.roomAcoustics.nBands; - hRenderConfig->roomAcoustics.acousticPreDelay = renderConfig.roomAcoustics.acousticPreDelay; - hRenderConfig->roomAcoustics.inputPreDelay = renderConfig.roomAcoustics.inputPreDelay; - mvr2r( renderConfig.roomAcoustics.pFc_input, hRenderConfig->roomAcoustics.pFc_input, CLDFB_NO_CHANNELS_MAX ); - mvr2r( renderConfig.roomAcoustics.pAcoustic_rt60, hRenderConfig->roomAcoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX ); - mvr2r( renderConfig.roomAcoustics.pAcoustic_dsr, hRenderConfig->roomAcoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX ); - mvr2r( renderConfig.directivity, hRenderConfig->directivity, 3 * MAX_NUM_OBJECTS ); - - hRenderConfig->roomAcoustics.use_er = 0; - if ( renderConfig.roomAcoustics.use_er == 1 ) - { - hRenderConfig->roomAcoustics.use_er = renderConfig.roomAcoustics.use_er; - hRenderConfig->roomAcoustics.lowComplexity = renderConfig.roomAcoustics.lowComplexity; - hRenderConfig->roomAcoustics.dimensions = renderConfig.roomAcoustics.dimensions; - hRenderConfig->roomAcoustics.ListenerOrigin = renderConfig.roomAcoustics.ListenerOrigin; - mvr2r( renderConfig.roomAcoustics.AbsCoeff, hRenderConfig->roomAcoustics.AbsCoeff, ER_ABS_COEFF ); - } + return combine_external_and_head_orientations_rend( &hIvasRend->headRotData, hIvasRend->hExternalOrientationData, hIvasRend->hCombinedOrientationData ); +} -#ifdef SPLIT_REND_WITH_HEAD_ROT - hRenderConfig->split_rend_config = renderConfig.split_rend_config; - /* Overwrite any pose correction settings if 0 DOF (no pose correction) was selected */ - if ( hRenderConfig->split_rend_config.dof == 0 ) - { - hRenderConfig->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE; - } - hRenderConfig->split_rend_config.codec = renderConfig.split_rend_config.codec; +/*---------------------------------------------------------------------* + * IVAS_REND_GetCombinedOrientation() + * + * + *---------------------------------------------------------------------*/ +// ToDo: not used +ivas_error IVAS_REND_GetCombinedOrientation( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_QUATERNION *pOrientation /* i/o: Quaternion pointer processed orientation */ +) +{ + int16_t i; - if ( ( error = ivas_split_rend_validate_config( &hRenderConfig->split_rend_config, ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + if ( hIvasRend == NULL || pOrientation == NULL ) { - return error; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - /* Must re-initialize split rendering config in case renderer config is updated after adding renderer inputs */ - /* if its not initialized yet then no need to re-initialize, initialization will happen while adding inputs*/ - if ( hIvasRend->splitRendEncBuffer.data != NULL && hIvasRend->hRendererConfig != NULL ) + if ( hIvasRend->hCombinedOrientationData != NULL ) { - int16_t cldfb_in_flag; - cldfb_in_flag = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); - closeSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer ); - - if ( ( error = ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, &hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, cldfb_in_flag, hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) != IVAS_ERR_OK ) - { - return error; - } - - if ( ( error = initSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, ( hIvasRend->num_subframes == 1 ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + for ( i = 0; i < hIvasRend->hCombinedOrientationData->num_subframes; ++i ) { - return error; + pOrientation[i] = hIvasRend->hCombinedOrientationData->Quaternions[i]; } } -#endif return IVAS_ERR_OK; } -#ifdef SPLIT_REND_WITH_HEAD_ROT /*-------------------------------------------------------------------* - * IVAS_REND_FeedSplitBinauralBitstream() - * - * + * Local functions *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_FeedSplitBinauralBitstream( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - IVAS_REND_BitstreamBuffer *hBits /* i : buffer for input bitstream */ -) +/* Take one channel from input buffer and copy it to each channel + in output buffer, with different gain applied per output channel. + This function takes 2 gain vectors - one for the beginning and one + for the end of the buffer. Gain values are lineraly interpolated + for all samples in between. */ +static void renderBufferChannelLerp( + const IVAS_REND_AudioBuffer inAudio, + const int32_t inChannelIdx, + const float *const gainsCurrent, + const float *const gainsPrev, + IVAS_REND_AudioBuffer outAudio ) { - ivas_error error; - input_base *inputBase; - input_split_post_rend *inputSplitPostRend; + const float *inSmpl; + float *outSmpl; + float fadeIn; + float fadeOut; + int32_t i; + const float *lastInSmpl; + int16_t outChnlIdx; + float currentGain; + float previousGain; - /* Validate function arguments */ - if ( hIvasRend == NULL || hBits == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + /* Pointer to behind last input sample */ + lastInSmpl = getSmplPtr( inAudio, inChannelIdx, inAudio.config.numSamplesPerChannel ); - if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + for ( outChnlIdx = 0; outChnlIdx < outAudio.config.numChannels; ++outChnlIdx ) { - return error; - } + currentGain = gainsCurrent[outChnlIdx]; + previousGain = gainsPrev == NULL ? 0.f : gainsPrev[outChnlIdx]; - inputSplitPostRend = (input_split_post_rend *) inputBase; - inputSplitPostRend->hBits = hBits; + /* Process current output channel only if applying non-zero gains */ + if ( fabsf( currentGain ) > EPSILON || ( gainsPrev != NULL && fabsf( previousGain ) > EPSILON ) ) + { + /* Reset input pointer to the beginning of input channel */ + inSmpl = getSmplPtr( inAudio, inChannelIdx, 0 ); - return IVAS_ERR_OK; -} -#endif + /* Set output pointer to first output channel sample */ + outSmpl = getSmplPtr( outAudio, outChnlIdx, 0 ); + if ( gainsPrev == NULL || fabsf( previousGain - currentGain ) <= EPSILON ) + { + /* If no interpolation from previous frame, apply current gain */ + do + { + *outSmpl += currentGain * ( *inSmpl ); + ++outSmpl; + ++inSmpl; -/*-------------------------------------------------------------------* - * IVAS_REND_SetHeadRotation() - * - * - *-------------------------------------------------------------------*/ + } while ( inSmpl != lastInSmpl ); + } + else + { + i = 0; -ivas_error IVAS_REND_SetHeadRotation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_QUATERNION headRot, /* i : head orientations for next rendering call */ - const IVAS_VECTOR3 Pos, /* i : listener positions for next rendering call */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - const IVAS_SPLIT_REND_ROT_AXIS rot_axis, /* i : external control for rotation axis for split rendering */ -#endif - const int16_t sf_idx /* i : subframe index */ + /* Otherwise use weighted average between previous and current gain */ + do + { + fadeIn = (float) i / ( outAudio.config.numSamplesPerChannel - 1 ); + fadeOut = 1.0f - fadeIn; + + *outSmpl += ( fadeIn * currentGain + fadeOut * previousGain ) * ( *inSmpl ); + ++outSmpl; + ++inSmpl; + ++i; + } while ( inSmpl != lastInSmpl ); + } + } + } + + return; +} + + +/* Take one channel from input buffer and copy it to each channel + in output buffer, with different gain applied per output channel */ +static void renderBufferChannel( + const IVAS_REND_AudioBuffer inAudio, + const int32_t inChannelIdx, + const float *const outputGains, + IVAS_REND_AudioBuffer outAudio ) +{ + renderBufferChannelLerp( inAudio, inChannelIdx, outputGains, NULL, outAudio ); + + return; +} + + +static ivas_error chooseCrossfade( + const IVAS_REND_HeadRotData *headRotData, + const float **pCrossfade ) +{ + *pCrossfade = headRotData->crossfade; + + return IVAS_ERR_OK; +} + +static ivas_error rotateFrameMc( + IVAS_REND_AudioBuffer inAudio, /* i : Input Audio buffer */ + AUDIO_CONFIG inConfig, /* i : Input Audio config */ + const LSSETUP_CUSTOM_STRUCT *pInCustomLs, /* i : Input Custom LS setup */ + const IVAS_REND_HeadRotData *headRotData, /* i : Head rotation data */ + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData, /* i : Combined head and external orientations */ + rotation_gains gains_prev, /* i/o: Previous frame rotation gains */ + const EFAP_HANDLE hEFAPdata, /* i : EFAP structure */ + IVAS_REND_AudioBuffer outAudio /* o : Output Audio buffer */ ) { int16_t i; - IVAS_QUATERNION rotQuat; + int16_t j; + const float *crossfade; + int16_t num_subframes; + int16_t subframe_idx, subframe_len; + int16_t azimuth, elevation; + int16_t is_planar_setup, lfe_idx; + int16_t nchan; + int16_t ch_in, ch_out; + int16_t ch_in_woLFE, ch_out_woLFE; + float *readPtr, *writePtr; + const float *ls_azimuth, *ls_elevation; + rotation_matrix Rmat; + rotation_gains gains; + float tmp_gains[MAX_INPUT_CHANNELS]; ivas_error error; - /* Validate function arguments */ - if ( hIvasRend == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } - - if ( getAudioConfigType( hIvasRend->outputConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) + push_wmops( "rotateFrameMc" ); + if ( ( error = chooseCrossfade( headRotData, &crossfade ) ) != IVAS_ERR_OK ) { - /* Head rotation can be set only with binaural output */ - return IVAS_ERR_INVALID_OUTPUT_FORMAT; + return error; } + num_subframes = ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->num_subframes : MAX_PARAM_SPATIAL_SUBFRAMES; - hIvasRend->headRotData.headRotEnabled = 1; - - /* reconfigure binaural rendering to allocate the necessary renderers and free unused ones */ - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) + if ( inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) { - if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + if ( ( error = getAudioConfigNumChannels( inConfig, &nchan ) ) != IVAS_ERR_OK ) { - if ( ( error = initMcBinauralRendering( &hIvasRend->inputsMc[i], hIvasRend->inputsMc[i].base.inConfig, hIvasRend->outputConfig, hIvasRend->hRendererConfig, TRUE ) ) != IVAS_ERR_OK ) - { - return error; - } + return error; } } - - /* check for Euler angle signaling */ - if ( headRot.w == -3.0f ) - { - Euler2Quat( deg2rad( headRot.x ), deg2rad( headRot.y ), deg2rad( headRot.z ), &rotQuat ); - } else { - rotQuat = headRot; + nchan = pInCustomLs->num_spk + pInCustomLs->num_lfe; } - if ( ( error = ivas_orient_trk_Process( hIvasRend->headRotData.hOrientationTracker, rotQuat, FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES, &hIvasRend->headRotData.headPositions[sf_idx] ) ) != IVAS_ERR_OK ) + if ( ( error = getMcConfigValues( inConfig, pInCustomLs, &ls_azimuth, &ls_elevation, &lfe_idx, &is_planar_setup ) ) != IVAS_ERR_OK ) { return error; } - hIvasRend->headRotData.Pos[sf_idx] = Pos; + /* initialize gains to passthrough */ + for ( ch_in = 0; ch_in < nchan; ch_in++ ) + { + set_zero( gains[ch_in], nchan ); + gains[ch_in][ch_in] = 1.f; + } -#ifdef SPLIT_REND_WITH_HEAD_ROT - hIvasRend->headRotData.sr_pose_pred_axis = rot_axis; -#endif + /* subframe loop */ - return IVAS_ERR_OK; -} + subframe_len = inAudio.config.numSamplesPerChannel / num_subframes; + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + { + for ( i = 0; i < 3; i++ ) + { + if ( hCombinedOrientationData != NULL ) + { + for ( j = 0; j < 3; j++ ) + { + Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; + } + } + else + { + /* Set to identity */ + set_zero( Rmat[i], 3 ); + Rmat[i][i] = 1.0f; + } + } + for ( ch_in = 0; ch_in < nchan; ch_in++ ) + { + /* skip LFE */ + if ( ch_in == lfe_idx ) + { + continue; + } -/*-------------------------------------------------------------------* - * IVAS_REND_DisableHeadRotation() - * - * - *-------------------------------------------------------------------*/ + /* input channel index without LFE */ + ch_in_woLFE = ( ( lfe_idx > 0 ) && ( ch_in >= lfe_idx ) ) ? ch_in - 1 : ch_in; -ivas_error IVAS_REND_DisableHeadRotation( - IVAS_REND_HANDLE hIvasRend /* i/o: Renderer handle */ -) -{ - int16_t i; - ivas_error error; + /* gains for current subframe rotation */ + rotateAziEle( ls_azimuth[ch_in_woLFE], ls_elevation[ch_in_woLFE], &azimuth, &elevation, Rmat, is_planar_setup ); - /* Validate function arguments */ - if ( hIvasRend == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + if ( hEFAPdata != NULL && ( ls_azimuth[ch_in_woLFE] != azimuth || ls_elevation[ch_in_woLFE] != elevation ) ) + { + efap_determine_gains( hEFAPdata, tmp_gains, azimuth, elevation, EFAP_MODE_EFAP ); - hIvasRend->headRotData.headRotEnabled = 0; + for ( ch_out = 0; ch_out < nchan; ch_out++ ) + { + /* skip LFE */ + if ( ch_out == lfe_idx ) + { + continue; + } - /* reconfigure binaural rendering to allocate the necessary renderers and free unused ones */ - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) - { - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) + /* output channel index without LFE */ + ch_out_woLFE = ( ( lfe_idx > 0 ) && ( ch_out >= lfe_idx ) ) ? ch_out - 1 : ch_out; + + gains[ch_in][ch_out] = tmp_gains[ch_out_woLFE]; + } + } + } + + /* apply panning gains by mtx multiplication */ + for ( ch_out = 0; ch_out < nchan; ch_out++ ) { - if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + for ( ch_in = 0; ch_in < nchan; ch_in++ ) { - if ( ( error = initMcBinauralRendering( &hIvasRend->inputsMc[i], hIvasRend->inputsMc[i].base.inConfig, hIvasRend->outputConfig, hIvasRend->hRendererConfig, TRUE ) ) != IVAS_ERR_OK ) + writePtr = getSmplPtr( outAudio, ch_out, subframe_idx * subframe_len ); + readPtr = getSmplPtr( inAudio, ch_in, subframe_idx * subframe_len ); + /* crossfade with previous rotation gains */ + for ( i = 0; i < subframe_len; i++ ) { - - return error; + *writePtr++ += + ( *readPtr ) * ( ( 1 - crossfade[i] ) * gains_prev[ch_in][ch_out] ) + + ( *readPtr ) * ( crossfade[i] * gains[ch_in][ch_out] ); + readPtr++; } } } + + /* move gains to gains_prev */ + for ( i = 0; i < nchan; i++ ) + { + mvr2r( gains[i], gains_prev[i], nchan ); + } } + pop_wmops(); return IVAS_ERR_OK; } -#ifdef SPLIT_REND_WITH_HEAD_ROT -/*-------------------------------------------------------------------* - * IVAS_REND_SetSplitRendBFI() - * - * - *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_SetSplitRendBFI( - IVAS_REND_HANDLE hIvasRend, - const int16_t bfi ) +static ivas_error rotateFrameSba( + IVAS_REND_AudioBuffer inAudio, /* i : Input Audio buffer */ + const AUDIO_CONFIG inConfig, /* i : Input Audio config */ + const IVAS_REND_HeadRotData *headRotData, /* i : Head rotation data */ + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData, /* i : Combined head and external orientations */ + rotation_gains gains_prev, /* i/o: Previous frame rotation gains */ + IVAS_REND_AudioBuffer outAudio /* o : Output Audio buffer */ +) { - hIvasRend->splitRendBFI = bfi; + int16_t i, l, n, m; + int16_t m1, m2; + int16_t shd_rot_max_order; + const float *crossfade; + int16_t num_subframes; + int16_t subframe_idx, subframe_len; - return IVAS_ERR_OK; -} -#endif + float *writePtr; + rotation_matrix Rmat; + float tmpRot[2 * HEADROT_ORDER + 1]; + rotation_gains gains; + ivas_error error; + int16_t idx; + float val, cf, oneminuscf; + push_wmops( "rotateFrameSba" ); -/*-------------------------------------------------------------------* - * IVAS_REND_SetOrientationTrackingMode() - * - * - *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_SetOrientationTrackingMode( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_HEAD_ORIENT_TRK_T orientation_tracking /* i : Head orientation tracking type */ -) -{ - return ivas_orient_trk_SetTrackingType( hIvasRend->headRotData.hOrientationTracker, orientation_tracking ); -} - - -/*-------------------------------------------------------------------* - * IVAS_REND_SetReferenceRotation() - * - * - *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_SetReferenceRotation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_QUATERNION refRot /* i : Reference rotation */ -) -{ - ivas_error error; - - /* Validate function arguments */ - if ( hIvasRend == NULL ) + if ( ( error = chooseCrossfade( headRotData, &crossfade ) ) != IVAS_ERR_OK ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + return error; } + num_subframes = ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->num_subframes : MAX_PARAM_SPATIAL_SUBFRAMES; - if ( ( error = ivas_orient_trk_SetReferenceRotation( hIvasRend->headRotData.hOrientationTracker, refRot ) ) != IVAS_ERR_OK ) + if ( ( error = getAmbisonicsOrder( inConfig, &shd_rot_max_order ) ) != IVAS_ERR_OK ) { return error; } - return IVAS_ERR_OK; -} + subframe_len = inAudio.config.numSamplesPerChannel / num_subframes; + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + { + /* initialize rotation matrices with zeros */ + for ( i = 0; i < HEADROT_SHMAT_DIM; i++ ) + { + set_zero( gains[i], HEADROT_SHMAT_DIM ); + } + for ( i = 0; i < 3; i++ ) + { + if ( hCombinedOrientationData != NULL ) + { + for ( l = 0; l < 3; l++ ) + { + Rmat[i][l] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][l]; + } + } + else + { + /* Set to identity */ + set_zero( Rmat[i], 3 ); + Rmat[i][i] = 1.0f; + } + } -/*-------------------------------------------------------------------* - * IVAS_REND_GetMainOrientation() - * - * - *-------------------------------------------------------------------*/ -// ToDo: not used -ivas_error IVAS_REND_GetMainOrientation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_QUATERNION *pOrientation /* i/o: Quaternion pointer for main orientation */ -) -{ - ivas_error error; + /* calculate ambisonics rotation matrices for the previous and current frames */ + SHrotmatgen( gains, Rmat, shd_rot_max_order ); - if ( hIvasRend == NULL || pOrientation == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + for ( i = 0; i < subframe_len; i++ ) + { + idx = subframe_idx * subframe_len + i; + cf = crossfade[i]; + oneminuscf = 1 - cf; + /* As the rotation matrix becomes block diagonal in a SH basis, we can*/ + /* apply each angular-momentum block individually to save complexity. */ - if ( ( error = ivas_orient_trk_GetMainOrientation( hIvasRend->headRotData.hOrientationTracker, pOrientation ) ) != IVAS_ERR_OK ) - { - return error; + /* loop over l blocks */ + m1 = 1; + m2 = 4; + for ( l = 1; l <= shd_rot_max_order; l++ ) + { + /* compute mtx-vector product for this l */ + for ( n = m1; n < m2; n++ ) + { + tmpRot[n - m1] = 0.f; + + for ( m = m1; m < m2; m++ ) + { + val = inAudio.data[m * inAudio.config.numSamplesPerChannel + idx]; + /* crossfade with previous rotation gains */ + tmpRot[n - m1] += ( cf * gains[n][m] * val + oneminuscf * gains_prev[n][m] * val ); + } + } + /* write back the result */ + for ( n = m1; n < m2; n++ ) + { + writePtr = getSmplPtr( outAudio, n, idx ); + ( *writePtr ) = tmpRot[n - m1]; + } + m1 = m2; + m2 += 2 * ( l + 1 ) + 1; + } + } + + /* move SHrotmat to SHrotmat_prev */ + for ( i = 0; i < HEADROT_SHMAT_DIM; i++ ) + { + mvr2r( gains[i], gains_prev[i], HEADROT_SHMAT_DIM ); + } } + pop_wmops(); + return IVAS_ERR_OK; } -/*-------------------------------------------------------------------* - * IVAS_REND_GetTrackedRotation() - * - * - *-------------------------------------------------------------------*/ -// ToDo: not used -ivas_error IVAS_REND_GetTrackedRotation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_QUATERNION *pRotation /* i/o: Quaternion pointer processed rotation */ -) +static ivas_error renderIsmToBinaural( + const input_ism *ismInput, + IVAS_REND_AudioBuffer outAudio ) { + float tmpTDRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; ivas_error error; + int16_t ism_md_subframe_update_ext; - if ( hIvasRend == NULL || pRotation == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + push_wmops( "renderIsmToBinaural" ); + /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ + ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + copyBufferTo2dArray( ismInput->base.inputBuffer, tmpTDRendBuffer ); - if ( ( error = ivas_orient_trk_GetTrackedRotation( hIvasRend->headRotData.hOrientationTracker, pRotation ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, ismInput->hReverb, ism_md_subframe_update_ext, + *ismInput->base.ctx.pOutSampleRate, outAudio.config.numSamplesPerChannel, tmpTDRendBuffer ) ) != IVAS_ERR_OK ) { return error; } + accumulate2dArrayToBuffer( tmpTDRendBuffer, &outAudio ); + + pop_wmops(); + return IVAS_ERR_OK; } -/*---------------------------------------------------------------------* - * IVAS_REND_SetReferenceVector( ) - * - * Sets a reference vector spanning from listenerPos to refPos. Only - * available in OTR_TRACKING_REF_VEC and OTR_TRACKING_REF_VEC_LEV modes. - *---------------------------------------------------------------------*/ - -ivas_error IVAS_REND_SetReferenceVector( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_VECTOR3 listenerPos, /* i : Listener position */ - const IVAS_VECTOR3 refPos /* i : Reference position */ -) +static int16_t getNumSubframesInBuffer( + const IVAS_REND_AudioBuffer *buffer, + const int32_t sampleRate ) { - if ( hIvasRend == NULL || hIvasRend->headRotData.hOrientationTracker == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; - return ivas_orient_trk_SetReferenceVector( hIvasRend->headRotData.hOrientationTracker, listenerPos, refPos ); -} + cldfb2tdSampleFact = buffer->config.is_cldfb ? 2 : 1; +#endif +#ifdef DEBUGGING +#ifdef SPLIT_REND_WITH_HEAD_ROT + assert( buffer->config.numSamplesPerChannel % ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES * cldfb2tdSampleFact ) == 0 ); +#else + assert( buffer->config.numSamplesPerChannel % ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) == 0 ); +#endif +#endif -/*---------------------------------------------------------------------* - * IVAS_REND_SetExternalOrientation() - * - * - *---------------------------------------------------------------------*/ +#ifdef SPLIT_REND_WITH_HEAD_ROT + return (int16_t) ( buffer->config.numSamplesPerChannel / ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES * cldfb2tdSampleFact ) ); +#else + return (int16_t) ( buffer->config.numSamplesPerChannel / ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); +#endif +} -ivas_error IVAS_REND_SetExternalOrientation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_QUATERNION *orientation, /* i : external orientation data */ - int8_t enableHeadRotation, /* i : flag to enable head rotation for this frame */ - int8_t enableExternalOrientation, /* i : flag to enable external orientation for this frame */ - int8_t enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ - int16_t numFramesToTargetOrientation, /* i : number of frames until target orientation is reached */ - const int16_t sf_idx /* i : subframe index */ -) + +static ivas_error renderIsmToBinauralRoom( + input_ism *ismInput, + IVAS_REND_AudioBuffer outAudio ) { - /* Validate function arguments */ - if ( hIvasRend == NULL || hIvasRend->hExternalOrientationData == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + int16_t position_changed; + int16_t i, j; + int16_t azi_rot, ele_rot; + int16_t subframe_idx; + int16_t tmp; + rotation_matrix Rmat; + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + ivas_error error; + pan_vector currentPanGains; + IVAS_REND_AudioBuffer tmpMcBuffer; + IVAS_ISM_METADATA rotatedPosPrev; + IVAS_ISM_METADATA rotatedPos; + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; + int8_t combinedOrientationEnabled; + float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; - if ( orientation == NULL ) + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) { - hIvasRend->hExternalOrientationData->enableExternalOrientation[sf_idx] = 0; + p_tmpRendBuffer[i] = tmpRendBuffer[i]; } - else - { - QuaternionInverse( *orientation, &hIvasRend->hExternalOrientationData->Quaternions[sf_idx] ); - hIvasRend->hExternalOrientationData->enableHeadRotation[sf_idx] = enableHeadRotation; - hIvasRend->hExternalOrientationData->enableExternalOrientation[sf_idx] = enableExternalOrientation; - hIvasRend->hExternalOrientationData->enableRotationInterpolation[sf_idx] = enableRotationInterpolation; - hIvasRend->hExternalOrientationData->numFramesToTargetOrientation[sf_idx] = numFramesToTargetOrientation; - } + push_wmops( "renderIsmToBinauralRoom" ); - return IVAS_ERR_OK; -} + rotatedPosPrev = defaultObjectPosition(); + rotatedPos = defaultObjectPosition(); + hCombinedOrientationData = ismInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = 0; + if ( hCombinedOrientationData != NULL ) + { + for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) + { -/*---------------------------------------------------------------------* - * IVAS_REND_CombineHeadAndExternalOrientation() - * - * - *---------------------------------------------------------------------*/ + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + { + combinedOrientationEnabled = 1; + break; + } + } + } -ivas_error IVAS_REND_CombineHeadAndExternalOrientation( - IVAS_REND_HANDLE hIvasRend /* i/o: Renderer handle */ -) -{ - if ( hIvasRend == NULL ) + if ( combinedOrientationEnabled ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + for ( subframe_idx = 0; subframe_idx < 1; subframe_idx++ ) + { + for ( i = 0; i < 3; i++ ) + { + if ( hCombinedOrientationData != NULL && ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] ) + { + for ( j = 0; j < 3; j++ ) + { + Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; + } + } + else + { + /* Set to identity */ + set_zero( Rmat[i], 3 ); + Rmat[i][i] = 1.0f; + } + } + } } - return combine_external_and_head_orientations_rend( &hIvasRend->headRotData, hIvasRend->hExternalOrientationData, hIvasRend->hCombinedOrientationData ); -} + /* get previous position */ + if ( combinedOrientationEnabled ) + { + rotateAziEle( ismInput->previousPos.azimuth, ismInput->previousPos.elevation, &azi_rot, &ele_rot, ismInput->rot_mat_prev, 0 ); + rotatedPosPrev.azimuth = (float) azi_rot; + rotatedPosPrev.elevation = (float) ele_rot; + } + else + { + rotatedPosPrev.azimuth = ismInput->previousPos.azimuth; + rotatedPosPrev.elevation = ismInput->previousPos.elevation; + } + /* get current position */ + if ( combinedOrientationEnabled ) + { + rotateAziEle( ismInput->currentPos.azimuth, ismInput->currentPos.elevation, &azi_rot, &ele_rot, Rmat, 0 ); + rotatedPos.azimuth = (float) azi_rot; + rotatedPos.elevation = (float) ele_rot; + } + else + { + rotatedPos.azimuth = ismInput->currentPos.azimuth; + rotatedPos.elevation = ismInput->currentPos.elevation; + } -/*---------------------------------------------------------------------* - * IVAS_REND_GetCombinedOrientation() - * - * - *---------------------------------------------------------------------*/ -// ToDo: not used -ivas_error IVAS_REND_GetCombinedOrientation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_QUATERNION *pOrientation /* i/o: Quaternion pointer processed orientation */ -) -{ - int16_t i; + position_changed = !ismInput->firstFrameRendered || checkObjectPositionChanged( &rotatedPos, &rotatedPosPrev ); - if ( hIvasRend == NULL || pOrientation == NULL ) + /* set previous gains if this is the first frame */ + if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, rotatedPosPrev.azimuth, rotatedPosPrev.elevation, ismInput->prev_pan_gains ) ) != IVAS_ERR_OK ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + return error; } - if ( hIvasRend->hCombinedOrientationData != NULL ) + /* compute gains only if position changed */ + if ( position_changed ) { - for ( i = 0; i < hIvasRend->hCombinedOrientationData->num_subframes; ++i ) + if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, + rotatedPos.azimuth, + rotatedPos.elevation, + currentPanGains ) ) != IVAS_ERR_OK ) { - pOrientation[i] = hIvasRend->hCombinedOrientationData->Quaternions[i]; + return error; } } - return IVAS_ERR_OK; -} + /* intermediate rendering to 7_1_4 */ + tmpMcBuffer = ismInput->base.inputBuffer; + if ( ( error = getAudioConfigNumChannels( IVAS_AUDIO_CONFIG_7_1_4, &tmp ) ) != IVAS_ERR_OK ) + { + return error; + } -/*-------------------------------------------------------------------* - * Local functions - *-------------------------------------------------------------------*/ + tmpMcBuffer.config.numChannels = tmp; + tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels ); -/* Take one channel from input buffer and copy it to each channel - in output buffer, with different gain applied per output channel. - This function takes 2 gain vectors - one for the beginning and one - for the end of the buffer. Gain values are lineraly interpolated - for all samples in between. */ -static void renderBufferChannelLerp( - const IVAS_REND_AudioBuffer inAudio, - const int32_t inChannelIdx, - const float *const gainsCurrent, - const float *const gainsPrev, - IVAS_REND_AudioBuffer outAudio ) -{ - const float *inSmpl; - float *outSmpl; - float fadeIn; - float fadeOut; - int32_t i; - const float *lastInSmpl; - int16_t outChnlIdx; - float currentGain; - float previousGain; + renderBufferChannelLerp( ismInput->base.inputBuffer, 0, + position_changed ? currentPanGains : ismInput->prev_pan_gains, + position_changed ? ismInput->prev_pan_gains : NULL, + tmpMcBuffer ); - /* Pointer to behind last input sample */ - lastInSmpl = getSmplPtr( inAudio, inChannelIdx, inAudio.config.numSamplesPerChannel ); + copyBufferTo2dArray( tmpMcBuffer, tmpRendBuffer ); - for ( outChnlIdx = 0; outChnlIdx < outAudio.config.numChannels; ++outChnlIdx ) + /* save gains for next frame */ + for ( i = 0; i < 3; i++ ) { - currentGain = gainsCurrent[outChnlIdx]; - previousGain = gainsPrev == NULL ? 0.f : gainsPrev[outChnlIdx]; + mvr2r( Rmat[i], ismInput->rot_mat_prev[i], 3 ); + } - /* Process current output channel only if applying non-zero gains */ - if ( fabsf( currentGain ) > EPSILON || ( gainsPrev != NULL && fabsf( previousGain ) > EPSILON ) ) - { - /* Reset input pointer to the beginning of input channel */ - inSmpl = getSmplPtr( inAudio, inChannelIdx, 0 ); + if ( position_changed ) + { + mvr2r( currentPanGains, ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); + } - /* Set output pointer to first output channel sample */ - outSmpl = getSmplPtr( outAudio, outChnlIdx, 0 ); - if ( gainsPrev == NULL || fabsf( previousGain - currentGain ) <= EPSILON ) - { - /* If no interpolation from previous frame, apply current gain */ - do - { - *outSmpl += currentGain * ( *inSmpl ); - ++outSmpl; - ++inSmpl; + /* render 7_1_4 with BRIRs */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( ismInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR, + NULL, NULL, NULL, NULL, p_tmpRendBuffer, *ismInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *ismInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_rend_crendProcess( ismInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR, + NULL, NULL, NULL, NULL, p_tmpRendBuffer, *ismInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *ismInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) - } while ( inSmpl != lastInSmpl ); - } - else - { - i = 0; +#endif + { + return error; + } - /* Otherwise use weighted average between previous and current gain */ - do - { - fadeIn = (float) i / ( outAudio.config.numSamplesPerChannel - 1 ); - fadeOut = 1.0f - fadeIn; + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - *outSmpl += ( fadeIn * currentGain + fadeOut * previousGain ) * ( *inSmpl ); - ++outSmpl; - ++inSmpl; - ++i; - } while ( inSmpl != lastInSmpl ); - } - } - } + free( tmpMcBuffer.data ); + pop_wmops(); - return; + return IVAS_ERR_OK; } -/* Take one channel from input buffer and copy it to each channel - in output buffer, with different gain applied per output channel */ -static void renderBufferChannel( - const IVAS_REND_AudioBuffer inAudio, - const int32_t inChannelIdx, - const float *const outputGains, +static ivas_error renderIsmToBinauralReverb( + input_ism *ismInput, IVAS_REND_AudioBuffer outAudio ) { - renderBufferChannelLerp( inAudio, inChannelIdx, outputGains, NULL, outAudio ); + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + ivas_error error; + int16_t ism_md_subframe_update_ext; - return; -} + push_wmops( "renderIsmToBinauralRoom" ); + /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ + ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); -static ivas_error chooseCrossfade( - const IVAS_REND_HeadRotData *headRotData, - const float **pCrossfade ) -{ - *pCrossfade = headRotData->crossfade; + if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, ismInput->hReverb, + ism_md_subframe_update_ext, *ismInput->base.ctx.pOutSampleRate, outAudio.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); + pop_wmops(); return IVAS_ERR_OK; } -static ivas_error rotateFrameMc( - IVAS_REND_AudioBuffer inAudio, /* i : Input Audio buffer */ - AUDIO_CONFIG inConfig, /* i : Input Audio config */ - const LSSETUP_CUSTOM_STRUCT *pInCustomLs, /* i : Input Custom LS setup */ - const IVAS_REND_HeadRotData *headRotData, /* i : Head rotation data */ - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData, /* i : Combined head and external orientations */ - rotation_gains gains_prev, /* i/o: Previous frame rotation gains */ - const EFAP_HANDLE hEFAPdata, /* i : EFAP structure */ - IVAS_REND_AudioBuffer outAudio /* o : Output Audio buffer */ -) -{ - int16_t i; - int16_t j; - const float *crossfade; - int16_t num_subframes; - int16_t subframe_idx, subframe_len; - int16_t azimuth, elevation; - int16_t is_planar_setup, lfe_idx; - int16_t nchan; - int16_t ch_in, ch_out; - int16_t ch_in_woLFE, ch_out_woLFE; - float *readPtr, *writePtr; - const float *ls_azimuth, *ls_elevation; - rotation_matrix Rmat; - rotation_gains gains; - float tmp_gains[MAX_INPUT_CHANNELS]; - ivas_error error; - push_wmops( "rotateFrameMc" ); - if ( ( error = chooseCrossfade( headRotData, &crossfade ) ) != IVAS_ERR_OK ) - { - return error; - } - num_subframes = ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->num_subframes : MAX_PARAM_SPATIAL_SUBFRAMES; +static ivas_error renderIsmToMc( + input_ism *ismInput, + const IVAS_REND_AudioBuffer outAudio ) +{ + int8_t position_changed; + pan_vector currentPanGains; + ivas_error error; - if ( inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) + push_wmops( "renderIsmToMc" ); + + position_changed = !ismInput->firstFrameRendered || checkObjectPositionChanged( &ismInput->currentPos, &ismInput->previousPos ); + if ( *ismInput->base.ctx.pOutConfig == IVAS_AUDIO_CONFIG_STEREO ) { - if ( ( error = getAudioConfigNumChannels( inConfig, &nchan ) ) != IVAS_ERR_OK ) + if ( ismInput->nonDiegeticPan ) { - return error; + ismInput->prev_pan_gains[0] = currentPanGains[0] = ( ismInput->nonDiegeticPanGain + 1.f ) * 0.5f; + ismInput->prev_pan_gains[1] = currentPanGains[1] = 1.f - currentPanGains[0]; } - } - else - { - nchan = pInCustomLs->num_spk + pInCustomLs->num_lfe; - } - - if ( ( error = getMcConfigValues( inConfig, pInCustomLs, &ls_azimuth, &ls_elevation, &lfe_idx, &is_planar_setup ) ) != IVAS_ERR_OK ) - { - return error; - } + else + { + set_zero( currentPanGains, MAX_OUTPUT_CHANNELS ); - /* initialize gains to passthrough */ - for ( ch_in = 0; ch_in < nchan; ch_in++ ) - { - set_zero( gains[ch_in], nchan ); - gains[ch_in][ch_in] = 1.f; - } + ivas_ism_get_stereo_gains( ismInput->currentPos.azimuth, ismInput->currentPos.elevation, ¤tPanGains[0], ¤tPanGains[1] ); - /* subframe loop */ + set_zero( ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); - subframe_len = inAudio.config.numSamplesPerChannel / num_subframes; - for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + ivas_ism_get_stereo_gains( ismInput->previousPos.azimuth, ismInput->previousPos.elevation, &ismInput->prev_pan_gains[0], &ismInput->prev_pan_gains[1] ); + } + } + else { - for ( i = 0; i < 3; i++ ) + /* compute gains only if position changed */ + if ( position_changed ) { - if ( hCombinedOrientationData != NULL ) - { - for ( j = 0; j < 3; j++ ) - { - Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; - } - } - else + // TODO tmu review when #215 is resolved + if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, + (int16_t) floorf( ismInput->currentPos.azimuth + 0.5f ), + (int16_t) floorf( ismInput->currentPos.elevation + 0.5f ), + currentPanGains ) ) != IVAS_ERR_OK ) { - /* Set to identity */ - set_zero( Rmat[i], 3 ); - Rmat[i][i] = 1.0f; + return error; } } - for ( ch_in = 0; ch_in < nchan; ch_in++ ) + /* set previous gains if this is the first frame */ + if ( !ismInput->firstFrameRendered ) { - /* skip LFE */ - if ( ch_in == lfe_idx ) - { - continue; - } - - /* input channel index without LFE */ - ch_in_woLFE = ( ( lfe_idx > 0 ) && ( ch_in >= lfe_idx ) ) ? ch_in - 1 : ch_in; - - /* gains for current subframe rotation */ - rotateAziEle( ls_azimuth[ch_in_woLFE], ls_elevation[ch_in_woLFE], &azimuth, &elevation, Rmat, is_planar_setup ); - - if ( hEFAPdata != NULL && ( ls_azimuth[ch_in_woLFE] != azimuth || ls_elevation[ch_in_woLFE] != elevation ) ) + // TODO tmu review when #215 is resolved + if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, + (int16_t) floorf( ismInput->previousPos.azimuth + 0.5f ), + (int16_t) floorf( ismInput->previousPos.elevation + 0.5f ), + ismInput->prev_pan_gains ) ) != IVAS_ERR_OK ) { - efap_determine_gains( hEFAPdata, tmp_gains, azimuth, elevation, EFAP_MODE_EFAP ); - - for ( ch_out = 0; ch_out < nchan; ch_out++ ) - { - /* skip LFE */ - if ( ch_out == lfe_idx ) - { - continue; - } - - /* output channel index without LFE */ - ch_out_woLFE = ( ( lfe_idx > 0 ) && ( ch_out >= lfe_idx ) ) ? ch_out - 1 : ch_out; - - gains[ch_in][ch_out] = tmp_gains[ch_out_woLFE]; - } + return error; } } + } - /* apply panning gains by mtx multiplication */ - for ( ch_out = 0; ch_out < nchan; ch_out++ ) - { - for ( ch_in = 0; ch_in < nchan; ch_in++ ) - { - writePtr = getSmplPtr( outAudio, ch_out, subframe_idx * subframe_len ); - readPtr = getSmplPtr( inAudio, ch_in, subframe_idx * subframe_len ); - /* crossfade with previous rotation gains */ - for ( i = 0; i < subframe_len; i++ ) - { - *writePtr++ += - ( *readPtr ) * ( ( 1 - crossfade[i] ) * gains_prev[ch_in][ch_out] ) + - ( *readPtr ) * ( crossfade[i] * gains[ch_in][ch_out] ); - readPtr++; - } - } - } + /* Assume num channels in audio buffer to be 1. + * This should have been validated in IVAS_REND_FeedInputAudio() */ + renderBufferChannelLerp( ismInput->base.inputBuffer, 0, + position_changed ? currentPanGains : ismInput->prev_pan_gains, + position_changed ? ismInput->prev_pan_gains : NULL, + outAudio ); - /* move gains to gains_prev */ - for ( i = 0; i < nchan; i++ ) - { - mvr2r( gains[i], gains_prev[i], nchan ); - } + if ( position_changed ) + { + mvr2r( currentPanGains, ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); } pop_wmops(); + return IVAS_ERR_OK; } -static ivas_error rotateFrameSba( - IVAS_REND_AudioBuffer inAudio, /* i : Input Audio buffer */ - const AUDIO_CONFIG inConfig, /* i : Input Audio config */ - const IVAS_REND_HeadRotData *headRotData, /* i : Head rotation data */ - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData, /* i : Combined head and external orientations */ - rotation_gains gains_prev, /* i/o: Previous frame rotation gains */ - IVAS_REND_AudioBuffer outAudio /* o : Output Audio buffer */ -) +static ivas_error renderIsmToSba( + input_ism *ismInput, + const AUDIO_CONFIG outConfig, + const IVAS_REND_AudioBuffer outAudio ) { - int16_t i, l, n, m; - int16_t m1, m2; - int16_t shd_rot_max_order; - const float *crossfade; - int16_t num_subframes; - int16_t subframe_idx, subframe_len; - - float *writePtr; - rotation_matrix Rmat; - float tmpRot[2 * HEADROT_ORDER + 1]; - rotation_gains gains; + int8_t position_changed; + int16_t ambiOrderOut; + int16_t numOutChannels; + pan_vector currentPanGains; ivas_error error; - int16_t idx; - float val, cf, oneminuscf; + error = IVAS_ERR_OK; - push_wmops( "rotateFrameSba" ); + push_wmops( "renderIsmToSba" ); - if ( ( error = chooseCrossfade( headRotData, &crossfade ) ) != IVAS_ERR_OK ) + if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) { return error; } - num_subframes = ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->num_subframes : MAX_PARAM_SPATIAL_SUBFRAMES; - if ( ( error = getAmbisonicsOrder( inConfig, &shd_rot_max_order ) ) != IVAS_ERR_OK ) + if ( ( error = getAmbisonicsOrder( outConfig, &ambiOrderOut ) ) != IVAS_ERR_OK ) { return error; } - subframe_len = inAudio.config.numSamplesPerChannel / num_subframes; - for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + position_changed = !ismInput->firstFrameRendered || checkObjectPositionChanged( &ismInput->currentPos, &ismInput->previousPos ); + + /* set previous gains if this is the first frame */ + if ( !ismInput->firstFrameRendered ) { - /* initialize rotation matrices with zeros */ - for ( i = 0; i < HEADROT_SHMAT_DIM; i++ ) - { - set_zero( gains[i], HEADROT_SHMAT_DIM ); - } + // TODO tmu review when #215 is resolved + ivas_dirac_dec_get_response( (int16_t) floorf( ismInput->previousPos.azimuth + 0.5f ), + (int16_t) floorf( ismInput->previousPos.elevation + 0.5f ), + ismInput->prev_pan_gains, + ambiOrderOut ); + } - for ( i = 0; i < 3; i++ ) - { - if ( hCombinedOrientationData != NULL ) - { - for ( l = 0; l < 3; l++ ) - { - Rmat[i][l] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][l]; - } - } - else - { - /* Set to identity */ - set_zero( Rmat[i], 3 ); - Rmat[i][i] = 1.0f; - } - } + /* compute gains only if position changed */ + if ( position_changed ) + { + // TODO tmu review when #215 is resolved + ivas_dirac_dec_get_response( (int16_t) floorf( ismInput->currentPos.azimuth + 0.5f ), + (int16_t) floorf( ismInput->currentPos.elevation + 0.5f ), + currentPanGains, + ambiOrderOut ); + } - /* calculate ambisonics rotation matrices for the previous and current frames */ - SHrotmatgen( gains, Rmat, shd_rot_max_order ); + /* Assume num channels in audio buffer to be 1. + * This should have been validated in IVAS_REND_FeedInputAudio() */ + renderBufferChannelLerp( ismInput->base.inputBuffer, 0, + position_changed ? currentPanGains : ismInput->prev_pan_gains, + position_changed ? ismInput->prev_pan_gains : NULL, + outAudio ); - for ( i = 0; i < subframe_len; i++ ) - { - idx = subframe_idx * subframe_len + i; - cf = crossfade[i]; - oneminuscf = 1 - cf; - /* As the rotation matrix becomes block diagonal in a SH basis, we can*/ - /* apply each angular-momentum block individually to save complexity. */ + if ( position_changed ) + { + mvr2r( currentPanGains, ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); + } + pop_wmops(); - /* loop over l blocks */ - m1 = 1; - m2 = 4; - for ( l = 1; l <= shd_rot_max_order; l++ ) - { - /* compute mtx-vector product for this l */ - for ( n = m1; n < m2; n++ ) - { - tmpRot[n - m1] = 0.f; + return error; +} - for ( m = m1; m < m2; m++ ) - { - val = inAudio.data[m * inAudio.config.numSamplesPerChannel + idx]; - /* crossfade with previous rotation gains */ - tmpRot[n - m1] += ( cf * gains[n][m] * val + oneminuscf * gains_prev[n][m] * val ); - } - } - /* write back the result */ - for ( n = m1; n < m2; n++ ) + +#ifdef SPLIT_REND_WITH_HEAD_ROT +static ivas_error renderIsmToSplitBinaural( + input_ism *ismInput, + const IVAS_REND_AudioBuffer outAudio ) +{ + ivas_error error; + float tmpProcessing[MAX_NUM_OBJECTS][L_FRAME48k]; + int16_t pos_idx; + const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + const SPLIT_REND_WRAPPER *pSplitRendWrapper; + IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES]; + int16_t i; + float tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; + int16_t output_frame = ismInput->base.inputBuffer.config.numSamplesPerChannel; + COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; + int16_t ism_md_subframe_update_ext; + + push_wmops( "renderIsmToSplitBinaural" ); + + pSplitRendWrapper = ismInput->base.ctx.pSplitRendWrapper; + pMultiBinPoseData = &pSplitRendWrapper->multiBinPoseData; + + /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ + ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + + pCombinedOrientationData = *ismInput->base.ctx.pCombinedOrientationData; + + if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) + { + for ( i = 1; i < pCombinedOrientationData->num_subframes; ++i ) + { + pCombinedOrientationData->Quaternions[i] = pCombinedOrientationData->Quaternions[0]; + } + } + + /* Save current head positions */ + for ( i = 0; i < pCombinedOrientationData->num_subframes; ++i ) + { + originalHeadRot[i] = pCombinedOrientationData->Quaternions[i]; + } + + /* Copy input audio to a processing buffer. */ + copyBufferTo2dArray( ismInput->base.inputBuffer, tmpProcessing ); + + for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + { + /* Update head positions */ + if ( pos_idx != 0 ) + { + for ( i = 0; i < pCombinedOrientationData->num_subframes; ++i ) + { + if ( originalHeadRot[i].w == -3.0f ) { - writePtr = getSmplPtr( outAudio, n, idx ); - ( *writePtr ) = tmpRot[n - m1]; + pCombinedOrientationData->Quaternions[i].w = -3.0f; + pCombinedOrientationData->Quaternions[i].x = originalHeadRot[i].x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; + pCombinedOrientationData->Quaternions[i].y = originalHeadRot[i].y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; + pCombinedOrientationData->Quaternions[i].z = originalHeadRot[i].z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; + } + else + { + pCombinedOrientationData->Quaternions[i].w = -3.0f; + Quat2EulerDegree( originalHeadRot[i], + &pCombinedOrientationData->Quaternions[i].z, + &pCombinedOrientationData->Quaternions[i].y, + &pCombinedOrientationData->Quaternions[i].x ); + pCombinedOrientationData->Quaternions[i].x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + pCombinedOrientationData->Quaternions[i].y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + pCombinedOrientationData->Quaternions[i].z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; } - m1 = m2; - m2 += 2 * ( l + 1 ) + 1; } } - /* move SHrotmat to SHrotmat_prev */ - for ( i = 0; i < HEADROT_SHMAT_DIM; i++ ) + /* Render */ + if ( ( error = ivas_td_binaural_renderer_ext( ( pos_idx == 0 ) ? &ismInput->tdRendWrapper : &ismInput->splitTdRendWrappers[pos_idx - 1], ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, + NULL, ism_md_subframe_update_ext, *ismInput->base.ctx.pOutSampleRate, output_frame, tmpProcessing ) ) != IVAS_ERR_OK ) { - mvr2r( gains[i], gains_prev[i], HEADROT_SHMAT_DIM ); + return error; } + + /* Copy rendered audio to tmp storage buffer. Copying directly to output would + * overwrite original audio, which is still needed for rendering next head pose. */ + mvr2r( tmpProcessing[0], tmpBinaural[2 * pos_idx], output_frame ); + mvr2r( tmpProcessing[1], tmpBinaural[2 * pos_idx + 1], output_frame ); + + /* Overwrite processing buffer with original input audio again */ + copyBufferTo2dArray( ismInput->base.inputBuffer, tmpProcessing ); } + /* Restore original head rotation */ + for ( i = 0; i < pCombinedOrientationData->num_subframes; ++i ) + { + pCombinedOrientationData->Quaternions[i] = originalHeadRot[i]; + } + + accumulate2dArrayToBuffer( tmpBinaural, &outAudio ); pop_wmops(); + /* Encoding to split rendering bitstream done at a higher level */ return IVAS_ERR_OK; } +#endif -static ivas_error renderIsmToBinaural( - const input_ism *ismInput, +static void renderIsmToMasa( + input_ism *ismInput, IVAS_REND_AudioBuffer outAudio ) { - float tmpTDRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - ivas_error error; - int16_t ism_md_subframe_update_ext; + float tmpRendBuffer[MAX_NUM_OBJECTS][L_FRAME48k]; - push_wmops( "renderIsmToBinaural" ); - /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ - ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); - copyBufferTo2dArray( ismInput->base.inputBuffer, tmpTDRendBuffer ); + push_wmops( "renderIsmToMasa" ); - if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, ismInput->hReverb, ism_md_subframe_update_ext, - *ismInput->base.ctx.pOutSampleRate, outAudio.config.numSamplesPerChannel, tmpTDRendBuffer ) ) != IVAS_ERR_OK ) - { - return error; - } + copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); - accumulate2dArrayToBuffer( tmpTDRendBuffer, &outAudio ); + ivas_omasa_ana( ismInput->hOMasa, tmpRendBuffer, ismInput->base.inputBuffer.config.numSamplesPerChannel, outAudio.config.numChannels, ismInput->base.inputBuffer.config.numChannels ); + + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); pop_wmops(); - return IVAS_ERR_OK; + return; } -static int16_t getNumSubframesInBuffer( - const IVAS_REND_AudioBuffer *buffer, - const int32_t sampleRate ) +static ivas_error renderInputIsm( + input_ism *ismInput, + const AUDIO_CONFIG outConfig, + const IVAS_REND_AudioBuffer outAudio ) { -#ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t cldfb2tdSampleFact; + ivas_error error; + IVAS_REND_AudioBuffer inAudio; - cldfb2tdSampleFact = buffer->config.is_cldfb ? 2 : 1; -#endif + error = IVAS_ERR_OK; + inAudio = ismInput->base.inputBuffer; -#ifdef DEBUGGING -#ifdef SPLIT_REND_WITH_HEAD_ROT - assert( buffer->config.numSamplesPerChannel % ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES * cldfb2tdSampleFact ) == 0 ); -#else - assert( buffer->config.numSamplesPerChannel % ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) == 0 ); -#endif -#endif + if ( ismInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) + { + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); + } + ismInput->base.numNewSamplesPerChannel = 0; + + /* Apply input gain to new audio */ + v_multc( inAudio.data, ismInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + error = renderIsmToMc( ismInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + error = renderIsmToSba( ismInput, outConfig, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + switch ( outConfig ) + { + case IVAS_AUDIO_CONFIG_BINAURAL: + error = renderIsmToBinaural( ismInput, outAudio ); + break; + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: + error = renderIsmToBinauralRoom( ismInput, outAudio ); + break; #ifdef SPLIT_REND_WITH_HEAD_ROT - return (int16_t) ( buffer->config.numSamplesPerChannel / ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES * cldfb2tdSampleFact ) ); -#else - return (int16_t) ( buffer->config.numSamplesPerChannel / ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + error = renderIsmToSplitBinaural( ismInput, outAudio ); + break; #endif + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: + error = renderIsmToBinauralReverb( ismInput, outAudio ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + renderIsmToMasa( ismInput, outAudio ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + + /* Check error here to keep switch statement more compact */ + if ( error != IVAS_ERR_OK ) + { + return error; + } + + ismInput->firstFrameRendered = TRUE; + + return error; } -static ivas_error renderIsmToBinauralRoom( - input_ism *ismInput, +static ivas_error renderActiveInputsIsm( + IVAS_REND_HANDLE hIvasRend, IVAS_REND_AudioBuffer outAudio ) { - int16_t position_changed; - int16_t i, j; - int16_t azi_rot, ele_rot; - int16_t subframe_idx; - int16_t tmp; - rotation_matrix Rmat; - float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + int16_t i; + input_ism *pCurrentInput; ivas_error error; - pan_vector currentPanGains; - IVAS_REND_AudioBuffer tmpMcBuffer; - IVAS_ISM_METADATA rotatedPosPrev; - IVAS_ISM_METADATA rotatedPos; - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; - int8_t combinedOrientationEnabled; - float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; - - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) - { - p_tmpRendBuffer[i] = tmpRendBuffer[i]; - } - push_wmops( "renderIsmToBinauralRoom" ); - - rotatedPosPrev = defaultObjectPosition(); - rotatedPos = defaultObjectPosition(); - - hCombinedOrientationData = ismInput->base.ctx.pCombinedOrientationData; - combinedOrientationEnabled = 0; - if ( hCombinedOrientationData != NULL ) + for ( i = 0, pCurrentInput = hIvasRend->inputsIsm; i < RENDERER_MAX_ISM_INPUTS; ++i, ++pCurrentInput ) { - for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) + if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) { + /* Skip inactive inputs */ + continue; + } - if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) - { - combinedOrientationEnabled = 1; - break; - } + if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) + { + return error; } } - if ( combinedOrientationEnabled ) - { - for ( subframe_idx = 0; subframe_idx < 1; subframe_idx++ ) - { - for ( i = 0; i < 3; i++ ) - { - if ( hCombinedOrientationData != NULL && ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] ) - { - for ( j = 0; j < 3; j++ ) - { - Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; - } - } - else - { - /* Set to identity */ - set_zero( Rmat[i], 3 ); - Rmat[i][i] = 1.0f; - } - } - } - } + return IVAS_ERR_OK; +} - /* get previous position */ - if ( combinedOrientationEnabled ) - { - rotateAziEle( ismInput->previousPos.azimuth, ismInput->previousPos.elevation, &azi_rot, &ele_rot, ismInput->rot_mat_prev, 0 ); - rotatedPosPrev.azimuth = (float) azi_rot; - rotatedPosPrev.elevation = (float) ele_rot; - } - else + +static ivas_error renderLfeToBinaural( + const input_mc *mcInput, +#ifdef SPLIT_REND_WITH_HEAD_ROT + const AUDIO_CONFIG outConfig, +#endif + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t lfe_idx; +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t pose_idx, num_poses; +#endif + float gain; + int16_t ear_idx; + float tmpLfeBuffer[MAX_BUFFER_LENGTH_PER_CHANNEL]; + int16_t frame_size, num_cpy_smpl_cur_frame, num_cpy_smpl_prev_frame; + const float *lfeInput; + float *writePtr; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + assert( ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) && "Must be binaural output" ); +#else + assert( ( outAudio.config.numChannels == 2 ) && "Must be binaural output" ); +#endif + + push_wmops( "renderLfeToBinaural" ); + + gain = GAIN_LFE; + + if ( mcInput->base.inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) { - rotatedPosPrev.azimuth = ismInput->previousPos.azimuth; - rotatedPosPrev.elevation = ismInput->previousPos.elevation; + lfe_idx = LFE_CHANNEL; } - - /* get current position */ - if ( combinedOrientationEnabled ) + else if ( mcInput->customLsInput.num_lfe > 0 ) { - rotateAziEle( ismInput->currentPos.azimuth, ismInput->currentPos.elevation, &azi_rot, &ele_rot, Rmat, 0 ); - rotatedPos.azimuth = (float) azi_rot; - rotatedPos.elevation = (float) ele_rot; + lfe_idx = mcInput->customLsInput.lfe_idx[0]; } else { - rotatedPos.azimuth = ismInput->currentPos.azimuth; - rotatedPos.elevation = ismInput->currentPos.elevation; + /* no LFE to render */ + return IVAS_ERR_OK; } - position_changed = !ismInput->firstFrameRendered || checkObjectPositionChanged( &rotatedPos, &rotatedPosPrev ); + /* --- Prepare LFE signal to be added to binaural output --- */ + lfeInput = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); + frame_size = mcInput->base.inputBuffer.config.numSamplesPerChannel; + num_cpy_smpl_prev_frame = mcInput->binauralDelaySmp; + num_cpy_smpl_cur_frame = frame_size - num_cpy_smpl_prev_frame; - /* set previous gains if this is the first frame */ - if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, rotatedPosPrev.azimuth, rotatedPosPrev.elevation, ismInput->prev_pan_gains ) ) != IVAS_ERR_OK ) - { - return error; - } + /* Assuming LFE should be delayed by less that the duration of one frame */ + assert( mcInput->binauralDelaySmp < frame_size ); - /* compute gains only if position changed */ - if ( position_changed ) - { - if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, - rotatedPos.azimuth, - rotatedPos.elevation, - currentPanGains ) ) != IVAS_ERR_OK ) - { - return error; - } - } + /* Get delayed LFE signal from previous frame, apply gain and save in tmp buffer */ + v_multc( mcInput->lfeDelayBuffer, gain, tmpLfeBuffer, num_cpy_smpl_prev_frame ); - /* intermediate rendering to 7_1_4 */ - tmpMcBuffer = ismInput->base.inputBuffer; + /* Continue filling tmp buffer, now with LFE signal from current frame */ + v_multc( lfeInput, gain, tmpLfeBuffer + num_cpy_smpl_prev_frame, num_cpy_smpl_cur_frame ); - if ( ( error = getAudioConfigNumChannels( IVAS_AUDIO_CONFIG_7_1_4, &tmp ) ) != IVAS_ERR_OK ) + /* Save remaining LFE samples of current frame for next frame */ + mvr2r( lfeInput + num_cpy_smpl_cur_frame, mcInput->lfeDelayBuffer, num_cpy_smpl_prev_frame ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + /* Copy LFE to left and right binaural channels for all poses */ + if ( mcInput->base.ctx.pSplitRendWrapper != NULL ) { - return error; + num_poses = mcInput->base.ctx.pSplitRendWrapper->multiBinPoseData.num_poses; } - - tmpMcBuffer.config.numChannels = tmp; - tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels ); - - renderBufferChannelLerp( ismInput->base.inputBuffer, 0, - position_changed ? currentPanGains : ismInput->prev_pan_gains, - position_changed ? ismInput->prev_pan_gains : NULL, - tmpMcBuffer ); - - copyBufferTo2dArray( tmpMcBuffer, tmpRendBuffer ); - - /* save gains for next frame */ - for ( i = 0; i < 3; i++ ) + else { - mvr2r( Rmat[i], ismInput->rot_mat_prev[i], 3 ); + num_poses = 1; } - if ( position_changed ) + for ( pose_idx = 0; pose_idx < num_poses; ++pose_idx ) { - mvr2r( currentPanGains, ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); + for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) + { + writePtr = getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS + ear_idx, 0 ); + v_add( writePtr, tmpLfeBuffer, writePtr, frame_size ); + } } - - - /* render 7_1_4 with BRIRs */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = ivas_rend_crendProcess( ismInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR, - NULL, NULL, NULL, NULL, p_tmpRendBuffer, *ismInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *ismInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_rend_crendProcess( ismInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR, - NULL, NULL, NULL, NULL, p_tmpRendBuffer, *ismInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *ismInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) - -#endif +#else /* SPLIT_REND_WITH_HEAD_ROT */ + /* Copy LFE to left and right ears */ + for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) { - return error; + writePtr = getSmplPtr( outAudio, ear_idx, 0 ); + v_add( writePtr, tmpLfeBuffer, writePtr, frame_size ); } +#endif /* SPLIT_REND_WITH_HEAD_ROT */ - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - - free( tmpMcBuffer.data ); pop_wmops(); return IVAS_ERR_OK; } -static ivas_error renderIsmToBinauralReverb( - input_ism *ismInput, +static ivas_error renderMcToBinaural( + input_mc *mcInput, + const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + AUDIO_CONFIG inConfig; ivas_error error; - int16_t ism_md_subframe_update_ext; - - push_wmops( "renderIsmToBinauralRoom" ); - - /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ - ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); - copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); + IVAS_REND_AudioBuffer tmpRotBuffer; + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; + int8_t combinedOrientationEnabled; + int16_t subframe_idx; + float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; + int16_t i; - if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, ismInput->hReverb, - ism_md_subframe_update_ext, *ismInput->base.ctx.pOutSampleRate, outAudio.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) { - return error; + p_tmpRendBuffer[i] = tmpRendBuffer[i]; } - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - pop_wmops(); - - return IVAS_ERR_OK; -} - - -static ivas_error renderIsmToMc( - input_ism *ismInput, - const IVAS_REND_AudioBuffer outAudio ) -{ - int8_t position_changed; - pan_vector currentPanGains; - ivas_error error; - - push_wmops( "renderIsmToMc" ); + push_wmops( "renderMcToBinaural" ); + inConfig = mcInput->base.inConfig; - position_changed = !ismInput->firstFrameRendered || checkObjectPositionChanged( &ismInput->currentPos, &ismInput->previousPos ); - if ( *ismInput->base.ctx.pOutConfig == IVAS_AUDIO_CONFIG_STEREO ) + hCombinedOrientationData = mcInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = 0; + if ( hCombinedOrientationData != NULL ) { - if ( ismInput->nonDiegeticPan ) + for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) { - ismInput->prev_pan_gains[0] = currentPanGains[0] = ( ismInput->nonDiegeticPanGain + 1.f ) * 0.5f; - ismInput->prev_pan_gains[1] = currentPanGains[1] = 1.f - currentPanGains[0]; + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + { + combinedOrientationEnabled = 1; + break; + } } - else - { - set_zero( currentPanGains, MAX_OUTPUT_CHANNELS ); - - ivas_ism_get_stereo_gains( ismInput->currentPos.azimuth, ismInput->currentPos.elevation, ¤tPanGains[0], ¤tPanGains[1] ); + } - set_zero( ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); + if ( ( inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM ) || ( combinedOrientationEnabled && ( inConfig == IVAS_AUDIO_CONFIG_5_1 || inConfig == IVAS_AUDIO_CONFIG_7_1 ) ) ) + { + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); - ivas_ism_get_stereo_gains( ismInput->previousPos.azimuth, ismInput->previousPos.elevation, &ismInput->prev_pan_gains[0], &ismInput->prev_pan_gains[1] ); + if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pCombinedOrientationData, NULL, mcInput->hReverb, + 0, *mcInput->base.ctx.pOutSampleRate, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + { + return error; } } else { - /* compute gains only if position changed */ - if ( position_changed ) + /* apply rotation */ + if ( combinedOrientationEnabled ) { - // TODO tmu review when #215 is resolved - if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, - (int16_t) floorf( ismInput->currentPos.azimuth + 0.5f ), - (int16_t) floorf( ismInput->currentPos.elevation + 0.5f ), - currentPanGains ) ) != IVAS_ERR_OK ) - { - return error; - } - } + tmpRotBuffer = mcInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); - /* set previous gains if this is the first frame */ - if ( !ismInput->firstFrameRendered ) - { - // TODO tmu review when #215 is resolved - if ( ( error = getEfapGains( *ismInput->base.ctx.pEfapOutWrapper, - (int16_t) floorf( ismInput->previousPos.azimuth + 0.5f ), - (int16_t) floorf( ismInput->previousPos.elevation + 0.5f ), - ismInput->prev_pan_gains ) ) != IVAS_ERR_OK ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, mcInput->rot_gains_prev[0], mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) +#else + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, mcInput->rot_gains_prev, mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) +#endif { return error; } - } - } - /* Assume num channels in audio buffer to be 1. - * This should have been validated in IVAS_REND_FeedInputAudio() */ - renderBufferChannelLerp( ismInput->base.inputBuffer, 0, - position_changed ? currentPanGains : ismInput->prev_pan_gains, - position_changed ? ismInput->prev_pan_gains : NULL, - outAudio ); + copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); + free( tmpRotBuffer.data ); + } + else + { + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + } - if ( position_changed ) - { - mvr2r( currentPanGains, ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); + /* call CREND */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) +#endif + { + return error; + } } - pop_wmops(); - - return IVAS_ERR_OK; -} - - -static ivas_error renderIsmToSba( - input_ism *ismInput, - const AUDIO_CONFIG outConfig, - const IVAS_REND_AudioBuffer outAudio ) -{ - int8_t position_changed; - int16_t ambiOrderOut; - int16_t numOutChannels; - pan_vector currentPanGains; - ivas_error error; - error = IVAS_ERR_OK; - - push_wmops( "renderIsmToSba" ); + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) - { - return error; - } +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) +#else + if ( ( error = renderLfeToBinaural( mcInput, outAudio ) ) != IVAS_ERR_OK ) +#endif - if ( ( error = getAmbisonicsOrder( outConfig, &ambiOrderOut ) ) != IVAS_ERR_OK ) { return error; } - position_changed = !ismInput->firstFrameRendered || checkObjectPositionChanged( &ismInput->currentPos, &ismInput->previousPos ); - - /* set previous gains if this is the first frame */ - if ( !ismInput->firstFrameRendered ) - { - // TODO tmu review when #215 is resolved - ivas_dirac_dec_get_response( (int16_t) floorf( ismInput->previousPos.azimuth + 0.5f ), - (int16_t) floorf( ismInput->previousPos.elevation + 0.5f ), - ismInput->prev_pan_gains, - ambiOrderOut ); - } - - /* compute gains only if position changed */ - if ( position_changed ) - { - // TODO tmu review when #215 is resolved - ivas_dirac_dec_get_response( (int16_t) floorf( ismInput->currentPos.azimuth + 0.5f ), - (int16_t) floorf( ismInput->currentPos.elevation + 0.5f ), - currentPanGains, - ambiOrderOut ); - } - - /* Assume num channels in audio buffer to be 1. - * This should have been validated in IVAS_REND_FeedInputAudio() */ - renderBufferChannelLerp( ismInput->base.inputBuffer, 0, - position_changed ? currentPanGains : ismInput->prev_pan_gains, - position_changed ? ismInput->prev_pan_gains : NULL, - outAudio ); - - if ( position_changed ) - { - mvr2r( currentPanGains, ismInput->prev_pan_gains, MAX_OUTPUT_CHANNELS ); - } pop_wmops(); - - return error; + return IVAS_ERR_OK; } -#ifdef SPLIT_REND_WITH_HEAD_ROT -static ivas_error renderIsmToSplitBinaural( - input_ism *ismInput, - const IVAS_REND_AudioBuffer outAudio ) +static ivas_error renderMcToBinauralRoom( + input_mc *mcInput, + const AUDIO_CONFIG outConfig, + IVAS_REND_AudioBuffer outAudio ) { + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + AUDIO_CONFIG inConfig; ivas_error error; - float tmpProcessing[MAX_NUM_OBJECTS][L_FRAME48k]; - int16_t pos_idx; - const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; - const SPLIT_REND_WRAPPER *pSplitRendWrapper; - IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES]; + IVAS_REND_AudioBuffer tmpRotBuffer; + float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; int16_t i; - float tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; - int16_t output_frame = ismInput->base.inputBuffer.config.numSamplesPerChannel; - COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; - int16_t ism_md_subframe_update_ext; - - push_wmops( "renderIsmToSplitBinaural" ); - - pSplitRendWrapper = ismInput->base.ctx.pSplitRendWrapper; - pMultiBinPoseData = &pSplitRendWrapper->multiBinPoseData; + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; + int8_t combinedOrientationEnabled; + int16_t subframe_idx; - /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ - ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + p_tmpRendBuffer[i] = tmpRendBuffer[i]; + } - pCombinedOrientationData = *ismInput->base.ctx.pCombinedOrientationData; + push_wmops( "renderMcToBinauralRoom" ); + inConfig = mcInput->base.inConfig; - if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) + hCombinedOrientationData = mcInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = 0; + if ( hCombinedOrientationData != NULL ) { - for ( i = 1; i < pCombinedOrientationData->num_subframes; ++i ) + for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) { - pCombinedOrientationData->Quaternions[i] = pCombinedOrientationData->Quaternions[0]; + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + { + combinedOrientationEnabled = 1; + break; + } } } - /* Save current head positions */ - for ( i = 0; i < pCombinedOrientationData->num_subframes; ++i ) + if ( ( mcInput->hReverb != NULL && outConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && ( ( inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM ) || ( combinedOrientationEnabled && ( inConfig == IVAS_AUDIO_CONFIG_5_1 || inConfig == IVAS_AUDIO_CONFIG_7_1 ) ) ) ) { - originalHeadRot[i] = pCombinedOrientationData->Quaternions[i]; - } - - /* Copy input audio to a processing buffer. */ - copyBufferTo2dArray( ismInput->base.inputBuffer, tmpProcessing ); + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); - for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pCombinedOrientationData, NULL, mcInput->hReverb, + 0, *mcInput->base.ctx.pOutSampleRate, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else { - /* Update head positions */ - if ( pos_idx != 0 ) + /* apply rotation */ + if ( combinedOrientationEnabled ) { - for ( i = 0; i < pCombinedOrientationData->num_subframes; ++i ) + tmpRotBuffer = mcInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); + + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, +#ifdef SPLIT_REND_WITH_HEAD_ROT + mcInput->rot_gains_prev[0], +#else + mcInput->rot_gains_prev, +#endif + mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) { - if ( originalHeadRot[i].w == -3.0f ) - { - pCombinedOrientationData->Quaternions[i].w = -3.0f; - pCombinedOrientationData->Quaternions[i].x = originalHeadRot[i].x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; - pCombinedOrientationData->Quaternions[i].y = originalHeadRot[i].y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; - pCombinedOrientationData->Quaternions[i].z = originalHeadRot[i].z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; - } - else - { - pCombinedOrientationData->Quaternions[i].w = -3.0f; - Quat2EulerDegree( originalHeadRot[i], - &pCombinedOrientationData->Quaternions[i].z, - &pCombinedOrientationData->Quaternions[i].y, - &pCombinedOrientationData->Quaternions[i].x ); - pCombinedOrientationData->Quaternions[i].x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; - pCombinedOrientationData->Quaternions[i].y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; - pCombinedOrientationData->Quaternions[i].z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; - } + return error; } + + copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); + free( tmpRotBuffer.data ); + } + else + { + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); } - /* Render */ - if ( ( error = ivas_td_binaural_renderer_ext( ( pos_idx == 0 ) ? &ismInput->tdRendWrapper : &ismInput->splitTdRendWrappers[pos_idx - 1], ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, - NULL, ism_md_subframe_update_ext, *ismInput->base.ctx.pOutSampleRate, output_frame, tmpProcessing ) ) != IVAS_ERR_OK ) + /* call CREND */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) +#endif { return error; } - - /* Copy rendered audio to tmp storage buffer. Copying directly to output would - * overwrite original audio, which is still needed for rendering next head pose. */ - mvr2r( tmpProcessing[0], tmpBinaural[2 * pos_idx], output_frame ); - mvr2r( tmpProcessing[1], tmpBinaural[2 * pos_idx + 1], output_frame ); - - /* Overwrite processing buffer with original input audio again */ - copyBufferTo2dArray( ismInput->base.inputBuffer, tmpProcessing ); } - /* Restore original head rotation */ - for ( i = 0; i < pCombinedOrientationData->num_subframes; ++i ) + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) +#else + if ( ( error = renderLfeToBinaural( mcInput, outAudio ) ) != IVAS_ERR_OK ) +#endif { - pCombinedOrientationData->Quaternions[i] = originalHeadRot[i]; + return error; } - accumulate2dArrayToBuffer( tmpBinaural, &outAudio ); pop_wmops(); - - /* Encoding to split rendering bitstream done at a higher level */ return IVAS_ERR_OK; } -#endif -static void renderIsmToMasa( - input_ism *ismInput, +static ivas_error renderMcCustomLsToBinauralRoom( + input_mc *mcInput, + const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { - float tmpRendBuffer[MAX_NUM_OBJECTS][L_FRAME48k]; + int16_t i; + int16_t tmp; + float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + ivas_error error; + IVAS_REND_AudioBuffer tmpRotBuffer; + IVAS_REND_AudioBuffer tmpMcBuffer; + IVAS_REND_AudioBuffer *tmpBufPtr; + float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; + int8_t combinedOrientationEnabled; + int16_t subframe_idx; - push_wmops( "renderIsmToMasa" ); + push_wmops( "renderMcCustomLsToBinauralRoom" ); + tmpRotBuffer = outAudio; /* avoid compilation warning */ - copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); - - ivas_omasa_ana( ismInput->hOMasa, tmpRendBuffer, ismInput->base.inputBuffer.config.numSamplesPerChannel, outAudio.config.numChannels, ismInput->base.inputBuffer.config.numChannels ); - - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - - pop_wmops(); + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; + } - return; -} + hCombinedOrientationData = mcInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = 0; + if ( hCombinedOrientationData != NULL ) + { + for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) + { + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + { + combinedOrientationEnabled = 1; + break; + } + } + } + /* apply rotation */ + if ( combinedOrientationEnabled ) + { + tmpRotBuffer = mcInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); -static ivas_error renderInputIsm( - input_ism *ismInput, - const AUDIO_CONFIG outConfig, - const IVAS_REND_AudioBuffer outAudio ) -{ - ivas_error error; - IVAS_REND_AudioBuffer inAudio; + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, +#ifdef SPLIT_REND_WITH_HEAD_ROT + mcInput->rot_gains_prev[0], +#else + mcInput->rot_gains_prev, +#endif + mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + } - error = IVAS_ERR_OK; - inAudio = ismInput->base.inputBuffer; + /* intermediate conversion to 7_1_4 */ + tmpMcBuffer = mcInput->base.inputBuffer; - if ( ismInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) + if ( ( error = getAudioConfigNumChannels( IVAS_AUDIO_CONFIG_7_1_4, &tmp ) ) != IVAS_ERR_OK ) { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); + return error; } - ismInput->base.numNewSamplesPerChannel = 0; - /* Apply input gain to new audio */ - v_multc( inAudio.data, ismInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + tmpMcBuffer.config.numChannels = tmp; + tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels ); - switch ( getAudioConfigType( outConfig ) ) + tmpBufPtr = ( combinedOrientationEnabled ) ? &tmpRotBuffer : &mcInput->base.inputBuffer; + for ( i = 0; i < mcInput->base.inputBuffer.config.numChannels; i++ ) { - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - error = renderIsmToMc( ismInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - error = renderIsmToSba( ismInput, outConfig, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - switch ( outConfig ) - { - case IVAS_AUDIO_CONFIG_BINAURAL: - error = renderIsmToBinaural( ismInput, outAudio ); - break; - case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: - error = renderIsmToBinauralRoom( ismInput, outAudio ); - break; + renderBufferChannel( *tmpBufPtr, i, mcInput->panGains[i], tmpMcBuffer ); + } + copyBufferTo2dArray( tmpMcBuffer, tmpCrendBuffer ); + + /* call CREND */ #ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: - error = renderIsmToSplitBinaural( ismInput, outAudio ); - break; + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, outConfig, NULL, NULL, NULL, NULL, + p_tmpCrendBuffer, *mcInput->base.ctx.pOutSampleRate, getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, outConfig, NULL, NULL, NULL, NULL, + p_tmpCrendBuffer, *mcInput->base.ctx.pOutSampleRate, getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) #endif - case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: - error = renderIsmToBinauralReverb( ismInput, outAudio ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; - } - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - renderIsmToMasa( ismInput, outAudio ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; + { + return error; } - /* Check error here to keep switch statement more compact */ - if ( error != IVAS_ERR_OK ) + accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) +#else + if ( ( error = renderLfeToBinaural( mcInput, outAudio ) ) != IVAS_ERR_OK ) +#endif { return error; } - ismInput->firstFrameRendered = TRUE; + if ( combinedOrientationEnabled ) + { + free( tmpRotBuffer.data ); + } + free( tmpMcBuffer.data ); - return error; + pop_wmops(); + return IVAS_ERR_OK; } -static ivas_error renderActiveInputsIsm( - IVAS_REND_HANDLE hIvasRend, +static void renderMcToMc( + const input_mc *mcInput, IVAS_REND_AudioBuffer outAudio ) { int16_t i; - input_ism *pCurrentInput; - ivas_error error; + IVAS_REND_AudioBuffer inAudio; - for ( i = 0, pCurrentInput = hIvasRend->inputsIsm; i < RENDERER_MAX_ISM_INPUTS; ++i, ++pCurrentInput ) - { - if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - /* Skip inactive inputs */ - continue; - } + push_wmops( "renderMcToMc" ); + inAudio = mcInput->base.inputBuffer; - if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) - { - return error; - } + for ( i = 0; i < inAudio.config.numChannels; ++i ) + { + renderBufferChannel( inAudio, i, mcInput->panGains[i], outAudio ); } - return IVAS_ERR_OK; + pop_wmops(); + return; } -static ivas_error renderLfeToBinaural( +static void renderMcToSba( const input_mc *mcInput, -#ifdef SPLIT_REND_WITH_HEAD_ROT - const AUDIO_CONFIG outConfig, -#endif IVAS_REND_AudioBuffer outAudio ) { - int16_t lfe_idx; -#ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t pose_idx, num_poses; -#endif - float gain; - int16_t ear_idx; - float tmpLfeBuffer[MAX_BUFFER_LENGTH_PER_CHANNEL]; - int16_t frame_size, num_cpy_smpl_cur_frame, num_cpy_smpl_prev_frame; - const float *lfeInput; - float *writePtr; - -#ifdef SPLIT_REND_WITH_HEAD_ROT - assert( ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) && "Must be binaural output" ); -#else - assert( ( outAudio.config.numChannels == 2 ) && "Must be binaural output" ); -#endif - - push_wmops( "renderLfeToBinaural" ); + int16_t i; + IVAS_REND_AudioBuffer inAudio; - gain = GAIN_LFE; + push_wmops( "renderMcToSba" ); + inAudio = mcInput->base.inputBuffer; - if ( mcInput->base.inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM ) - { - lfe_idx = LFE_CHANNEL; - } - else if ( mcInput->customLsInput.num_lfe > 0 ) - { - lfe_idx = mcInput->customLsInput.lfe_idx[0]; - } - else + for ( i = 0; i < inAudio.config.numChannels; ++i ) { - /* no LFE to render */ - return IVAS_ERR_OK; + renderBufferChannel( inAudio, i, mcInput->panGains[i], outAudio ); } - /* --- Prepare LFE signal to be added to binaural output --- */ - lfeInput = getSmplPtr( mcInput->base.inputBuffer, lfe_idx, 0 ); - frame_size = mcInput->base.inputBuffer.config.numSamplesPerChannel; - num_cpy_smpl_prev_frame = mcInput->binauralDelaySmp; - num_cpy_smpl_cur_frame = frame_size - num_cpy_smpl_prev_frame; - - /* Assuming LFE should be delayed by less that the duration of one frame */ - assert( mcInput->binauralDelaySmp < frame_size ); + pop_wmops(); + return; +} - /* Get delayed LFE signal from previous frame, apply gain and save in tmp buffer */ - v_multc( mcInput->lfeDelayBuffer, gain, tmpLfeBuffer, num_cpy_smpl_prev_frame ); - /* Continue filling tmp buffer, now with LFE signal from current frame */ - v_multc( lfeInput, gain, tmpLfeBuffer + num_cpy_smpl_prev_frame, num_cpy_smpl_cur_frame ); +static void renderMcToMasa( + input_mc *mcInput, + IVAS_REND_AudioBuffer outAudio ) +{ + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - /* Save remaining LFE samples of current frame for next frame */ - mvr2r( lfeInput + num_cpy_smpl_cur_frame, mcInput->lfeDelayBuffer, num_cpy_smpl_prev_frame ); + push_wmops( "renderMcToMasa" ); + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); -#ifdef SPLIT_REND_WITH_HEAD_ROT - /* Copy LFE to left and right binaural channels for all poses */ - if ( mcInput->base.ctx.pSplitRendWrapper != NULL ) - { - num_poses = mcInput->base.ctx.pSplitRendWrapper->multiBinPoseData.num_poses; - } - else - { - num_poses = 1; - } + ivas_mcmasa_ana( mcInput->hMcMasa, tmpRendBuffer, mcInput->base.inputBuffer.config.numSamplesPerChannel, outAudio.config.numChannels, mcInput->base.inputBuffer.config.numChannels ); - for ( pose_idx = 0; pose_idx < num_poses; ++pose_idx ) - { - for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) - { - writePtr = getSmplPtr( outAudio, pose_idx * BINAURAL_CHANNELS + ear_idx, 0 ); - v_add( writePtr, tmpLfeBuffer, writePtr, frame_size ); - } - } -#else /* SPLIT_REND_WITH_HEAD_ROT */ - /* Copy LFE to left and right ears */ - for ( ear_idx = 0; ear_idx < BINAURAL_CHANNELS; ++ear_idx ) - { - writePtr = getSmplPtr( outAudio, ear_idx, 0 ); - v_add( writePtr, tmpLfeBuffer, writePtr, frame_size ); - } -#endif /* SPLIT_REND_WITH_HEAD_ROT */ + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); pop_wmops(); - - return IVAS_ERR_OK; + return; } -static ivas_error renderMcToBinaural( +#ifdef SPLIT_REND_WITH_HEAD_ROT +static ivas_error renderMcToSplitBinaural( input_mc *mcInput, const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { - float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - AUDIO_CONFIG inConfig; - ivas_error error; - IVAS_REND_AudioBuffer tmpRotBuffer; - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; - int8_t combinedOrientationEnabled; - int16_t subframe_idx; + int16_t i, j, pos_idx; + int16_t sf; + int16_t output_frame; + ivas_error error; + const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + const SPLIT_REND_WRAPPER *pSplitRendWrapper; + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; - int16_t i; + float tmpSplitBinauralBuffer[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; + AUDIO_CONFIG inConfig; + IVAS_REND_AudioBuffer tmpRotBuffer; + COMBINED_ORIENTATION_DATA combinedOrientationDataLocal; + COMBINED_ORIENTATION_HANDLE pCombinedOrientationDataLocal; - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + push_wmops( "renderMcToSplitBinaural" ); + inConfig = mcInput->base.inConfig; + output_frame = mcInput->base.inputBuffer.config.numSamplesPerChannel; + + pSplitRendWrapper = mcInput->base.ctx.pSplitRendWrapper; + pMultiBinPoseData = &pSplitRendWrapper->multiBinPoseData; + + for ( i = 0; i < MAX_OUTPUT_CHANNELS; ++i ) { p_tmpRendBuffer[i] = tmpRendBuffer[i]; } - push_wmops( "renderMcToBinaural" ); - inConfig = mcInput->base.inConfig; - - hCombinedOrientationData = mcInput->base.ctx.pCombinedOrientationData; - combinedOrientationEnabled = 0; - if ( hCombinedOrientationData != NULL ) + /* save current head positions */ + pCombinedOrientationDataLocal = *mcInput->base.ctx.pCombinedOrientationData; + combinedOrientationDataLocal = *pCombinedOrientationDataLocal; + if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { - for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) + for ( sf = 1; sf < combinedOrientationDataLocal.num_subframes; ++sf ) { - if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + combinedOrientationDataLocal.Quaternions[sf] = combinedOrientationDataLocal.Quaternions[0]; + for ( i = 0; i < 3; i++ ) { - combinedOrientationEnabled = 1; - break; + for ( j = 0; j < 3; j++ ) + { + combinedOrientationDataLocal.Rmat[sf][i][j] = combinedOrientationDataLocal.Rmat[0][i][j]; + } } } } - if ( ( inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM ) || ( combinedOrientationEnabled && ( inConfig == IVAS_AUDIO_CONFIG_5_1 || inConfig == IVAS_AUDIO_CONFIG_7_1 ) ) ) + /* temporary buffer for rotation in source format for CREND */ + tmpRotBuffer = mcInput->base.inputBuffer; + if ( inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM && inConfig != IVAS_AUDIO_CONFIG_5_1 && inConfig != IVAS_AUDIO_CONFIG_7_1 ) { - copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + } - if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pCombinedOrientationData, NULL, mcInput->hReverb, - 0, *mcInput->base.ctx.pOutSampleRate, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + { + /* Update head positions */ + IVAS_QUATERNION Quaternions_orig[MAX_PARAM_SPATIAL_SUBFRAMES], Quaternions_abs; + for ( i = 0; i < combinedOrientationDataLocal.num_subframes; i++ ) { - return error; + Quaternions_orig[i] = combinedOrientationDataLocal.Quaternions[i]; + Quaternions_abs.w = -3.0f; + Quat2EulerDegree( combinedOrientationDataLocal.Quaternions[i], &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + Quaternions_abs.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternions_abs.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternions_abs.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; + combinedOrientationDataLocal.Quaternions[i] = Quaternions_abs; + QuatToRotMat( combinedOrientationDataLocal.Quaternions[i], combinedOrientationDataLocal.Rmat[i] ); } - } - else - { - /* apply rotation */ - if ( combinedOrientationEnabled ) + + if ( inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM || inConfig == IVAS_AUDIO_CONFIG_5_1 || inConfig == IVAS_AUDIO_CONFIG_7_1 ) { - tmpRotBuffer = mcInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); + /* tdrend processing overview: + * 1. copy from inputBuffer to tmpRendBuffer + * 2. td_binaural_renderer_ext: inplace processing in tmpRendBuffer + * 3. copy from tmpRendBuffer to tmpSplitBinBuffer + * 4. LFE mixing + * 5. tmpSplitBinBuffer accumulated to outBuffer */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, mcInput->rot_gains_prev[0], mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) -#else - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, mcInput->rot_gains_prev, mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) -#endif + /* copy input to tdrend input/output buffer */ + copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + + /* perform rotation in source format to tmpRotBuffer */ + pCombinedOrientationDataLocal = &combinedOrientationDataLocal; + + /* Render */ + if ( ( error = ivas_td_binaural_renderer_ext( ( pos_idx == 0 ) ? &mcInput->tdRendWrapper : &mcInput->splitTdRendWrappers[pos_idx - 1], mcInput->base.inConfig, &mcInput->customLsInput, &pCombinedOrientationDataLocal, NULL, mcInput->hReverb, 0, /* Ism Audio Metadata Delay Sync in ms for External Renderer */ *mcInput->base.ctx.pOutSampleRate, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { return error; } - copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); - free( tmpRotBuffer.data ); + /* Copy rendered audio to tmp storage buffer. Copying directly to output would + * overwrite original audio, which is still needed for rendering next head pose. */ + mvr2r( tmpRendBuffer[0], tmpSplitBinauralBuffer[2 * pos_idx], output_frame ); + mvr2r( tmpRendBuffer[1], tmpSplitBinauralBuffer[2 * pos_idx + 1], output_frame ); } else { - copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + /* crend processing overview: + * 1. rotateFrameMc: inputBuffer to tmpRotBuffer + * 2. crend_process: tmpRotBuffer to tmpRendBuffer + * 3. copy from tmpRendBuffer to tmpSplitBinBuffer + * 4. LFE mixing + * 5. tmpSplitBinBuffer accumulated to outBuffer */ + + + /* copy input for in-place rotation */ + mvr2r( mcInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); + + /* perform rotation in source format to tmpRotBuffer */ + pCombinedOrientationDataLocal = &combinedOrientationDataLocal; + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, &pCombinedOrientationDataLocal, mcInput->rot_gains_prev[pos_idx], mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + + copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); + + /* call CREND (rotation already performed) */ + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, getNumSubframesInBuffer( &mcInput->base.inputBuffer, *mcInput->base.ctx.pOutSampleRate ), pos_idx ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Copy rendererd audio to tmp storage buffer, Copying directly to output would + * overwrite original audio, which is still needed for rendering next head pose. */ + mvr2r( tmpRendBuffer[0], tmpSplitBinauralBuffer[2 * pos_idx], output_frame ); + mvr2r( tmpRendBuffer[1], tmpSplitBinauralBuffer[2 * pos_idx + 1], output_frame ); } - /* call CREND */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) -#endif + /* restore original headrotation data */ + for ( i = 0; i < combinedOrientationDataLocal.num_subframes; i++ ) { - return error; + combinedOrientationDataLocal.Quaternions[i] = Quaternions_orig[i]; } } - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); + if ( inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM && inConfig != IVAS_AUDIO_CONFIG_5_1 && inConfig != IVAS_AUDIO_CONFIG_7_1 ) + { + /* free temporary buffer for rotation in source format for CREND */ + free( tmpRotBuffer.data ); + } -#ifdef SPLIT_REND_WITH_HEAD_ROT if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) -#else - if ( ( error = renderLfeToBinaural( mcInput, outAudio ) ) != IVAS_ERR_OK ) -#endif - { return error; } + accumulate2dArrayToBuffer( tmpSplitBinauralBuffer, &outAudio ); + pop_wmops(); return IVAS_ERR_OK; } +#endif -static ivas_error renderMcToBinauralRoom( +static ivas_error renderInputMc( input_mc *mcInput, const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { - float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - AUDIO_CONFIG inConfig; ivas_error error; - IVAS_REND_AudioBuffer tmpRotBuffer; - float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; - int16_t i; - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; - int8_t combinedOrientationEnabled; - int16_t subframe_idx; + IVAS_REND_AudioBuffer inAudio; - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + error = IVAS_ERR_OK; + + inAudio = mcInput->base.inputBuffer; + + if ( mcInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) { - p_tmpRendBuffer[i] = tmpRendBuffer[i]; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); } + mcInput->base.numNewSamplesPerChannel = 0; - push_wmops( "renderMcToBinauralRoom" ); - inConfig = mcInput->base.inConfig; + /* Apply input gain to new audio */ + v_multc( inAudio.data, mcInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); - hCombinedOrientationData = mcInput->base.ctx.pCombinedOrientationData; - combinedOrientationEnabled = 0; - if ( hCombinedOrientationData != NULL ) + switch ( getAudioConfigType( outConfig ) ) { - for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) - { - if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + renderMcToMc( mcInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + renderMcToSba( mcInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + switch ( outConfig ) { - combinedOrientationEnabled = 1; - break; + case IVAS_AUDIO_CONFIG_BINAURAL: + error = renderMcToBinaural( mcInput, outConfig, outAudio ); + break; + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: + if ( mcInput->base.inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM ) + { + error = renderMcCustomLsToBinauralRoom( mcInput, outConfig, outAudio ); + } + else + { + error = renderMcToBinauralRoom( mcInput, outConfig, outAudio ); + } + break; +#ifdef SPLIT_REND_WITH_HEAD_ROT + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + error = renderMcToSplitBinaural( mcInput, outConfig, outAudio ); + break; +#endif + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; } - } + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + renderMcToMasa( mcInput, outAudio ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; } - if ( ( mcInput->hReverb != NULL && outConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && ( ( inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM ) || ( combinedOrientationEnabled && ( inConfig == IVAS_AUDIO_CONFIG_5_1 || inConfig == IVAS_AUDIO_CONFIG_7_1 ) ) ) ) + return error; +} + + +static ivas_error renderActiveInputsMc( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t i; + input_mc *pCurrentInput; + ivas_error error; + + for ( i = 0, pCurrentInput = hIvasRend->inputsMc; i < RENDERER_MAX_MC_INPUTS; ++i, ++pCurrentInput ) { - copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + /* Skip inactive inputs */ + continue; + } - if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pCombinedOrientationData, NULL, mcInput->hReverb, - 0, *mcInput->base.ctx.pOutSampleRate, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = renderInputMc( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) { return error; } } - else - { - /* apply rotation */ - if ( combinedOrientationEnabled ) - { - tmpRotBuffer = mcInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, -#ifdef SPLIT_REND_WITH_HEAD_ROT - mcInput->rot_gains_prev[0], -#else - mcInput->rot_gains_prev, -#endif - mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERR_OK; +} - copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); - free( tmpRotBuffer.data ); - } - else - { - copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); - } - /* call CREND */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } - } +static void renderSbaToMc( + const input_sba *sbaInput, + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t i; + IVAS_REND_AudioBuffer inAudio; - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); + push_wmops( "renderSbaToMc" ); + inAudio = sbaInput->base.inputBuffer; -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) -#else - if ( ( error = renderLfeToBinaural( mcInput, outAudio ) ) != IVAS_ERR_OK ) -#endif + for ( i = 0; i < inAudio.config.numChannels; ++i ) { - return error; + renderBufferChannel( inAudio, i, sbaInput->hoaDecMtx[i], outAudio ); } pop_wmops(); - return IVAS_ERR_OK; + return; } -static ivas_error renderMcCustomLsToBinauralRoom( - input_mc *mcInput, - const AUDIO_CONFIG outConfig, +static void renderSbaToSba( + const input_sba *sbaInput, IVAS_REND_AudioBuffer outAudio ) { int16_t i; - int16_t tmp; - float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - ivas_error error; - IVAS_REND_AudioBuffer tmpRotBuffer; - IVAS_REND_AudioBuffer tmpMcBuffer; - IVAS_REND_AudioBuffer *tmpBufPtr; - float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; - int8_t combinedOrientationEnabled; - int16_t subframe_idx; - - push_wmops( "renderMcCustomLsToBinauralRoom" ); - tmpRotBuffer = outAudio; /* avoid compilation warning */ + IVAS_REND_AudioBuffer inAudio; - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) - { - p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; - } + push_wmops( "renderSbaToSba" ); + inAudio = sbaInput->base.inputBuffer; - hCombinedOrientationData = mcInput->base.ctx.pCombinedOrientationData; - combinedOrientationEnabled = 0; - if ( hCombinedOrientationData != NULL ) + for ( i = 0; i < inAudio.config.numChannels; ++i ) { - for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) - { - if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) - { - combinedOrientationEnabled = 1; - break; - } - } + renderBufferChannel( inAudio, i, sbaInput->hoaDecMtx[i], outAudio ); } - /* apply rotation */ - if ( combinedOrientationEnabled ) - { - tmpRotBuffer = mcInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); + pop_wmops(); + return; +} - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, #ifdef SPLIT_REND_WITH_HEAD_ROT - mcInput->rot_gains_prev[0], -#else - mcInput->rot_gains_prev, -#endif - mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) - { - return error; - } - } - - /* intermediate conversion to 7_1_4 */ - tmpMcBuffer = mcInput->base.inputBuffer; - - if ( ( error = getAudioConfigNumChannels( IVAS_AUDIO_CONFIG_7_1_4, &tmp ) ) != IVAS_ERR_OK ) - { - return error; - } +static ivas_error splitBinLc3plusDecode( + SPLIT_POST_REND_WRAPPER *hSplitBin, + IVAS_SPLIT_REND_BITS_HANDLE bits, + float outputBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k], + IVAS_SPLIT_REND_POSE_CORRECTION_MODE pose_correction ) +{ + ivas_error error; + float *channel_ptrs[MAX_HEAD_ROT_POSES * 2]; + int32_t lc3plusBitrateId, lc3plusBitstreamSize; - tmpMcBuffer.config.numChannels = tmp; - tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels ); + push_wmops( "splitBinLc3plusDecode" ); + assert( hSplitBin->hLc3plusDec != NULL ); - tmpBufPtr = ( combinedOrientationEnabled ) ? &tmpRotBuffer : &mcInput->base.inputBuffer; - for ( i = 0; i < mcInput->base.inputBuffer.config.numChannels; i++ ) + /* Find next byte boundary */ + while ( bits->bits_read % 8 != 0 ) { - renderBufferChannel( *tmpBufPtr, i, mcInput->panGains[i], tmpMcBuffer ); + ++bits->bits_read; } - copyBufferTo2dArray( tmpMcBuffer, tmpCrendBuffer ); + /* Read LC3plus bitstream size info */ + lc3plusBitrateId = ivas_split_rend_bitstream_read_int32( bits, 8 ); + lc3plusBitstreamSize = ivas_get_lc3plus_size_from_id( (int8_t) lc3plusBitrateId, pose_correction, (int16_t) ( hSplitBin->hLc3plusDec->config.ivas_frame_duration_us / 1000 ) ); - /* call CREND */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, outConfig, NULL, NULL, NULL, NULL, - p_tmpCrendBuffer, *mcInput->base.ctx.pOutSampleRate, getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, outConfig, NULL, NULL, NULL, NULL, - p_tmpCrendBuffer, *mcInput->base.ctx.pOutSampleRate, getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) -#endif + for ( int16_t i = 0; i < BINAURAL_CHANNELS * hSplitBin->multiBinPoseData.num_poses; ++i ) { - return error; + channel_ptrs[i] = outputBuffer[i]; } - accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); - -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) -#else - if ( ( error = renderLfeToBinaural( mcInput, outAudio ) ) != IVAS_ERR_OK ) -#endif + if ( ( error = IVAS_LC3PLUS_DEC_Decode( hSplitBin->hLc3plusDec, &bits->bits_buf[bits->bits_read / 8], lc3plusBitstreamSize, channel_ptrs ) ) != IVAS_ERR_OK ) { return error; } - if ( combinedOrientationEnabled ) - { - free( tmpRotBuffer.data ); - } - free( tmpMcBuffer.data ); - pop_wmops(); return IVAS_ERR_OK; } -static void renderMcToMc( - const input_mc *mcInput, - IVAS_REND_AudioBuffer outAudio ) +static ivas_error renderSplitBinauralWithPostRot( + input_split_post_rend *splitBinInput, + IVAS_REND_AudioBuffer outAudio, + const int16_t SplitRendBFI ) { - int16_t i; - IVAS_REND_AudioBuffer inAudio; + float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + ivas_error error; + float Cldfb_RealBuffer_Binaural_5ms[MAX_PARAM_SPATIAL_SUBFRAMES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural_5ms[MAX_PARAM_SPATIAL_SUBFRAMES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES]; + int16_t sf_idx, ch_idx; + IVAS_SPLIT_REND_BITS_DATA bits; + float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + float tmpCrendBuffer_sf[BINAURAL_CHANNELS][L_FRAME48k]; + COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; + SPLIT_POST_REND_WRAPPER *hSplitBin; + int8_t isPostRendInputCldfb; + int16_t chnlIdx, slotIdx, smplIdx; + int16_t preRendFrameSize_ms; + int16_t outBufNumSamplesPerChannel, outBufNumColPerChannel; + int16_t numSamplesPerChannelCacheSize, numColPerChannelCacheSize; + float *readPtr, *writePtr; - push_wmops( "renderMcToMc" ); - inAudio = mcInput->base.inputBuffer; + isPostRendInputCldfb = 0; + push_wmops( "renderSplitBinauralWithPostRot" ); + error = IVAS_ERR_OK; - for ( i = 0; i < inAudio.config.numChannels; ++i ) + pCombinedOrientationData = *splitBinInput->base.ctx.pCombinedOrientationData; + hSplitBin = &splitBinInput->splitPostRendWrapper; + convertBitsBufferToInternalBitsBuff( *splitBinInput->hBits, &bits ); + + if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD && splitBinInput->splitPostRendWrapper.hSplitBinLCLDDec == NULL ) { - renderBufferChannel( inAudio, i, mcInput->panGains[i], outAudio ); + if ( ( error = ivas_splitBinLCLDDecOpen( &splitBinInput->splitPostRendWrapper.hSplitBinLCLDDec, *splitBinInput->base.ctx.pOutSampleRate, BINAURAL_CHANNELS ) ) != IVAS_ERR_OK ) + { + return error; + } } + else if ( bits.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && splitBinInput->splitPostRendWrapper.hLc3plusDec == NULL ) + { + LC3PLUS_CONFIG config; - pop_wmops(); - return; -} + if ( outAudio.config.numSamplesPerChannel == 240 ) + { + config.lc3plus_frame_duration_us = bits.codec_frame_size_ms * 1000; + config.ivas_frame_duration_us = ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) ? config.lc3plus_frame_duration_us : 20000; + } + else + { + config.lc3plus_frame_duration_us = 5000; + config.ivas_frame_duration_us = 20000; + } + config.channels = BINAURAL_CHANNELS; + config.samplerate = *splitBinInput->base.ctx.pOutSampleRate; + if ( ( error = IVAS_LC3PLUS_DEC_Open( config, + &splitBinInput->splitPostRendWrapper.hLc3plusDec ) ) != IVAS_ERR_OK ) + { + return error; + } + } -static void renderMcToSba( - const input_mc *mcInput, - IVAS_REND_AudioBuffer outAudio ) -{ - int16_t i; - IVAS_REND_AudioBuffer inAudio; + outBufNumSamplesPerChannel = outAudio.config.numSamplesPerChannel / pCombinedOrientationData->num_subframes; + for ( sf_idx = 0; sf_idx < pCombinedOrientationData->num_subframes; sf_idx++ ) + { + QuaternionsPost[sf_idx] = pCombinedOrientationData->Quaternions[sf_idx]; + } - push_wmops( "renderMcToSba" ); - inAudio = mcInput->base.inputBuffer; - for ( i = 0; i < inAudio.config.numChannels; ++i ) + if ( !SplitRendBFI ) { - renderBufferChannel( inAudio, i, mcInput->panGains[i], outAudio ); + hSplitBin->first_good_frame_received = 1; } - pop_wmops(); - return; -} + if ( hSplitBin->first_good_frame_received == 1 ) + { + if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) + { + if ( !SplitRendBFI ) + { +#ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG + ivas_splitBinPostRendMdDec( &bits, hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData, hSplitBin->hBinHrSplitPreRend ); +#else + ivas_splitBinPostRendMdDec( &bits, hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData ); +#endif + } + } + /*copy pose correction after MD is parsed*/ + hSplitBin->multiBinPoseData.poseCorrectionMode = bits.pose_correction; -static void renderMcToMasa( - input_mc *mcInput, - IVAS_REND_AudioBuffer outAudio ) -{ - float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + /* decode audio */ + if ( splitBinInput->base.inConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) + { + if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) + { + isPostRendInputCldfb = 1; + } - push_wmops( "renderMcToMasa" ); - copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + preRendFrameSize_ms = bits.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS ? (int16_t) ( hSplitBin->hLc3plusDec->config.ivas_frame_duration_us ) / 1000 : 20; - ivas_mcmasa_ana( mcInput->hMcMasa, tmpRendBuffer, mcInput->base.inputBuffer.config.numSamplesPerChannel, outAudio.config.numChannels, mcInput->base.inputBuffer.config.numChannels ); + numSamplesPerChannelCacheSize = (int16_t) ( *splitBinInput->base.ctx.pOutSampleRate * ( preRendFrameSize_ms - bits.codec_frame_size_ms ) / 1000 ); - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); + outBufNumColPerChannel = MAX_PARAM_SPATIAL_SUBFRAMES; + numColPerChannelCacheSize = CLDFB_NO_COL_MAX - outBufNumColPerChannel; + + for ( sf_idx = 0; sf_idx < pCombinedOrientationData->num_subframes; sf_idx++ ) + { + if ( splitBinInput->numCachedSamples == 0 ) + { + if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) + { + ivas_splitBinLCLDDecProcess( hSplitBin->hSplitBinLCLDDec, &bits, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, SplitRendBFI ); + + /* copy data over to 5ms buffer */ + for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) + { + for ( slotIdx = 0; slotIdx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; ++slotIdx ) + { + mvr2r( Cldfb_RealBuffer_Binaural[chnlIdx][slotIdx], Cldfb_RealBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_ImagBuffer_Binaural[chnlIdx][slotIdx], Cldfb_ImagBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx], CLDFB_NO_CHANNELS_MAX ); + } + } + + /* cache the remaining 15ms */ + splitBinInput->numCachedSamples = numColPerChannelCacheSize; + writePtr = splitBinInput->bufferData; + for ( slotIdx = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slotIdx < CLDFB_NO_COL_MAX; ++slotIdx ) + { + for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) + { + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + *writePtr++ = Cldfb_RealBuffer_Binaural[chnlIdx][slotIdx][smplIdx]; + } + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + *writePtr++ = Cldfb_ImagBuffer_Binaural[chnlIdx][slotIdx][smplIdx]; + } + } + } + } + else + { + if ( ( error = splitBinLc3plusDecode( hSplitBin, &bits, tmpCrendBuffer, bits.pose_correction ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* cache the remaining 15ms */ + splitBinInput->numCachedSamples = numSamplesPerChannelCacheSize; + mvr2r( &tmpCrendBuffer[0][outBufNumSamplesPerChannel], splitBinInput->bufferData, numSamplesPerChannelCacheSize ); + mvr2r( &tmpCrendBuffer[1][outBufNumSamplesPerChannel], splitBinInput->bufferData + numSamplesPerChannelCacheSize, numSamplesPerChannelCacheSize ); + } + } + else + { + /* copy from cache */ + if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) + { + int16_t readOffset = ( numColPerChannelCacheSize - splitBinInput->numCachedSamples ); + readPtr = splitBinInput->bufferData; + isPostRendInputCldfb = 1; + + readPtr += 2 * readOffset * CLDFB_NO_CHANNELS_MAX * BINAURAL_CHANNELS; + for ( slotIdx = 0; slotIdx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; ++slotIdx ) + { + for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) + { + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + Cldfb_RealBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx][smplIdx] = *readPtr++; + } + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + Cldfb_ImagBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx][smplIdx] = *readPtr++; + } + } + } + + splitBinInput->numCachedSamples -= outBufNumColPerChannel; + } + else + { + int16_t readOffset = numSamplesPerChannelCacheSize - splitBinInput->numCachedSamples; + mvr2r( splitBinInput->bufferData + readOffset, &tmpCrendBuffer[0][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); + mvr2r( splitBinInput->bufferData + readOffset + numSamplesPerChannelCacheSize, &tmpCrendBuffer[1][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); + splitBinInput->numCachedSamples -= outBufNumSamplesPerChannel; + } + } + } + } + else + { + copyBufferTo2dArray( splitBinInput->base.inputBuffer, tmpCrendBuffer ); + } + + /* apply pose correction if enabled */ + for ( sf_idx = 0; sf_idx < pCombinedOrientationData->num_subframes; sf_idx++ ) + { + if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE && isPostRendInputCldfb ) + { + /* 0DOF with LCLD codec requires CLDFB synthesis */ + int16_t slot_idx; + + for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) + { + float *RealBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + + for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) + { + RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural_5ms[sf_idx][ch_idx][slot_idx]; + ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural_5ms[sf_idx][ch_idx][slot_idx]; + } + + cldfbSynthesis( RealBuffer, + ImagBuffer, + &( tmpCrendBuffer[ch_idx][sf_idx * outBufNumSamplesPerChannel] ), + hSplitBin->hBinHrSplitPostRend->cldfbSyn[0]->no_channels * CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, + hSplitBin->hBinHrSplitPostRend->cldfbSyn[ch_idx] ); + } + } + else if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) + { + mvr2r( &tmpCrendBuffer[0][sf_idx * outBufNumSamplesPerChannel], tmpCrendBuffer_sf[0], outBufNumSamplesPerChannel ); + mvr2r( &tmpCrendBuffer[1][sf_idx * outBufNumSamplesPerChannel], tmpCrendBuffer_sf[1], outBufNumSamplesPerChannel ); + + ivas_rend_CldfbSplitPostRendProcess( hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData, QuaternionsPost[sf_idx], Cldfb_RealBuffer_Binaural_5ms[sf_idx], Cldfb_ImagBuffer_Binaural_5ms[sf_idx], tmpCrendBuffer_sf, isPostRendInputCldfb ); + + mvr2r( tmpCrendBuffer_sf[0], &tmpCrendBuffer[0][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); + mvr2r( tmpCrendBuffer_sf[1], &tmpCrendBuffer[1][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); + } + } + } + else + { + if ( splitBinInput->base.inConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) + { + for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) + { + set_zero( tmpCrendBuffer[ch_idx], outAudio.config.numSamplesPerChannel ); + } + } + else + { + copyBufferTo2dArray( splitBinInput->base.inputBuffer, tmpCrendBuffer ); + } + } + + convertInternalBitsBuffToBitsBuffer( splitBinInput->hBits, bits ); + accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); pop_wmops(); - return; + return error; } -#ifdef SPLIT_REND_WITH_HEAD_ROT -static ivas_error renderMcToSplitBinaural( - input_mc *mcInput, +static ivas_error renderSbaToMultiBinaural( + input_sba *sbaInput, const AUDIO_CONFIG outConfig, - IVAS_REND_AudioBuffer outAudio ) + float out[][L_FRAME48k] ) { - int16_t i, j, pos_idx; + float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; int16_t sf; - int16_t output_frame; - ivas_error error; - const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; - const SPLIT_REND_WRAPPER *pSplitRendWrapper; - float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; - float tmpSplitBinauralBuffer[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; - AUDIO_CONFIG inConfig; - IVAS_REND_AudioBuffer tmpRotBuffer; + int16_t i, j, pos_idx; COMBINED_ORIENTATION_DATA combinedOrientationDataLocal; COMBINED_ORIENTATION_HANDLE pCombinedOrientationDataLocal; + ivas_error error; + IVAS_REND_AudioBuffer tmpRotBuffer; + const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; - push_wmops( "renderMcToSplitBinaural" ); - inConfig = mcInput->base.inConfig; - output_frame = mcInput->base.inputBuffer.config.numSamplesPerChannel; - - pSplitRendWrapper = mcInput->base.ctx.pSplitRendWrapper; - pMultiBinPoseData = &pSplitRendWrapper->multiBinPoseData; - - for ( i = 0; i < MAX_OUTPUT_CHANNELS; ++i ) + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) { - p_tmpRendBuffer[i] = tmpRendBuffer[i]; + p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; } + push_wmops( "renderSbaToMultiBinaural" ); + pMultiBinPoseData = &sbaInput->base.ctx.pSplitRendWrapper->multiBinPoseData; - /* save current head positions */ - pCombinedOrientationDataLocal = *mcInput->base.ctx.pCombinedOrientationData; + pCombinedOrientationDataLocal = *sbaInput->base.ctx.pCombinedOrientationData; combinedOrientationDataLocal = *pCombinedOrientationDataLocal; if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { - for ( sf = 1; sf < combinedOrientationDataLocal.num_subframes; ++sf ) + for ( sf = 1; sf < combinedOrientationDataLocal.num_subframes; sf++ ) { combinedOrientationDataLocal.Quaternions[sf] = combinedOrientationDataLocal.Quaternions[0]; for ( i = 0; i < 3; i++ ) @@ -6965,16 +7057,11 @@ static ivas_error renderMcToSplitBinaural( } } - /* temporary buffer for rotation in source format for CREND */ - tmpRotBuffer = mcInput->base.inputBuffer; - if ( inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM && inConfig != IVAS_AUDIO_CONFIG_5_1 && inConfig != IVAS_AUDIO_CONFIG_7_1 ) - { - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - } + tmpRotBuffer = sbaInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) { - /* Update head positions */ IVAS_QUATERNION Quaternions_orig[MAX_PARAM_SPATIAL_SUBFRAMES], Quaternions_abs; for ( i = 0; i < combinedOrientationDataLocal.num_subframes; i++ ) { @@ -6989,2138 +7076,2202 @@ static ivas_error renderMcToSplitBinaural( QuatToRotMat( combinedOrientationDataLocal.Quaternions[i], combinedOrientationDataLocal.Rmat[i] ); } - if ( inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM || inConfig == IVAS_AUDIO_CONFIG_5_1 || inConfig == IVAS_AUDIO_CONFIG_7_1 ) - { - /* tdrend processing overview: - * 1. copy from inputBuffer to tmpRendBuffer - * 2. td_binaural_renderer_ext: inplace processing in tmpRendBuffer - * 3. copy from tmpRendBuffer to tmpSplitBinBuffer - * 4. LFE mixing - * 5. tmpSplitBinBuffer accumulated to outBuffer */ - /* copy input to tdrend input/output buffer */ - copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); + /* copy input for in-place rotation */ + mvr2r( sbaInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); - /* perform rotation in source format to tmpRotBuffer */ - pCombinedOrientationDataLocal = &combinedOrientationDataLocal; + pCombinedOrientationDataLocal = &combinedOrientationDataLocal; - /* Render */ - if ( ( error = ivas_td_binaural_renderer_ext( ( pos_idx == 0 ) ? &mcInput->tdRendWrapper : &mcInput->splitTdRendWrappers[pos_idx - 1], mcInput->base.inConfig, &mcInput->customLsInput, &pCombinedOrientationDataLocal, NULL, mcInput->hReverb, 0, /* Ism Audio Metadata Delay Sync in ms for External Renderer */ *mcInput->base.ctx.pOutSampleRate, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) - { - return error; - } - - /* Copy rendered audio to tmp storage buffer. Copying directly to output would - * overwrite original audio, which is still needed for rendering next head pose. */ - mvr2r( tmpRendBuffer[0], tmpSplitBinauralBuffer[2 * pos_idx], output_frame ); - mvr2r( tmpRendBuffer[1], tmpSplitBinauralBuffer[2 * pos_idx + 1], output_frame ); - } - else + if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, &pCombinedOrientationDataLocal, sbaInput->rot_gains_prev[pos_idx], tmpRotBuffer ) ) != IVAS_ERR_OK ) { - /* crend processing overview: - * 1. rotateFrameMc: inputBuffer to tmpRotBuffer - * 2. crend_process: tmpRotBuffer to tmpRendBuffer - * 3. copy from tmpRendBuffer to tmpSplitBinBuffer - * 4. LFE mixing - * 5. tmpSplitBinBuffer accumulated to outBuffer */ - - - /* copy input for in-place rotation */ - mvr2r( mcInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); - - /* perform rotation in source format to tmpRotBuffer */ - pCombinedOrientationDataLocal = &combinedOrientationDataLocal; - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, &pCombinedOrientationDataLocal, mcInput->rot_gains_prev[pos_idx], mcInput->efapInWrapper.hEfap, tmpRotBuffer ) ) != IVAS_ERR_OK ) - { - return error; - } + return error; + } - copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); + copyBufferTo2dArray( tmpRotBuffer, tmpCrendBuffer ); - /* call CREND (rotation already performed) */ - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, getNumSubframesInBuffer( &mcInput->base.inputBuffer, *mcInput->base.ctx.pOutSampleRate ), pos_idx ) ) != IVAS_ERR_OK ) - { - return error; - } + assert( sbaInput->crendWrapper->hCrend[0]->hReverb == NULL ); - /* Copy rendererd audio to tmp storage buffer, Copying directly to output would - * overwrite original audio, which is still needed for rendering next head pose. */ - mvr2r( tmpRendBuffer[0], tmpSplitBinauralBuffer[2 * pos_idx], output_frame ); - mvr2r( tmpRendBuffer[1], tmpSplitBinauralBuffer[2 * pos_idx + 1], output_frame ); + /* call CREND */ + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &sbaInput->base.inputBuffer, *sbaInput->base.ctx.pOutSampleRate ), pos_idx ) ) != IVAS_ERR_OK ) + { + return error; } - /* restore original headrotation data */ for ( i = 0; i < combinedOrientationDataLocal.num_subframes; i++ ) { combinedOrientationDataLocal.Quaternions[i] = Quaternions_orig[i]; } - } - if ( inConfig != IVAS_AUDIO_CONFIG_LS_CUSTOM && inConfig != IVAS_AUDIO_CONFIG_5_1 && inConfig != IVAS_AUDIO_CONFIG_7_1 ) - { - /* free temporary buffer for rotation in source format for CREND */ - free( tmpRotBuffer.data ); - } - if ( ( error = renderLfeToBinaural( mcInput, outConfig, outAudio ) ) != IVAS_ERR_OK ) - { - return error; + /* move to output */ + for ( i = 0; i < BINAURAL_CHANNELS; i++ ) + { + mvr2r( tmpCrendBuffer[i], out[pos_idx * BINAURAL_CHANNELS + i], tmpRotBuffer.config.numSamplesPerChannel ); + } } - accumulate2dArrayToBuffer( tmpSplitBinauralBuffer, &outAudio ); + free( tmpRotBuffer.data ); pop_wmops(); return IVAS_ERR_OK; } -#endif -static ivas_error renderInputMc( - input_mc *mcInput, +static ivas_error renderSbaToMultiBinauralCldfb( + input_sba *sbaInput, + float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + const int16_t low_res_pre_rend_rot, + const int16_t num_subframes ) +{ + float Cldfb_RealBuffer[MAX_OUTPUT_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer[MAX_OUTPUT_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + + copyBufferToCLDFBarray( sbaInput->base.inputBuffer, Cldfb_RealBuffer, Cldfb_ImagBuffer ); + + ivas_rend_CldfbMultiBinRendProcess( sbaInput->cldfbRendWrapper.hCldfbRend, sbaInput->base.ctx.pCombinedOrientationData, &sbaInput->base.ctx.pSplitRendWrapper->multiBinPoseData, + Cldfb_RealBuffer, Cldfb_ImagBuffer, Cldfb_Out_Real, Cldfb_Out_Imag, low_res_pre_rend_rot, num_subframes ); + + return IVAS_ERR_OK; +} + + +static ivas_error renderSbaToSplitBinaural( + input_sba *sbaInput, const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { + float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; ivas_error error; - IVAS_REND_AudioBuffer inAudio; + float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + int16_t low_res_pre_rend_rot; - error = IVAS_ERR_OK; + low_res_pre_rend_rot = 1; - inAudio = mcInput->base.inputBuffer; + push_wmops( "renderSbaToSplitBinaural" ); + error = IVAS_ERR_OK; - if ( mcInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) + if ( sbaInput->base.ctx.hhRendererConfig[0]->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); - } - mcInput->base.numNewSamplesPerChannel = 0; - - /* Apply input gain to new audio */ - v_multc( inAudio.data, mcInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + if ( ( renderSbaToMultiBinauralCldfb( sbaInput, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, low_res_pre_rend_rot, + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) + { + return error; + } - switch ( getAudioConfigType( outConfig ) ) + accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); + } + else { - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - renderMcToMc( mcInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - renderMcToSba( mcInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - switch ( outConfig ) - { - case IVAS_AUDIO_CONFIG_BINAURAL: - error = renderMcToBinaural( mcInput, outConfig, outAudio ); - break; - case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: - case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: - if ( mcInput->base.inConfig == IVAS_AUDIO_CONFIG_LS_CUSTOM ) - { - error = renderMcCustomLsToBinauralRoom( mcInput, outConfig, outAudio ); - } - else - { - error = renderMcToBinauralRoom( mcInput, outConfig, outAudio ); - } - break; -#ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: - error = renderMcToSplitBinaural( mcInput, outConfig, outAudio ); - break; -#endif - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; - } - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - renderMcToMasa( mcInput, outAudio ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; + if ( ( renderSbaToMultiBinaural( sbaInput, outConfig, tmpCrendBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + + accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); } + pop_wmops(); return error; } +#endif -static ivas_error renderActiveInputsMc( - IVAS_REND_HANDLE hIvasRend, +static ivas_error renderSbaToBinaural( + input_sba *sbaInput, + const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { - int16_t i; - input_mc *pCurrentInput; + float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; ivas_error error; + IVAS_REND_AudioBuffer tmpRotBuffer; + float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; + int16_t i; + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; + int8_t combinedOrientationEnabled; + int16_t subframe_idx; - for ( i = 0, pCurrentInput = hIvasRend->inputsMc; i < RENDERER_MAX_MC_INPUTS; ++i, ++pCurrentInput ) + push_wmops( "renderSbaToBinaural" ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( sbaInput->base.ctx.hhRendererConfig[0]->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) { - if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + + if ( ( error = renderSbaToMultiBinauralCldfb( sbaInput, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, 0, + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) { - /* Skip inactive inputs */ - continue; + return error; } - if ( ( error = renderInputMc( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) + accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); + } + else +#endif + { + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; + } + + hCombinedOrientationData = sbaInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = 0; + if ( hCombinedOrientationData != NULL ) + { + for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) + { + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + { + combinedOrientationEnabled = 1; + break; + } + } + } + + /* apply rotation */ + if ( combinedOrientationEnabled ) + { + tmpRotBuffer = sbaInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + + /* copy input for in-place rotation */ + mvr2r( sbaInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, + sbaInput->base.ctx.pCombinedOrientationData, sbaInput->rot_gains_prev[0], tmpRotBuffer ) ) != IVAS_ERR_OK ) +#else + if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, + sbaInput->base.ctx.pCombinedOrientationData, sbaInput->rot_gains_prev, tmpRotBuffer ) ) != IVAS_ERR_OK ) +#endif + { + return error; + } + + copyBufferTo2dArray( tmpRotBuffer, tmpCrendBuffer ); + free( tmpRotBuffer.data ); + } + else + { + copyBufferTo2dArray( sbaInput->base.inputBuffer, tmpCrendBuffer ); + } + + /* call CREND */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) +#endif { return error; } + + accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); } + pop_wmops(); return IVAS_ERR_OK; } -static void renderSbaToMc( - const input_sba *sbaInput, +static ivas_error renderSbaToBinauralRoom( + input_sba *sbaInput, + const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { int16_t i; - IVAS_REND_AudioBuffer inAudio; + int16_t tmp; + float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + ivas_error error; + IVAS_REND_AudioBuffer tmpRotBuffer; + IVAS_REND_AudioBuffer tmpMcBuffer; + IVAS_REND_AudioBuffer *tmpBufPtr; + float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; + const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; + int8_t combinedOrientationEnabled; + int16_t subframe_idx; - push_wmops( "renderSbaToMc" ); - inAudio = sbaInput->base.inputBuffer; + tmpRotBuffer = outAudio; /* avoid compilation warning */ + push_wmops( "renderSbaToBinauralRoom" ); - for ( i = 0; i < inAudio.config.numChannels; ++i ) + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) { - renderBufferChannel( inAudio, i, sbaInput->hoaDecMtx[i], outAudio ); + p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; } - pop_wmops(); - return; -} - - -static void renderSbaToSba( - const input_sba *sbaInput, - IVAS_REND_AudioBuffer outAudio ) -{ - int16_t i; - IVAS_REND_AudioBuffer inAudio; - - push_wmops( "renderSbaToSba" ); - inAudio = sbaInput->base.inputBuffer; - - for ( i = 0; i < inAudio.config.numChannels; ++i ) + hCombinedOrientationData = sbaInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = 0; + if ( hCombinedOrientationData != NULL ) { - renderBufferChannel( inAudio, i, sbaInput->hoaDecMtx[i], outAudio ); + for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) + { + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) + { + combinedOrientationEnabled = 1; + break; + } + } } - pop_wmops(); - return; -} + /* apply rotation */ + if ( combinedOrientationEnabled ) + { + tmpRotBuffer = sbaInput->base.inputBuffer; + tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + /* copy input for in-place rotation */ + mvr2r( sbaInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); + + if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, + sbaInput->base.ctx.pCombinedOrientationData, #ifdef SPLIT_REND_WITH_HEAD_ROT -static ivas_error splitBinLc3plusDecode( - SPLIT_POST_REND_WRAPPER *hSplitBin, - IVAS_SPLIT_REND_BITS_HANDLE bits, - float outputBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k], - IVAS_SPLIT_REND_POSE_CORRECTION_MODE pose_correction ) -{ - ivas_error error; - float *channel_ptrs[MAX_HEAD_ROT_POSES * 2]; - int32_t lc3plusBitrateId, lc3plusBitstreamSize; + sbaInput->rot_gains_prev[0], +#else + sbaInput->rot_gains_prev, +#endif + tmpRotBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + } - push_wmops( "splitBinLc3plusDecode" ); - assert( hSplitBin->hLc3plusDec != NULL ); + /* intermediate rendering to 7_1_4 */ + tmpMcBuffer = sbaInput->base.inputBuffer; - /* Find next byte boundary */ - while ( bits->bits_read % 8 != 0 ) + if ( ( error = getAudioConfigNumChannels( IVAS_AUDIO_CONFIG_7_1_4, &tmp ) ) != IVAS_ERR_OK ) { - ++bits->bits_read; + return error; } - /* Read LC3plus bitstream size info */ - lc3plusBitrateId = ivas_split_rend_bitstream_read_int32( bits, 8 ); - lc3plusBitstreamSize = ivas_get_lc3plus_size_from_id( (int8_t) lc3plusBitrateId, pose_correction, (int16_t) ( hSplitBin->hLc3plusDec->config.ivas_frame_duration_us / 1000 ) ); - for ( int16_t i = 0; i < BINAURAL_CHANNELS * hSplitBin->multiBinPoseData.num_poses; ++i ) + tmpMcBuffer.config.numChannels = tmp; + tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); + set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numChannels * tmpMcBuffer.config.numSamplesPerChannel ); + + tmpBufPtr = ( combinedOrientationEnabled ) ? &tmpRotBuffer : &sbaInput->base.inputBuffer; + for ( i = 0; i < sbaInput->base.inputBuffer.config.numChannels; i++ ) { - channel_ptrs[i] = outputBuffer[i]; + renderBufferChannel( *tmpBufPtr, i, sbaInput->hoaDecMtx[i], tmpMcBuffer ); } - if ( ( error = IVAS_LC3PLUS_DEC_Decode( hSplitBin->hLc3plusDec, &bits->bits_buf[bits->bits_read / 8], lc3plusBitstreamSize, channel_ptrs ) ) != IVAS_ERR_OK ) + copyBufferTo2dArray( tmpMcBuffer, tmpCrendBuffer ); + + /* call CREND */ + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, outConfig, + NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, +#ifdef SPLIT_REND_WITH_HEAD_ROT + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) +#else + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) +#endif { return error; } + accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); + + if ( combinedOrientationEnabled ) + { + free( tmpRotBuffer.data ); + } + free( tmpMcBuffer.data ); + pop_wmops(); return IVAS_ERR_OK; } -static ivas_error renderSplitBinauralWithPostRot( +#ifdef SPLIT_REND_WITH_HEAD_ROT +static ivas_error renderInputSplitBin( input_split_post_rend *splitBinInput, + const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio, const int16_t SplitRendBFI ) { - float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; ivas_error error; - float Cldfb_RealBuffer_Binaural_5ms[MAX_PARAM_SPATIAL_SUBFRAMES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer_Binaural_5ms[MAX_PARAM_SPATIAL_SUBFRAMES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; - IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES]; - int16_t sf_idx, ch_idx; - IVAS_SPLIT_REND_BITS_DATA bits; - float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - float tmpCrendBuffer_sf[BINAURAL_CHANNELS][L_FRAME48k]; - COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; - SPLIT_POST_REND_WRAPPER *hSplitBin; - int8_t isPostRendInputCldfb; - int16_t chnlIdx, slotIdx, smplIdx; - int16_t preRendFrameSize_ms; - int16_t outBufNumSamplesPerChannel, outBufNumColPerChannel; - int16_t numSamplesPerChannelCacheSize, numColPerChannelCacheSize; - float *readPtr, *writePtr; + IVAS_REND_AudioBuffer inAudio; - isPostRendInputCldfb = 0; - push_wmops( "renderSplitBinauralWithPostRot" ); - error = IVAS_ERR_OK; + inAudio = splitBinInput->base.inputBuffer; - pCombinedOrientationData = *splitBinInput->base.ctx.pCombinedOrientationData; - hSplitBin = &splitBinInput->splitPostRendWrapper; - convertBitsBufferToInternalBitsBuff( *splitBinInput->hBits, &bits ); + splitBinInput->base.numNewSamplesPerChannel = 0; - if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD && splitBinInput->splitPostRendWrapper.hSplitBinLCLDDec == NULL ) + /* Apply input gain to new audio */ + v_multc( inAudio.data, + splitBinInput->base.gain, + inAudio.data, + inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); /* TODO: the output buffer is empty at this point, should be moved to a point after decoding the split bitstream */ + switch ( outConfig ) { - if ( ( error = ivas_splitBinLCLDDecOpen( &splitBinInput->splitPostRendWrapper.hSplitBinLCLDDec, *splitBinInput->base.ctx.pOutSampleRate, BINAURAL_CHANNELS ) ) != IVAS_ERR_OK ) - { - return error; - } + case IVAS_AUDIO_CONFIG_BINAURAL: + error = renderSplitBinauralWithPostRot( splitBinInput, outAudio, SplitRendBFI ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; } - else if ( bits.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && splitBinInput->splitPostRendWrapper.hLc3plusDec == NULL ) - { - LC3PLUS_CONFIG config; - if ( outAudio.config.numSamplesPerChannel == 240 ) - { - config.lc3plus_frame_duration_us = bits.codec_frame_size_ms * 1000; - config.ivas_frame_duration_us = ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) ? config.lc3plus_frame_duration_us : 20000; - } - else - { - config.lc3plus_frame_duration_us = 5000; - config.ivas_frame_duration_us = 20000; - } - config.channels = BINAURAL_CHANNELS; - config.samplerate = *splitBinInput->base.ctx.pOutSampleRate; + return error; +} +#endif - if ( ( error = IVAS_LC3PLUS_DEC_Open( config, - &splitBinInput->splitPostRendWrapper.hLc3plusDec ) ) != IVAS_ERR_OK ) - { - return error; - } - } - outBufNumSamplesPerChannel = outAudio.config.numSamplesPerChannel / pCombinedOrientationData->num_subframes; - for ( sf_idx = 0; sf_idx < pCombinedOrientationData->num_subframes; sf_idx++ ) - { - QuaternionsPost[sf_idx] = pCombinedOrientationData->Quaternions[sf_idx]; - } +static void renderSbaToMasa( + input_sba *sbaInput, + IVAS_REND_AudioBuffer outAudio ) +{ + float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + push_wmops( "renderMcToMasa" ); + copyBufferTo2dArray( sbaInput->base.inputBuffer, tmpRendBuffer ); + ivas_dirac_ana( sbaInput->hDirAC, tmpRendBuffer, sbaInput->base.inputBuffer.config.numSamplesPerChannel, outAudio.config.numChannels ); + accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - if ( !SplitRendBFI ) - { - hSplitBin->first_good_frame_received = 1; - } + pop_wmops(); + return; +} - if ( hSplitBin->first_good_frame_received == 1 ) - { - if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) - { - if ( !SplitRendBFI ) - { -#ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG - ivas_splitBinPostRendMdDec( &bits, hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData, hSplitBin->hBinHrSplitPreRend ); + +static ivas_error renderInputSba( + input_sba *sbaInput, + const AUDIO_CONFIG outConfig, + IVAS_REND_AudioBuffer outAudio ) +{ + ivas_error error; + IVAS_REND_AudioBuffer inAudio; +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; +#endif + + error = IVAS_ERR_OK; + inAudio = sbaInput->base.inputBuffer; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + cldfb2tdSampleFact = outAudio.config.is_cldfb ? 2 : 1; + if ( ( sbaInput->base.numNewSamplesPerChannel * cldfb2tdSampleFact != outAudio.config.numSamplesPerChannel ) && + ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) && ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) #else - ivas_splitBinPostRendMdDec( &bits, hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData ); + if ( sbaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) #endif - } - } + { + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); + } + sbaInput->base.numNewSamplesPerChannel = 0; - /*copy pose correction after MD is parsed*/ - hSplitBin->multiBinPoseData.poseCorrectionMode = bits.pose_correction; + /* Apply input gain to new audio */ + v_multc( inAudio.data, sbaInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); - /* decode audio */ - if ( splitBinInput->base.inConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) - { - if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + renderSbaToMc( sbaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + renderSbaToSba( sbaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + switch ( outConfig ) { - isPostRendInputCldfb = 1; +#ifdef SPLIT_REND_WITH_HEAD_ROT + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + error = renderSbaToSplitBinaural( sbaInput, outConfig, outAudio ); + break; +#endif + case IVAS_AUDIO_CONFIG_BINAURAL: + error = renderSbaToBinaural( sbaInput, outConfig, outAudio ); + break; + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: + error = renderSbaToBinauralRoom( sbaInput, outConfig, outAudio ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; } + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + renderSbaToMasa( sbaInput, outAudio ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } - preRendFrameSize_ms = bits.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS ? (int16_t) ( hSplitBin->hLc3plusDec->config.ivas_frame_duration_us ) / 1000 : 20; + return error; +} - numSamplesPerChannelCacheSize = (int16_t) ( *splitBinInput->base.ctx.pOutSampleRate * ( preRendFrameSize_ms - bits.codec_frame_size_ms ) / 1000 ); - outBufNumColPerChannel = MAX_PARAM_SPATIAL_SUBFRAMES; - numColPerChannelCacheSize = CLDFB_NO_COL_MAX - outBufNumColPerChannel; +#ifdef SPLIT_REND_WITH_HEAD_ROT +static ivas_error renderActiveInputsSplitBin( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t i; + input_split_post_rend *pCurrentInput; + ivas_error error; - for ( sf_idx = 0; sf_idx < pCombinedOrientationData->num_subframes; sf_idx++ ) - { - if ( splitBinInput->numCachedSamples == 0 ) - { - if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) - { - ivas_splitBinLCLDDecProcess( hSplitBin->hSplitBinLCLDDec, &bits, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, SplitRendBFI ); + for ( i = 0, pCurrentInput = hIvasRend->inputsSplitPost; i < RENDERER_MAX_BIN_INPUTS; ++i, ++pCurrentInput ) + { + if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + /* Skip inactive inputs */ + continue; + } - /* copy data over to 5ms buffer */ - for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) - { - for ( slotIdx = 0; slotIdx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; ++slotIdx ) - { - mvr2r( Cldfb_RealBuffer_Binaural[chnlIdx][slotIdx], Cldfb_RealBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx], CLDFB_NO_CHANNELS_MAX ); - mvr2r( Cldfb_ImagBuffer_Binaural[chnlIdx][slotIdx], Cldfb_ImagBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx], CLDFB_NO_CHANNELS_MAX ); - } - } + if ( ( error = renderInputSplitBin( pCurrentInput, hIvasRend->outputConfig, outAudio, hIvasRend->splitRendBFI ) ) != IVAS_ERR_OK ) + { + return error; + } + } - /* cache the remaining 15ms */ - splitBinInput->numCachedSamples = numColPerChannelCacheSize; - writePtr = splitBinInput->bufferData; - for ( slotIdx = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slotIdx < CLDFB_NO_COL_MAX; ++slotIdx ) - { - for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) - { - for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) - { - *writePtr++ = Cldfb_RealBuffer_Binaural[chnlIdx][slotIdx][smplIdx]; - } - for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) - { - *writePtr++ = Cldfb_ImagBuffer_Binaural[chnlIdx][slotIdx][smplIdx]; - } - } - } - } - else - { - if ( ( error = splitBinLc3plusDecode( hSplitBin, &bits, tmpCrendBuffer, bits.pose_correction ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERR_OK; +} +#endif - /* cache the remaining 15ms */ - splitBinInput->numCachedSamples = numSamplesPerChannelCacheSize; - mvr2r( &tmpCrendBuffer[0][outBufNumSamplesPerChannel], splitBinInput->bufferData, numSamplesPerChannelCacheSize ); - mvr2r( &tmpCrendBuffer[1][outBufNumSamplesPerChannel], splitBinInput->bufferData + numSamplesPerChannelCacheSize, numSamplesPerChannelCacheSize ); - } - } - else - { - /* copy from cache */ - if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) - { - int16_t readOffset = ( numColPerChannelCacheSize - splitBinInput->numCachedSamples ); - readPtr = splitBinInput->bufferData; - isPostRendInputCldfb = 1; - readPtr += 2 * readOffset * CLDFB_NO_CHANNELS_MAX * BINAURAL_CHANNELS; - for ( slotIdx = 0; slotIdx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; ++slotIdx ) - { - for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) - { - for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) - { - Cldfb_RealBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx][smplIdx] = *readPtr++; - } - for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) - { - Cldfb_ImagBuffer_Binaural_5ms[sf_idx][chnlIdx][slotIdx][smplIdx] = *readPtr++; - } - } - } +static ivas_error renderActiveInputsSba( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t i; + input_sba *pCurrentInput; + ivas_error error; - splitBinInput->numCachedSamples -= outBufNumColPerChannel; - } - else - { - int16_t readOffset = numSamplesPerChannelCacheSize - splitBinInput->numCachedSamples; - mvr2r( splitBinInput->bufferData + readOffset, &tmpCrendBuffer[0][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); - mvr2r( splitBinInput->bufferData + readOffset + numSamplesPerChannelCacheSize, &tmpCrendBuffer[1][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); - splitBinInput->numCachedSamples -= outBufNumSamplesPerChannel; - } - } - } - } - else + for ( i = 0, pCurrentInput = hIvasRend->inputsSba; i < RENDERER_MAX_SBA_INPUTS; ++i, ++pCurrentInput ) + { + if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) { - copyBufferTo2dArray( splitBinInput->base.inputBuffer, tmpCrendBuffer ); + /* Skip inactive inputs */ + continue; } - /* apply pose correction if enabled */ - for ( sf_idx = 0; sf_idx < pCombinedOrientationData->num_subframes; sf_idx++ ) + if ( ( error = renderInputSba( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) { - if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE && isPostRendInputCldfb ) - { - /* 0DOF with LCLD codec requires CLDFB synthesis */ - int16_t slot_idx; + return error; + } + } - for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) - { - float *RealBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; - float *ImagBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + return IVAS_ERR_OK; +} - for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) - { - RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural_5ms[sf_idx][ch_idx][slot_idx]; - ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural_5ms[sf_idx][ch_idx][slot_idx]; - } - cldfbSynthesis( RealBuffer, - ImagBuffer, - &( tmpCrendBuffer[ch_idx][sf_idx * outBufNumSamplesPerChannel] ), - hSplitBin->hBinHrSplitPostRend->cldfbSyn[0]->no_channels * CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, - hSplitBin->hBinHrSplitPostRend->cldfbSyn[ch_idx] ); - } - } - else if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) - { - mvr2r( &tmpCrendBuffer[0][sf_idx * outBufNumSamplesPerChannel], tmpCrendBuffer_sf[0], outBufNumSamplesPerChannel ); - mvr2r( &tmpCrendBuffer[1][sf_idx * outBufNumSamplesPerChannel], tmpCrendBuffer_sf[1], outBufNumSamplesPerChannel ); +static void copyMasaMetadataToDiracRenderer( + MASA_METADATA_FRAME *meta, + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom ) +{ + int16_t band, sf, bin; + int16_t meta_write_index; - ivas_rend_CldfbSplitPostRendProcess( hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData, QuaternionsPost[sf_idx], Cldfb_RealBuffer_Binaural_5ms[sf_idx], Cldfb_ImagBuffer_Binaural_5ms[sf_idx], tmpCrendBuffer_sf, isPostRendInputCldfb ); + hSpatParamRendCom->numParametricDirections = meta->descriptive_meta.numberOfDirections + 1; + hSpatParamRendCom->numSimultaneousDirections = meta->descriptive_meta.numberOfDirections + 1; - mvr2r( tmpCrendBuffer_sf[0], &tmpCrendBuffer[0][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); - mvr2r( tmpCrendBuffer_sf[1], &tmpCrendBuffer[1][sf_idx * outBufNumSamplesPerChannel], outBufNumSamplesPerChannel ); - } - } - } - else + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) { - if ( splitBinInput->base.inConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) + meta_write_index = ( hSpatParamRendCom->dirac_bs_md_write_idx + sf ) % hSpatParamRendCom->dirac_md_buffer_length; + + for ( band = 0; band < MASA_MAXIMUM_CODING_SUBBANDS; band++ ) { - for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) + for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1]; bin++ ) { - set_zero( tmpCrendBuffer[ch_idx], outAudio.config.numSamplesPerChannel ); + hSpatParamRendCom->azimuth[meta_write_index][bin] = (int16_t) meta->directional_meta[0].azimuth[sf][band]; + hSpatParamRendCom->elevation[meta_write_index][bin] = (int16_t) meta->directional_meta[0].elevation[sf][band]; + hSpatParamRendCom->energy_ratio1[meta_write_index][bin] = meta->directional_meta[0].energy_ratio[sf][band]; + hSpatParamRendCom->diffuseness_vector[meta_write_index][bin] = 1.0f - meta->directional_meta[0].energy_ratio[sf][band]; + hSpatParamRendCom->spreadCoherence[meta_write_index][bin] = meta->directional_meta[0].spread_coherence[sf][band]; + hSpatParamRendCom->surroundingCoherence[meta_write_index][bin] = meta->common_meta.surround_coherence[sf][band]; + + if ( hSpatParamRendCom->numSimultaneousDirections == 2 ) + { + hSpatParamRendCom->azimuth2[meta_write_index][bin] = (int16_t) meta->directional_meta[1].azimuth[sf][band]; + hSpatParamRendCom->elevation2[meta_write_index][bin] = (int16_t) meta->directional_meta[1].elevation[sf][band]; + hSpatParamRendCom->energy_ratio2[meta_write_index][bin] = meta->directional_meta[1].energy_ratio[sf][band]; + hSpatParamRendCom->diffuseness_vector[meta_write_index][bin] -= meta->directional_meta[1].energy_ratio[sf][band]; + hSpatParamRendCom->spreadCoherence2[meta_write_index][bin] = meta->directional_meta[1].spread_coherence[sf][band]; + } } } - else - { - copyBufferTo2dArray( splitBinInput->base.inputBuffer, tmpCrendBuffer ); - } } - convertInternalBitsBuffToBitsBuffer( splitBinInput->hBits, bits ); - accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); + hSpatParamRendCom->dirac_bs_md_write_idx = ( hSpatParamRendCom->dirac_bs_md_write_idx + MAX_PARAM_SPATIAL_SUBFRAMES ) % hSpatParamRendCom->dirac_md_buffer_length; - pop_wmops(); - return error; + return; } -static ivas_error renderSbaToMultiBinaural( - input_sba *sbaInput, - const AUDIO_CONFIG outConfig, - float out[][L_FRAME48k] ) +static void renderMasaToMasa( + input_masa *masaInput, + IVAS_REND_AudioBuffer outAudio ) { - float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; - int16_t sf; - int16_t i, j, pos_idx; - COMBINED_ORIENTATION_DATA combinedOrientationDataLocal; - COMBINED_ORIENTATION_HANDLE pCombinedOrientationDataLocal; - ivas_error error; - IVAS_REND_AudioBuffer tmpRotBuffer; - const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + int16_t sf, band, dir, numDirs; + float ratioSum; + MASA_DECODER_EXT_OUT_META_HANDLE outMeta; + MASA_METADATA_FRAME *inMeta; + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + int16_t ts, i, j, l_ts; + float Chan_RealBuffer[MASA_MAX_TRANSPORT_CHANNELS][CLDFB_NO_CHANNELS_MAX]; + float Chan_ImagBuffer[MASA_MAX_TRANSPORT_CHANNELS][CLDFB_NO_CHANNELS_MAX]; + int16_t band_m_idx, block_m_idx; + int16_t mrange[2]; + int16_t brange[2]; + int16_t numAnalysisChannels; - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) - { - p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; - } - push_wmops( "renderSbaToMultiBinaural" ); - pMultiBinPoseData = &sbaInput->base.ctx.pSplitRendWrapper->multiBinPoseData; + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); - pCombinedOrientationDataLocal = *sbaInput->base.ctx.pCombinedOrientationData; - combinedOrientationDataLocal = *pCombinedOrientationDataLocal; - if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) + /* Calculate energy */ + l_ts = masaInput->base.inputBuffer.config.numSamplesPerChannel / CLDFB_NO_COL_MAX; + numAnalysisChannels = masaInput->hMasaPrerend->num_Cldfb_instances; + + /* do processing over all CLDFB time slots */ + for ( block_m_idx = 0; block_m_idx < MAX_PARAM_SPATIAL_SUBFRAMES; block_m_idx++ ) { - for ( sf = 1; sf < combinedOrientationDataLocal.num_subframes; sf++ ) + mrange[0] = DirAC_block_grouping[block_m_idx]; + mrange[1] = DirAC_block_grouping[block_m_idx + 1]; + + set_zero( masaInput->hMasaPrerend->energy[block_m_idx], MASA_FREQUENCY_BANDS ); + + for ( ts = mrange[0]; ts < mrange[1]; ts++ ) { - combinedOrientationDataLocal.Quaternions[sf] = combinedOrientationDataLocal.Quaternions[0]; - for ( i = 0; i < 3; i++ ) + for ( i = 0; i < numAnalysisChannels; i++ ) { - for ( j = 0; j < 3; j++ ) + cldfbAnalysis_ts( &( tmpBuffer[i][l_ts * ts] ), Chan_RealBuffer[i], Chan_ImagBuffer[i], l_ts, masaInput->hMasaPrerend->cldfbAnaEnc[i] ); + } + + /* Compute channel energy for metadata processing */ + for ( band_m_idx = 0; band_m_idx < MASA_FREQUENCY_BANDS; band_m_idx++ ) + { + brange[0] = MASA_band_grouping_24[band_m_idx]; + brange[1] = MASA_band_grouping_24[band_m_idx + 1]; + for ( j = brange[0]; j < brange[1]; j++ ) { - combinedOrientationDataLocal.Rmat[sf][i][j] = combinedOrientationDataLocal.Rmat[0][i][j]; + for ( i = 0; i < numAnalysisChannels; i++ ) + { + masaInput->hMasaPrerend->energy[block_m_idx][band_m_idx] += Chan_RealBuffer[0][j] * Chan_RealBuffer[0][j] + Chan_ImagBuffer[0][j] * Chan_ImagBuffer[0][j]; + } } } } } - tmpRotBuffer = sbaInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - - for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) + /* Copy audio channels if mismatch in number of transports */ + if ( masaInput->base.inputBuffer.config.numChannels == 1 && outAudio.config.numChannels == 2 ) { - IVAS_QUATERNION Quaternions_orig[MAX_PARAM_SPATIAL_SUBFRAMES], Quaternions_abs; - for ( i = 0; i < combinedOrientationDataLocal.num_subframes; i++ ) - { - Quaternions_orig[i] = combinedOrientationDataLocal.Quaternions[i]; - Quaternions_abs.w = -3.0f; - Quat2EulerDegree( combinedOrientationDataLocal.Quaternions[i], &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ - - Quaternions_abs.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; - Quaternions_abs.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; - Quaternions_abs.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; - combinedOrientationDataLocal.Quaternions[i] = Quaternions_abs; - QuatToRotMat( combinedOrientationDataLocal.Quaternions[i], combinedOrientationDataLocal.Rmat[i] ); - } - - - /* copy input for in-place rotation */ - mvr2r( sbaInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); + mvr2r( tmpBuffer[0], tmpBuffer[1], masaInput->base.inputBuffer.config.numSamplesPerChannel ); + } + else if ( masaInput->base.inputBuffer.config.numChannels == 2 && outAudio.config.numChannels == 1 ) + { + v_add( tmpBuffer[0], tmpBuffer[1], tmpBuffer[0], masaInput->base.inputBuffer.config.numSamplesPerChannel ); + } - pCombinedOrientationDataLocal = &combinedOrientationDataLocal; + /* Copy metadata */ + outMeta = masaInput->hMasaPrerend->hMasaOut; + inMeta = &masaInput->masaMetadata; + numDirs = inMeta->descriptive_meta.numberOfDirections + 1; - if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, &pCombinedOrientationDataLocal, sbaInput->rot_gains_prev[pos_idx], tmpRotBuffer ) ) != IVAS_ERR_OK ) + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) { - return error; - } - - copyBufferTo2dArray( tmpRotBuffer, tmpCrendBuffer ); - - assert( sbaInput->crendWrapper->hCrend[0]->hReverb == NULL ); + /* Remainder is always set to zero and energy removal is compensated in following steps + * to other ratios. */ + inMeta->common_meta.remainder_to_total_ratio[sf][band] = 0.0f; - /* call CREND */ - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &sbaInput->base.inputBuffer, *sbaInput->base.ctx.pOutSampleRate ), pos_idx ) ) != IVAS_ERR_OK ) - { - return error; - } + ratioSum = 0; + for ( dir = 0; dir < numDirs; dir++ ) + { + ratioSum += inMeta->directional_meta[dir].energy_ratio[sf][band]; + } + ratioSum += inMeta->common_meta.diffuse_to_total_ratio[sf][band]; - for ( i = 0; i < combinedOrientationDataLocal.num_subframes; i++ ) - { - combinedOrientationDataLocal.Quaternions[i] = Quaternions_orig[i]; + if ( ratioSum == 0.0f ) + { + for ( dir = 0; dir < numDirs; dir++ ) + { + inMeta->directional_meta[dir].energy_ratio[sf][band] = 0.0f; + } + inMeta->common_meta.diffuse_to_total_ratio[sf][band] = 1.0f; + } + else if ( ratioSum != 1.0f ) + { + for ( dir = 0; dir < numDirs; dir++ ) + { + inMeta->directional_meta[dir].energy_ratio[sf][band] /= ratioSum; + } + inMeta->common_meta.diffuse_to_total_ratio[sf][band] /= ratioSum; + } } + } - - /* move to output */ - for ( i = 0; i < BINAURAL_CHANNELS; i++ ) + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) { - mvr2r( tmpCrendBuffer[i], out[pos_idx * BINAURAL_CHANNELS + i], tmpRotBuffer.config.numSamplesPerChannel ); + outMeta->diffuseToTotalRatio[sf][band] = UINT8_MAX; + for ( dir = 0; dir < numDirs; dir++ ) + { + outMeta->directionIndex[dir][sf][band] = index_theta_phi_16( &inMeta->directional_meta[dir].elevation[sf][band], &inMeta->directional_meta[dir].azimuth[sf][band], masaInput->hMasaPrerend->sph_grid16 ); + outMeta->directToTotalRatio[dir][sf][band] = (uint8_t) floorf( inMeta->directional_meta[dir].energy_ratio[sf][band] * UINT8_MAX ); + outMeta->diffuseToTotalRatio[sf][band] -= outMeta->directToTotalRatio[dir][sf][band]; + outMeta->spreadCoherence[dir][sf][band] = (uint8_t) floorf( inMeta->directional_meta[dir].spread_coherence[sf][band] * UINT8_MAX ); + } + outMeta->surroundCoherence[sf][band] = (uint8_t) floorf( inMeta->common_meta.surround_coherence[sf][band] * UINT8_MAX ); } } - free( tmpRotBuffer.data ); - - pop_wmops(); - return IVAS_ERR_OK; -} - - -static ivas_error renderSbaToMultiBinauralCldfb( - input_sba *sbaInput, - float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - const int16_t low_res_pre_rend_rot, - const int16_t num_subframes ) -{ - float Cldfb_RealBuffer[MAX_OUTPUT_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer[MAX_OUTPUT_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - - copyBufferToCLDFBarray( sbaInput->base.inputBuffer, Cldfb_RealBuffer, Cldfb_ImagBuffer ); + copy_masa_descriptive_meta( &( outMeta->descriptiveMeta ), &( inMeta->descriptive_meta ) ); - ivas_rend_CldfbMultiBinRendProcess( sbaInput->cldfbRendWrapper.hCldfbRend, sbaInput->base.ctx.pCombinedOrientationData, &sbaInput->base.ctx.pSplitRendWrapper->multiBinPoseData, - Cldfb_RealBuffer, Cldfb_ImagBuffer, Cldfb_Out_Real, Cldfb_Out_Imag, low_res_pre_rend_rot, num_subframes ); + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); - return IVAS_ERR_OK; + return; } -static ivas_error renderSbaToSplitBinaural( - input_sba *sbaInput, +static ivas_error renderInputMasa( + input_masa *masaInput, const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { - float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - ivas_error error; - float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - int16_t low_res_pre_rend_rot; - - low_res_pre_rend_rot = 1; + IVAS_REND_AudioBuffer inAudio; + int16_t ch; + float *tmpBuffer[MAX_OUTPUT_CHANNELS]; + float tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; +#endif - push_wmops( "renderSbaToSplitBinaural" ); - error = IVAS_ERR_OK; + if ( !masaInput->metadataHasBeenFed ) + { + return IVAS_ERR_MISSING_METADATA; + } - if ( sbaInput->base.ctx.hhRendererConfig[0]->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) + inAudio = masaInput->base.inputBuffer; +#ifdef SPLIT_REND_WITH_HEAD_ROT + cldfb2tdSampleFact = outAudio.config.is_cldfb ? 2 : 1; + if ( ( masaInput->base.numNewSamplesPerChannel * cldfb2tdSampleFact != outAudio.config.numSamplesPerChannel ) && + ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) && ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) +#else + if ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) +#endif { - if ( ( renderSbaToMultiBinauralCldfb( sbaInput, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, low_res_pre_rend_rot, - getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) - { - return error; - } + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); + } + masaInput->base.numNewSamplesPerChannel = 0; - accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); + /* Apply input gain to new audio */ + v_multc( inAudio.data, masaInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + + + /* Todo split rend MASA ext: Need to add split renderer path to below. Options are: + * - Join together with binaural renderer path + * - Add a separate path for split renderer processing that is tuned for it */ + if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + { + /* MASA prerendering path for MASA -> MASA */ + renderMasaToMasa( masaInput, outAudio ); } else { - if ( ( renderSbaToMultiBinaural( sbaInput, outConfig, tmpCrendBuffer ) ) != IVAS_ERR_OK ) + int16_t num_subframes; + /* MASA external renderer -> other formats */ + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) { - return error; + tmpBuffer[ch] = tmpBuffer_buff[ch]; } - accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); + + num_subframes = masaInput->base.inputBuffer.config.numSamplesPerChannel / ( *masaInput->base.ctx.pOutSampleRate / ( IVAS_NUM_FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES ) ); + + switch ( masaInput->hMasaExtRend->renderer_type ) + { + case RENDERER_DIRAC: + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom ); + ivas_masa_ext_dirac_render( masaInput->hMasaExtRend, tmpBuffer, masaInput->hMasaExtRend->nchan_input, num_subframes ); + break; + case RENDERER_STEREO_PARAMETRIC: + case RENDERER_BINAURAL_PARAMETRIC: + case RENDERER_BINAURAL_PARAMETRIC_ROOM: + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom ); + ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes ); + break; + case RENDERER_DISABLE: + break; /* This happens for 1TC MASA to MONO where we just copy input transport to output */ + default: + return ( IVAS_ERROR( IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED, "Wrong output config for MASA input in external renderer\n" ) ); + } + + accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); } - pop_wmops(); - return error; + return IVAS_ERR_OK; } -#endif -static ivas_error renderSbaToBinaural( - input_sba *sbaInput, - const AUDIO_CONFIG outConfig, +static ivas_error renderActiveInputsMasa( + IVAS_REND_HANDLE hIvasRend, IVAS_REND_AudioBuffer outAudio ) { - float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - ivas_error error; - IVAS_REND_AudioBuffer tmpRotBuffer; - float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; int16_t i; - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; - int8_t combinedOrientationEnabled; - int16_t subframe_idx; + input_masa *pCurrentInput; + ivas_error error; - push_wmops( "renderSbaToBinaural" ); -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( sbaInput->base.ctx.hhRendererConfig[0]->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) + for ( i = 0, pCurrentInput = hIvasRend->inputsMasa; i < RENDERER_MAX_MASA_INPUTS; ++i, ++pCurrentInput ) { - float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + { + /* Skip inactive inputs */ + continue; + } - if ( ( error = renderSbaToMultiBinauralCldfb( sbaInput, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, 0, - getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) + if ( ( error = renderInputMasa( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) { return error; } + } - accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); + return IVAS_ERR_OK; +} + + +/*---------------------------------------------------------------------* + * IVAS_REND_GetMasaMetadata( ) + * + * Get metadata of the estimated MASA frame + *---------------------------------------------------------------------*/ + +ivas_error IVAS_REND_GetMasaMetadata( + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ + MASA_DECODER_EXT_OUT_META_HANDLE *hMasaExtOutMeta, /* o : pointer to handle, which will be set to point to analyzed MASA metadata */ + const IVAS_REND_AudioConfigType inputType /* i : Input type */ +) +{ + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + /* Get the metadata handle */ + if ( inputType == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) + { + *hMasaExtOutMeta = hIvasRend->inputsIsm->hOMasa->hMasaOut; + } + else if ( inputType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + *hMasaExtOutMeta = hIvasRend->inputsMc->hMcMasa->hMasaOut; + } + else if ( inputType == IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) + { + *hMasaExtOutMeta = hIvasRend->inputsSba->hDirAC->hMasaOut; } else -#endif { - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) - { - p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; - } + return IVAS_ERR_NOT_SUPPORTED_OPTION; + } - hCombinedOrientationData = sbaInput->base.ctx.pCombinedOrientationData; - combinedOrientationEnabled = 0; - if ( hCombinedOrientationData != NULL ) - { - for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) - { - if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) - { - combinedOrientationEnabled = 1; - break; - } - } - } + return IVAS_ERR_OK; +} - /* apply rotation */ - if ( combinedOrientationEnabled ) - { - tmpRotBuffer = sbaInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); - /* copy input for in-place rotation */ - mvr2r( sbaInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); +/*---------------------------------------------------------------------* + * IVAS_REND_MergeMasaMetadata( ) + * + * Merge MASA metadata from two formats + *---------------------------------------------------------------------*/ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, - sbaInput->base.ctx.pCombinedOrientationData, sbaInput->rot_gains_prev[0], tmpRotBuffer ) ) != IVAS_ERR_OK ) -#else - if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, - sbaInput->base.ctx.pCombinedOrientationData, sbaInput->rot_gains_prev, tmpRotBuffer ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } +ivas_error IVAS_REND_MergeMasaMetadata( + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ + MASA_DECODER_EXT_OUT_META_HANDLE *hMasaExtOutMeta, /* o : pointer to handle, which will be set to point to merged metadata */ + const IVAS_REND_AudioConfigType inputType1, /* i : Input type 1 */ + const IVAS_REND_AudioConfigType inputType2 /* i : Input type 2 */ +) +{ + MASA_DECODER_EXT_OUT_META_HANDLE inMeta2; + float( *inEne1 )[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; + float( *inEne2 )[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; - copyBufferTo2dArray( tmpRotBuffer, tmpCrendBuffer ); - free( tmpRotBuffer.data ); - } - else - { - copyBufferTo2dArray( sbaInput->base.inputBuffer, tmpCrendBuffer ); - } + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } - /* call CREND */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) -#else - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, - getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) -#endif - { - return error; - } + /* Input1 metadata and energy */ + if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) + { + *hMasaExtOutMeta = hIvasRend->inputsIsm->hOMasa->hMasaOut; + inEne1 = &( hIvasRend->inputsIsm->hOMasa->energy ); + } + else if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + *hMasaExtOutMeta = hIvasRend->inputsMc->hMcMasa->hMasaOut; + inEne1 = &( hIvasRend->inputsMc->hMcMasa->energy ); + } + else if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) + { + *hMasaExtOutMeta = hIvasRend->inputsSba->hDirAC->hMasaOut; + inEne1 = &( hIvasRend->inputsSba->hDirAC->energy ); + } + else if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + { + *hMasaExtOutMeta = hIvasRend->inputsMasa->hMasaPrerend->hMasaOut; + inEne1 = &( hIvasRend->inputsMasa->hMasaPrerend->energy ); + } + else + { + return IVAS_ERR_NOT_SUPPORTED_OPTION; + } - accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); + /* Input2 metadata and energy */ + if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) + { + inMeta2 = hIvasRend->inputsIsm->hOMasa->hMasaOut; + inEne2 = &( hIvasRend->inputsIsm->hOMasa->energy ); + } + else if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + { + inMeta2 = hIvasRend->inputsMc->hMcMasa->hMasaOut; + inEne2 = &( hIvasRend->inputsMc->hMcMasa->energy ); + } + else if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) + { + inMeta2 = hIvasRend->inputsSba->hDirAC->hMasaOut; + inEne2 = &( hIvasRend->inputsSba->hDirAC->energy ); + } + else if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + { + inMeta2 = hIvasRend->inputsMasa->hMasaPrerend->hMasaOut; + inEne2 = &( hIvasRend->inputsMasa->hMasaPrerend->energy ); + } + else + { + return IVAS_ERR_NOT_SUPPORTED_OPTION; } - pop_wmops(); + /* Merge metadata */ + ivas_prerend_merge_masa_metadata( *hMasaExtOutMeta, *hMasaExtOutMeta, inputType1, *inEne1, inMeta2, inputType2, *inEne2 ); + ( *hMasaExtOutMeta )->descriptiveMeta.numberOfChannels = hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA1 ? 0u : 1u; + return IVAS_ERR_OK; } -static ivas_error renderSbaToBinauralRoom( - input_sba *sbaInput, - const AUDIO_CONFIG outConfig, - IVAS_REND_AudioBuffer outAudio ) +/*---------------------------------------------------------------------* + * IVAS_REND_SetTotalNumberOfObjects( ) + * + * Set the total number of objects to the first object data + *---------------------------------------------------------------------*/ + +ivas_error IVAS_REND_SetTotalNumberOfObjects( + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ + const uint16_t total_num_objects /* i : total number of objects */ +) { - int16_t i; - int16_t tmp; - float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - ivas_error error; - IVAS_REND_AudioBuffer tmpRotBuffer; - IVAS_REND_AudioBuffer tmpMcBuffer; - IVAS_REND_AudioBuffer *tmpBufPtr; - float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; - const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; - int8_t combinedOrientationEnabled; - int16_t subframe_idx; + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } - tmpRotBuffer = outAudio; /* avoid compilation warning */ - push_wmops( "renderSbaToBinauralRoom" ); + hIvasRend->inputsIsm[0].total_num_objects = total_num_objects; + + return IVAS_ERR_OK; +} + + +/*---------------------------------------------------------------------* + * IVAS_REND_SetIsmMetadataDelay( ) + * + * Set the Metadata Delay in ms in order to sync with audio delay + *---------------------------------------------------------------------*/ + +ivas_error IVAS_REND_SetIsmMetadataDelay( + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ + const float sync_md_delay /* i : ISM Metadata Delay in ms to sync with audio delay */ +) +{ + int16_t i; - for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + if ( hIvasRend == NULL ) { - p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - hCombinedOrientationData = sbaInput->base.ctx.pCombinedOrientationData; - combinedOrientationEnabled = 0; - if ( hCombinedOrientationData != NULL ) + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) { - for ( subframe_idx = 0; subframe_idx < ( *hCombinedOrientationData )->num_subframes; subframe_idx++ ) - { - if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) - { - combinedOrientationEnabled = 1; - break; - } - } + hIvasRend->inputsIsm[i].ism_metadata_delay_ms = sync_md_delay; } - /* apply rotation */ - if ( combinedOrientationEnabled ) - { - tmpRotBuffer = sbaInput->base.inputBuffer; - tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); + return IVAS_ERR_OK; +} - /* copy input for in-place rotation */ - mvr2r( sbaInput->base.inputBuffer.data, tmpRotBuffer.data, tmpRotBuffer.config.numChannels * tmpRotBuffer.config.numSamplesPerChannel ); - if ( ( error = rotateFrameSba( sbaInput->base.inputBuffer, sbaInput->base.inConfig, sbaInput->base.ctx.pHeadRotData, - sbaInput->base.ctx.pCombinedOrientationData, +/*-------------------------------------------------------------------* + * getSamplesInternal() + * + * + *-------------------------------------------------------------------*/ + +static ivas_error getSamplesInternal( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ #ifdef SPLIT_REND_WITH_HEAD_ROT - sbaInput->rot_gains_prev[0], + IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */, + IVAS_REND_BitstreamBuffer *hBits /*i/o: buffer for input/output bitstream. Needed in split rendering mode*/ #else - sbaInput->rot_gains_prev, + IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ +#endif +) +{ + ivas_error error; + int16_t numOutChannels; +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; + IVAS_REND_AudioBuffer outAudioOrig; #endif - tmpRotBuffer ) ) != IVAS_ERR_OK ) - { - return error; - } - } - - /* intermediate rendering to 7_1_4 */ - tmpMcBuffer = sbaInput->base.inputBuffer; - if ( ( error = getAudioConfigNumChannels( IVAS_AUDIO_CONFIG_7_1_4, &tmp ) ) != IVAS_ERR_OK ) + /* Validate function arguments */ + if ( hIvasRend == NULL || outAudio.data == NULL ) { - return error; + return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - tmpMcBuffer.config.numChannels = tmp; - tmpMcBuffer.data = malloc( tmpMcBuffer.config.numSamplesPerChannel * tmpMcBuffer.config.numChannels * sizeof( float ) ); - set_zero( tmpMcBuffer.data, tmpMcBuffer.config.numChannels * tmpMcBuffer.config.numSamplesPerChannel ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + cldfb2tdSampleFact = ( outAudio.config.is_cldfb ) ? 2 : 1; - tmpBufPtr = ( combinedOrientationEnabled ) ? &tmpRotBuffer : &sbaInput->base.inputBuffer; - for ( i = 0; i < sbaInput->base.inputBuffer.config.numChannels; i++ ) + if ( outAudio.config.numSamplesPerChannel <= 0 || ( MAX_BUFFER_LENGTH_PER_CHANNEL < outAudio.config.numSamplesPerChannel && outAudio.config.is_cldfb == 0 ) || + ( ( MAX_BUFFER_LENGTH_PER_CHANNEL * cldfb2tdSampleFact ) < outAudio.config.numSamplesPerChannel && outAudio.config.is_cldfb == 1 ) ) +#else + if ( outAudio.config.numSamplesPerChannel <= 0 || MAX_BUFFER_LENGTH_PER_CHANNEL < outAudio.config.numSamplesPerChannel ) +#endif { - renderBufferChannel( *tmpBufPtr, i, sbaInput->hoaDecMtx[i], tmpMcBuffer ); + return IVAS_ERR_INVALID_BUFFER_SIZE; } - copyBufferTo2dArray( tmpMcBuffer, tmpCrendBuffer ); + if ( outAudio.config.numChannels <= 0 || MAX_OUTPUT_CHANNELS < outAudio.config.numChannels ) + { + return IVAS_ERR_WRONG_NUM_CHANNELS; + } - /* call CREND */ - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, IVAS_AUDIO_CONFIG_7_1_4, outConfig, - NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, #ifdef SPLIT_REND_WITH_HEAD_ROT - getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ), 0 ) ) != IVAS_ERR_OK ) + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && + hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && + hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM && + ( outAudio.config.numSamplesPerChannel * 1000 / cldfb2tdSampleFact ) != ( hIvasRend->num_subframes * BINAURAL_RENDERING_FRAME_SIZE_MS ) * hIvasRend->sampleRateOut ) #else - getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) ) ) != IVAS_ERR_OK ) + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && + outAudio.config.numSamplesPerChannel * 1000 != ( hIvasRend->num_subframes * BINAURAL_RENDERING_FRAME_SIZE_MS ) * hIvasRend->sampleRateOut ) #endif { - return error; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Binaural rendering requires specific frame size" ); } - accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); - - if ( combinedOrientationEnabled ) + /* Check that there is allowed configuration for MASA format output */ + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) { - free( tmpRotBuffer.data ); - } - free( tmpMcBuffer.data ); + int16_t i; + int16_t numMasaInputs = 0; + int16_t numOtherInputs = 0; - pop_wmops(); - return IVAS_ERR_OK; -} + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) + { + numMasaInputs += hIvasRend->inputsMasa[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; + } + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; i++ ) + { + numOtherInputs += hIvasRend->inputsMc[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; + } -#ifdef SPLIT_REND_WITH_HEAD_ROT -static ivas_error renderInputSplitBin( - input_split_post_rend *splitBinInput, - const AUDIO_CONFIG outConfig, - IVAS_REND_AudioBuffer outAudio, - const int16_t SplitRendBFI ) -{ - ivas_error error; - IVAS_REND_AudioBuffer inAudio; + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; i++ ) + { + numOtherInputs += hIvasRend->inputsSba[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; + } - inAudio = splitBinInput->base.inputBuffer; + /* For ISM, we check only first as all ISMs are handled together via OMASA when merging to MASA. */ + numOtherInputs += hIvasRend->inputsIsm[0].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; - splitBinInput->base.numNewSamplesPerChannel = 0; + if ( numMasaInputs == 0 || numOtherInputs == 0 ) + { + return IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED; + } + } - /* Apply input gain to new audio */ - v_multc( inAudio.data, - splitBinInput->base.gain, - inAudio.data, - inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); /* TODO: the output buffer is empty at this point, should be moved to a point after decoding the split bitstream */ - switch ( outConfig ) + if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { - case IVAS_AUDIO_CONFIG_BINAURAL: - error = renderSplitBinauralWithPostRot( splitBinInput, outAudio, SplitRendBFI ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; + return error; } - return error; -} -#endif - - -static void renderSbaToMasa( - input_sba *sbaInput, - IVAS_REND_AudioBuffer outAudio ) -{ - float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - - push_wmops( "renderMcToMasa" ); - copyBufferTo2dArray( sbaInput->base.inputBuffer, tmpRendBuffer ); - ivas_dirac_ana( sbaInput->hDirAC, tmpRendBuffer, sbaInput->base.inputBuffer.config.numSamplesPerChannel, outAudio.config.numChannels ); - accumulate2dArrayToBuffer( tmpRendBuffer, &outAudio ); - - pop_wmops(); - return; -} - - -static ivas_error renderInputSba( - input_sba *sbaInput, - const AUDIO_CONFIG outConfig, - IVAS_REND_AudioBuffer outAudio ) -{ - ivas_error error; - IVAS_REND_AudioBuffer inAudio; #ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t cldfb2tdSampleFact; + if ( numOutChannels != outAudio.config.numChannels && hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) +#else + if ( numOutChannels != outAudio.config.numChannels ) #endif + { + return IVAS_ERR_WRONG_NUM_CHANNELS; + } - error = IVAS_ERR_OK; - inAudio = sbaInput->base.inputBuffer; + /* Clear original output buffer */ + set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); #ifdef SPLIT_REND_WITH_HEAD_ROT - cldfb2tdSampleFact = outAudio.config.is_cldfb ? 2 : 1; - if ( ( sbaInput->base.numNewSamplesPerChannel * cldfb2tdSampleFact != outAudio.config.numSamplesPerChannel ) && - ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) && ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) -#else - if ( sbaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) -#endif + outAudioOrig = outAudio; + /* Use internal buffer if outputting split rendering bitstream */ + if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || + ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); + int16_t num_poses_orig; + num_poses_orig = hIvasRend->splitRendWrapper.multiBinPoseData.num_poses; + outAudio = hIvasRend->splitRendEncBuffer; + ivas_renderSplitGetMultiBinPoseData( &hIvasRend->hRendererConfig->split_rend_config, &hIvasRend->splitRendWrapper.multiBinPoseData, hIvasRend->headRotData.sr_pose_pred_axis ); + assert( num_poses_orig == hIvasRend->splitRendWrapper.multiBinPoseData.num_poses && "number of poses should not change dynamically" ); + + /* Clear output buffer for split rendering bitstream */ + set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); } - sbaInput->base.numNewSamplesPerChannel = 0; +#endif - /* Apply input gain to new audio */ - v_multc( inAudio.data, sbaInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + if ( ( error = renderActiveInputsIsm( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } - switch ( getAudioConfigType( outConfig ) ) + if ( ( error = renderActiveInputsMc( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) { - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - renderSbaToMc( sbaInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - renderSbaToSba( sbaInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - switch ( outConfig ) - { -#ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: - error = renderSbaToSplitBinaural( sbaInput, outConfig, outAudio ); - break; -#endif - case IVAS_AUDIO_CONFIG_BINAURAL: - error = renderSbaToBinaural( sbaInput, outConfig, outAudio ); - break; - case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: - case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: - error = renderSbaToBinauralRoom( sbaInput, outConfig, outAudio ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; - } - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - renderSbaToMasa( sbaInput, outAudio ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; + return error; } - return error; -} + if ( ( error = renderActiveInputsSba( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = renderActiveInputsMasa( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } #ifdef SPLIT_REND_WITH_HEAD_ROT -static ivas_error renderActiveInputsSplitBin( - IVAS_REND_HANDLE hIvasRend, - IVAS_REND_AudioBuffer outAudio ) -{ - int16_t i; - input_split_post_rend *pCurrentInput; - ivas_error error; - - for ( i = 0, pCurrentInput = hIvasRend->inputsSplitPost; i < RENDERER_MAX_BIN_INPUTS; ++i, ++pCurrentInput ) + if ( ( error = renderActiveInputsSplitBin( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) { - if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - /* Skip inactive inputs */ - continue; - } - - if ( ( error = renderInputSplitBin( pCurrentInput, hIvasRend->outputConfig, outAudio, hIvasRend->splitRendBFI ) ) != IVAS_ERR_OK ) - { - return error; - } + return error; } - return IVAS_ERR_OK; -} + if ( outAudio.config.is_cldfb == 0 ) + { +#ifndef DISABLE_LIMITER +#ifdef DEBUGGING + hIvasRend->numClipping += #endif + limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); +#endif + } +#else +#ifndef DISABLE_LIMITER +#ifdef DEBUGGING + hIvasRend->numClipping += +#endif + limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); +#endif +#endif -static ivas_error renderActiveInputsSba( - IVAS_REND_HANDLE hIvasRend, - IVAS_REND_AudioBuffer outAudio ) -{ - int16_t i; - input_sba *pCurrentInput; - ivas_error error; - - for ( i = 0, pCurrentInput = hIvasRend->inputsSba; i < RENDERER_MAX_SBA_INPUTS; ++i, ++pCurrentInput ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) { - if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) + IVAS_SPLIT_REND_BITS_DATA bits; + int16_t cldfb_in_flag; + float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + int16_t ch; + float *tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS], tmpBinaural_buff[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) { - /* Skip inactive inputs */ - continue; + tmpBinaural[ch] = tmpBinaural_buff[ch]; } - if ( ( error = renderInputSba( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) + if ( outAudio.config.is_cldfb == 1 ) { - return error; + cldfb_in_flag = 1; + copyBufferToCLDFBarray( outAudio, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural ); + } + else + { + cldfb_in_flag = 0; + copyBufferTo2dArray( outAudio, tmpBinaural_buff ); } - } - - return IVAS_ERR_OK; -} + /* Encode split rendering bitstream */ + convertBitsBufferToInternalBitsBuff( *hBits, &bits ); -static void copyMasaMetadataToDiracRenderer( - MASA_METADATA_FRAME *meta, - SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom ) -{ - int16_t band, sf, bin; - int16_t meta_write_index; + if ( ( error = ivas_renderMultiBinToSplitBinaural( &hIvasRend->splitRendWrapper, hIvasRend->headRotData.headPositions[0], hIvasRend->hRendererConfig->split_rend_config.splitRendBitRate, hIvasRend->hRendererConfig->split_rend_config.codec, hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, + &bits, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, ( const int16_t )( ( BINAURAL_MAXBANDS * hIvasRend->sampleRateOut ) / 48000 ), tmpBinaural, 1, cldfb_in_flag, ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + { + return error; + } - hSpatParamRendCom->numParametricDirections = meta->descriptive_meta.numberOfDirections + 1; - hSpatParamRendCom->numSimultaneousDirections = meta->descriptive_meta.numberOfDirections + 1; + convertInternalBitsBuffToBitsBuffer( hBits, bits ); - for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) - { - meta_write_index = ( hSpatParamRendCom->dirac_bs_md_write_idx + sf ) % hSpatParamRendCom->dirac_md_buffer_length; + /* reset to outAudioOrig in case of PCM output */ + outAudio = outAudioOrig; - for ( band = 0; band < MASA_MAXIMUM_CODING_SUBBANDS; band++ ) + if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) { - for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1]; bin++ ) - { - hSpatParamRendCom->azimuth[meta_write_index][bin] = (int16_t) meta->directional_meta[0].azimuth[sf][band]; - hSpatParamRendCom->elevation[meta_write_index][bin] = (int16_t) meta->directional_meta[0].elevation[sf][band]; - hSpatParamRendCom->energy_ratio1[meta_write_index][bin] = meta->directional_meta[0].energy_ratio[sf][band]; - hSpatParamRendCom->diffuseness_vector[meta_write_index][bin] = 1.0f - meta->directional_meta[0].energy_ratio[sf][band]; - hSpatParamRendCom->spreadCoherence[meta_write_index][bin] = meta->directional_meta[0].spread_coherence[sf][band]; - hSpatParamRendCom->surroundingCoherence[meta_write_index][bin] = meta->common_meta.surround_coherence[sf][band]; - - if ( hSpatParamRendCom->numSimultaneousDirections == 2 ) - { - hSpatParamRendCom->azimuth2[meta_write_index][bin] = (int16_t) meta->directional_meta[1].azimuth[sf][band]; - hSpatParamRendCom->elevation2[meta_write_index][bin] = (int16_t) meta->directional_meta[1].elevation[sf][band]; - hSpatParamRendCom->energy_ratio2[meta_write_index][bin] = meta->directional_meta[1].energy_ratio[sf][band]; - hSpatParamRendCom->diffuseness_vector[meta_write_index][bin] -= meta->directional_meta[1].energy_ratio[sf][band]; - hSpatParamRendCom->spreadCoherence2[meta_write_index][bin] = meta->directional_meta[1].spread_coherence[sf][band]; - } - } + accumulate2dArrayToBuffer( tmpBinaural_buff, &outAudio ); } } +#endif - hSpatParamRendCom->dirac_bs_md_write_idx = ( hSpatParamRendCom->dirac_bs_md_write_idx + MAX_PARAM_SPATIAL_SUBFRAMES ) % hSpatParamRendCom->dirac_md_buffer_length; - - return; + return IVAS_ERR_OK; } -static void renderMasaToMc( - input_masa *masaInput, - IVAS_REND_AudioBuffer outAudio ) +/*-------------------------------------------------------------------* + * IVAS_REND_GetSamples() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_GetSamples( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ +) { - int16_t ch; - float *tmpBuffer[MAX_OUTPUT_CHANNELS], tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) - { - tmpBuffer[ch] = tmpBuffer_buff[ch]; - } +#ifdef SPLIT_REND_WITH_HEAD_ROT + return getSamplesInternal( hIvasRend, outAudio, NULL ); +#else + return getSamplesInternal( hIvasRend, outAudio ); +#endif +} + + +#ifdef SPLIT_REND_WITH_HEAD_ROT +/*-------------------------------------------------------------------* + * IVAS_REND_GetSplitBinauralBitstream() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_GetSplitBinauralBitstream( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ + IVAS_REND_BitstreamBuffer *hBits /* o : buffer for output bitstream */ +) +{ + int16_t cldfb_in_flag; - copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); + cldfb_in_flag = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); + hIvasRend->splitRendEncBuffer.config.is_cldfb = cldfb_in_flag; - if ( masaInput->decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + if ( hIvasRend->hRendererConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && + ( hIvasRend->hRendererConfig->split_rend_config.dof == 0 || hIvasRend->hRendererConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) ) { - ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms * hIvasRend->num_subframes * (int16_t) ( hIvasRend->sampleRateOut / 1000 ); } else { - ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = (int16_t) ( hIvasRend->sampleRateOut / FRAMES_PER_SEC ); } + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel *= cldfb_in_flag ? 2 : 1; - accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); - - return; + /* hIvasRend->splitRendEncBuffer used for BINAURAL_SPLIT_CODED output + outAudio used for BINAURAL_SPLIT_PCM output */ + return getSamplesInternal( hIvasRend, outAudio, hBits ); } -static void renderMasaToSba( - input_masa *masaInput, - IVAS_REND_AudioBuffer outAudio ) +/*-------------------------------------------------------------------* + * IVAS_REND_GetSplitBinauralSamples() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_GetSplitBinauralSamples( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ + bool *needNewFrame ) { - int16_t ch; - float *tmpBuffer[MAX_OUTPUT_CHANNELS], tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + ivas_error error; - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + if ( ( error = getSamplesInternal( hIvasRend, outAudio, NULL ) ) != IVAS_ERR_OK ) { - tmpBuffer[ch] = tmpBuffer_buff[ch]; + return error; } + *needNewFrame = hIvasRend->inputsSplitPost[0].numCachedSamples == 0; - copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); - - ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); - - accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); - - return; + return IVAS_ERR_OK; } +#endif -static void renderMasaToBinaural( - input_masa *masaInput, -#ifdef SPLIT_REND_WITH_HEAD_ROT - IVAS_REND_AudioBuffer outAudio, - const int16_t is_split_rend_mode -#else - IVAS_REND_AudioBuffer outAudio -#endif +/*-------------------------------------------------------------------* + * IVAS_REND_Close() + * + * + *-------------------------------------------------------------------*/ + +void IVAS_REND_Close( + IVAS_REND_HANDLE *phIvasRend /* i/o: Pointer to renderer handle */ ) { - int16_t ch; - float *tmpBuffer[MAX_OUTPUT_CHANNELS], tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + uint16_t i; + IVAS_REND_HANDLE hIvasRend; - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + /* Validate function arguments */ + if ( phIvasRend == NULL || *phIvasRend == NULL ) { - tmpBuffer[ch] = tmpBuffer_buff[ch]; + return; } + hIvasRend = *phIvasRend; - copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); - -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( is_split_rend_mode ) + if ( hIvasRend->efapOutWrapper.hEfap != NULL ) { - int16_t sf, i, j; - COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; - pCombinedOrientationData = *masaInput->base.ctx.pCombinedOrientationData; - for ( sf = 1; sf < pCombinedOrientationData->num_subframes; sf++ ) - { - pCombinedOrientationData->Quaternions[sf] = pCombinedOrientationData->Quaternions[0]; - for ( i = 0; i < 3; i++ ) - { - for ( j = 0; j < 3; j++ ) - { - pCombinedOrientationData->Rmat[sf][i][j] = pCombinedOrientationData->Rmat[0][i][j]; - } - } - } + efap_free_data( &hIvasRend->efapOutWrapper.hEfap ); } -#endif - - ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + /* clear inputs */ + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) + { + clearInputIsm( &hIvasRend->inputsIsm[i] ); + } + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) + { + clearInputMc( &hIvasRend->inputsMc[i] ); + } + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) + { + clearInputSba( &hIvasRend->inputsSba[i] ); + } + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + clearInputMasa( &hIvasRend->inputsMasa[i] ); + } #ifdef SPLIT_REND_WITH_HEAD_ROT - if ( is_split_rend_mode ) + for ( i = 0; i < RENDERER_MAX_BIN_INPUTS; ++i ) { - accumulateCLDFBArrayToBuffer( masaInput->decDummy->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural, masaInput->decDummy->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural, &outAudio ); + clearInputSplitRend( &hIvasRend->inputsSplitPost[i] ); } - else - { #endif - accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); + + /* clear Config. Renderer */ + ivas_render_config_close( &( hIvasRend->hRendererConfig ) ); + + ivas_limiter_close( &hIvasRend->hLimiter ); + #ifdef SPLIT_REND_WITH_HEAD_ROT - } + /* Split binaural rendering */ + closeSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer ); #endif - return; -} + closeHeadRotation( hIvasRend ); + ivas_external_orientation_close( &hIvasRend->hExternalOrientationData ); + ivas_combined_orientation_close( &hIvasRend->hCombinedOrientationData ); -static void renderMasaToMasa( - input_masa *masaInput, - IVAS_REND_AudioBuffer outAudio ) -{ - int16_t sf, band, dir, numDirs; - float ratioSum; - MASA_DECODER_EXT_OUT_META_HANDLE outMeta; - MASA_METADATA_FRAME *inMeta; - float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; - int16_t ts, i, j, l_ts; - float Chan_RealBuffer[MASA_MAX_TRANSPORT_CHANNELS][CLDFB_NO_CHANNELS_MAX]; - float Chan_ImagBuffer[MASA_MAX_TRANSPORT_CHANNELS][CLDFB_NO_CHANNELS_MAX]; - int16_t band_m_idx, block_m_idx; - int16_t mrange[2]; - int16_t brange[2]; - int16_t numAnalysisChannels; + free( hIvasRend ); + *phIvasRend = NULL; - copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + return; +} - /* Calculate energy */ - l_ts = masaInput->base.inputBuffer.config.numSamplesPerChannel / CLDFB_NO_COL_MAX; - numAnalysisChannels = masaInput->hMasaPrerend->num_Cldfb_instances; - /* do processing over all CLDFB time slots */ - for ( block_m_idx = 0; block_m_idx < MAX_PARAM_SPATIAL_SUBFRAMES; block_m_idx++ ) - { - mrange[0] = DirAC_block_grouping[block_m_idx]; - mrange[1] = DirAC_block_grouping[block_m_idx + 1]; +#ifdef SPLIT_REND_WITH_HEAD_ROT +/*-------------------------------------------------------------------* + * IVAS_REND_openCldfb() + * + * + *-------------------------------------------------------------------*/ - set_zero( masaInput->hMasaPrerend->energy[block_m_idx], MASA_FREQUENCY_BANDS ); +ivas_error IVAS_REND_openCldfb( + IVAS_CLDFB_FILTER_BANK_HANDLE cldfbAna[IVAS_MAX_INPUT_CHANNELS], + IVAS_CLDFB_FILTER_BANK_HANDLE cldfbSyn[IVAS_MAX_OUTPUT_CHANNELS], + const int16_t num_in_chs, + const int16_t num_out_chs, + const int32_t output_Fs ) +{ + int16_t n; + ivas_error error; - for ( ts = mrange[0]; ts < mrange[1]; ts++ ) + for ( n = 0; n < num_in_chs; n++ ) + { + if ( ( error = openCldfb( &( cldfbAna[n] ), CLDFB_ANALYSIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) { - for ( i = 0; i < numAnalysisChannels; i++ ) - { - cldfbAnalysis_ts( &( tmpBuffer[i][l_ts * ts] ), Chan_RealBuffer[i], Chan_ImagBuffer[i], l_ts, masaInput->hMasaPrerend->cldfbAnaEnc[i] ); - } - - /* Compute channel energy for metadata processing */ - for ( band_m_idx = 0; band_m_idx < MASA_FREQUENCY_BANDS; band_m_idx++ ) - { - brange[0] = MASA_band_grouping_24[band_m_idx]; - brange[1] = MASA_band_grouping_24[band_m_idx + 1]; - for ( j = brange[0]; j < brange[1]; j++ ) - { - for ( i = 0; i < numAnalysisChannels; i++ ) - { - masaInput->hMasaPrerend->energy[block_m_idx][band_m_idx] += Chan_RealBuffer[0][j] * Chan_RealBuffer[0][j] + Chan_ImagBuffer[0][j] * Chan_ImagBuffer[0][j]; - } - } - } + return error; } } + for ( ; n < IVAS_MAX_INPUT_CHANNELS; n++ ) + { + cldfbAna[n] = NULL; + } - /* Copy audio channels if mismatch in number of transports */ - if ( masaInput->base.inputBuffer.config.numChannels == 1 && outAudio.config.numChannels == 2 ) + for ( n = 0; n < num_out_chs; n++ ) { - mvr2r( tmpBuffer[0], tmpBuffer[1], masaInput->base.inputBuffer.config.numSamplesPerChannel ); + if ( ( error = openCldfb( &( cldfbSyn[n] ), CLDFB_SYNTHESIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } } - else if ( masaInput->base.inputBuffer.config.numChannels == 2 && outAudio.config.numChannels == 1 ) + for ( ; n < IVAS_MAX_OUTPUT_CHANNELS; n++ ) { - v_add( tmpBuffer[0], tmpBuffer[1], tmpBuffer[0], masaInput->base.inputBuffer.config.numSamplesPerChannel ); + cldfbSyn[n] = NULL; } - /* Copy metadata */ - outMeta = masaInput->hMasaPrerend->hMasaOut; - inMeta = &masaInput->masaMetadata; - numDirs = inMeta->descriptive_meta.numberOfDirections + 1; + return IVAS_ERR_OK; +} - for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) - { - for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) - { - /* Remainder is always set to zero and energy removal is compensated in following steps - * to other ratios. */ - inMeta->common_meta.remainder_to_total_ratio[sf][band] = 0.0f; - ratioSum = 0; - for ( dir = 0; dir < numDirs; dir++ ) - { - ratioSum += inMeta->directional_meta[dir].energy_ratio[sf][band]; - } - ratioSum += inMeta->common_meta.diffuse_to_total_ratio[sf][band]; +/*-------------------------------------------------------------------* + * IVAS_REND_closeCldfb() + * + * + *-------------------------------------------------------------------*/ - if ( ratioSum == 0.0f ) - { - for ( dir = 0; dir < numDirs; dir++ ) - { - inMeta->directional_meta[dir].energy_ratio[sf][band] = 0.0f; - } - inMeta->common_meta.diffuse_to_total_ratio[sf][band] = 1.0f; - } - else if ( ratioSum != 1.0f ) - { - for ( dir = 0; dir < numDirs; dir++ ) - { - inMeta->directional_meta[dir].energy_ratio[sf][band] /= ratioSum; - } - inMeta->common_meta.diffuse_to_total_ratio[sf][band] /= ratioSum; - } +void IVAS_REND_closeCldfb( + IVAS_CLDFB_FILTER_BANK_HANDLE cldfbAna[IVAS_MAX_INPUT_CHANNELS], + IVAS_CLDFB_FILTER_BANK_HANDLE cldfbSyn[IVAS_MAX_OUTPUT_CHANNELS] ) +{ + int16_t n; + + for ( n = 0; n < IVAS_MAX_INPUT_CHANNELS; n++ ) + { + if ( cldfbAna[n] != NULL ) + { + deleteCldfb( &( cldfbAna[n] ) ); + cldfbAna[n] = NULL; } } - for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + for ( n = 0; n < IVAS_MAX_OUTPUT_CHANNELS; n++ ) { - for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) + if ( cldfbSyn[n] != NULL ) { - outMeta->diffuseToTotalRatio[sf][band] = UINT8_MAX; - for ( dir = 0; dir < numDirs; dir++ ) - { - outMeta->directionIndex[dir][sf][band] = index_theta_phi_16( &inMeta->directional_meta[dir].elevation[sf][band], &inMeta->directional_meta[dir].azimuth[sf][band], masaInput->hMasaPrerend->sph_grid16 ); - outMeta->directToTotalRatio[dir][sf][band] = (uint8_t) floorf( inMeta->directional_meta[dir].energy_ratio[sf][band] * UINT8_MAX ); - outMeta->diffuseToTotalRatio[sf][band] -= outMeta->directToTotalRatio[dir][sf][band]; - outMeta->spreadCoherence[dir][sf][band] = (uint8_t) floorf( inMeta->directional_meta[dir].spread_coherence[sf][band] * UINT8_MAX ); - } - outMeta->surroundCoherence[sf][band] = (uint8_t) floorf( inMeta->common_meta.surround_coherence[sf][band] * UINT8_MAX ); + deleteCldfb( &( cldfbSyn[n] ) ); + cldfbSyn[n] = NULL; } } - copy_masa_descriptive_meta( &( outMeta->descriptiveMeta ), &( inMeta->descriptive_meta ) ); - - accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); - return; } -static ivas_error renderInputMasa( - input_masa *masaInput, - const AUDIO_CONFIG outConfig, - IVAS_REND_AudioBuffer outAudio ) -{ - IVAS_REND_AudioBuffer inAudio; -#ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t cldfb2tdSampleFact; -#endif +/*-------------------------------------------------------------------* + * IVAS_REND_cldfbSynthesis_wrapper() + * + * + *-------------------------------------------------------------------*/ - if ( !masaInput->metadataHasBeenFed ) - { - return IVAS_ERR_MISSING_METADATA; - } +void IVAS_REND_cldfbAnalysis_ts_wrapper( + const float *timeIn, /* i : time buffer */ + float realBuffer[IVAS_CLDFB_NO_CHANNELS_MAX], /* o : real value buffer */ + float imagBuffer[IVAS_CLDFB_NO_CHANNELS_MAX], /* o : imag value buffer */ + const int16_t samplesToProcess, /* i : samples to process */ + IVAS_CLDFB_FILTER_BANK_HANDLE h_cldfb /* i : filterbank state */ +) +{ + cldfbAnalysis_ts( timeIn, realBuffer, imagBuffer, samplesToProcess, h_cldfb ); - inAudio = masaInput->base.inputBuffer; -#ifdef SPLIT_REND_WITH_HEAD_ROT - cldfb2tdSampleFact = outAudio.config.is_cldfb ? 2 : 1; - if ( ( masaInput->base.numNewSamplesPerChannel * cldfb2tdSampleFact != outAudio.config.numSamplesPerChannel ) && - ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) && ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) -#else - if ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) -#endif - { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); - } - masaInput->base.numNewSamplesPerChannel = 0; + return; +} - /* Apply input gain to new audio */ - v_multc( inAudio.data, masaInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); - switch ( getAudioConfigType( outConfig ) ) - { - case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: - renderMasaToMc( masaInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: - renderMasaToSba( masaInput, outAudio ); - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: - switch ( outConfig ) - { - case IVAS_AUDIO_CONFIG_BINAURAL: +/*-------------------------------------------------------------------* + * IVAS_REND_cldfbSynthesis_wrapper() + * + * + *-------------------------------------------------------------------*/ -#ifdef SPLIT_REND_WITH_HEAD_ROT - renderMasaToBinaural( masaInput, outAudio, 0 ); -#else - renderMasaToBinaural( masaInput, outAudio ); -#endif - break; - /* ToDo */ - // #ifdef FIX_196_REFACTOR_RENDERER_OUTPUT_CONFIG - // case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: - // case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: - // #else - // case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: - // #endif - // renderMasaToBinauralRoom( masaInput, outConfig, outAudio ); - // break; -#ifdef SPLIT_REND_WITH_HEAD_ROT - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: - case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: - renderMasaToBinaural( masaInput, outAudio, 1 ); - break; -#endif - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; - } - break; - case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: - renderMasaToMasa( masaInput, outAudio ); - break; - default: - return IVAS_ERR_INVALID_OUTPUT_FORMAT; - } +void IVAS_REND_cldfbSynthesis_wrapper( + float **realBuffer, /* i : real values */ + float **imagBuffer, /* i : imag values */ + float *timeOut, /* o : output time domain samples */ + const int16_t samplesToProcess, /* i : number of processed samples */ + IVAS_CLDFB_FILTER_BANK_HANDLE h_cldfb /* i : filter bank state */ +) +{ + cldfbSynthesis( realBuffer, imagBuffer, timeOut, samplesToProcess, h_cldfb ); - return IVAS_ERR_OK; + return; } +#endif -static ivas_error renderActiveInputsMasa( - IVAS_REND_HANDLE hIvasRend, - IVAS_REND_AudioBuffer outAudio ) +#ifdef DEBUGGING +/*-------------------------------------------------------------------* + * IVAS_REND_GetNoCLipping() + * + * + *-------------------------------------------------------------------*/ + +int32_t IVAS_REND_GetNoCLipping( + IVAS_REND_CONST_HANDLE hIvasRend ) { - int16_t i; - input_masa *pCurrentInput; - ivas_error error; - int16_t sf_idx; + return hIvasRend->numClipping; +} - for ( i = 0, pCurrentInput = hIvasRend->inputsMasa; i < RENDERER_MAX_MASA_INPUTS; ++i, ++pCurrentInput ) +int32_t IVAS_REND_GetCntFramesLimited( + IVAS_REND_CONST_HANDLE hIvasRend ) +{ + if ( hIvasRend->hLimiter == NULL ) { - if ( pCurrentInput->base.inConfig == IVAS_AUDIO_CONFIG_INVALID ) - { - /* Skip inactive inputs */ - continue; - } - - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && pCurrentInput->decDummy->hHeadTrackData != NULL ) - { - for ( sf_idx = 0; sf_idx < hIvasRend->num_subframes; ++sf_idx ) - { - pCurrentInput->decDummy->hHeadTrackData->Quaternions[sf_idx] = hIvasRend->headRotData.headPositions[sf_idx]; - pCurrentInput->decDummy->hHeadTrackData->Pos[sf_idx] = hIvasRend->headRotData.Pos[sf_idx]; - } - } - - if ( ( error = renderInputMasa( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) - { - return error; - } + return 0; } - return IVAS_ERR_OK; + return hIvasRend->hLimiter->cnt_frames_limited; } +#endif +static ivas_error ivas_masa_ext_rend_dirac_rend_init( + input_masa *inputMasa ) +{ + int16_t nchan_out_woLFE; + int16_t nchan_transport; + uint16_t i, j, k; + float ls_azimuth[MAX_OUTPUT_CHANNELS]; + float ls_elevation[MAX_OUTPUT_CHANNELS]; + int32_t output_Fs; + ivas_error error; + DIRAC_REND_HANDLE hDirACRend; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; -/*---------------------------------------------------------------------* - * IVAS_REND_GetMasaMetadata( ) - * - * Get metadata of the estimated MASA frame - *---------------------------------------------------------------------*/ + error = IVAS_ERR_OK; -ivas_error IVAS_REND_GetMasaMetadata( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ - MASA_DECODER_EXT_OUT_META_HANDLE *hMasaExtOutMeta, /* o : pointer to handle, which will be set to point to analyzed MASA metadata */ - const IVAS_REND_AudioConfigType inputType /* i : Input type */ -) -{ - if ( hIvasRend == NULL ) - { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; - } + hDirACRend = NULL; + output_Fs = *( inputMasa->base.ctx.pOutSampleRate ); - /* Get the metadata handle */ - if ( inputType == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) - { - *hMasaExtOutMeta = hIvasRend->inputsIsm->hOMasa->hMasaOut; - } - else if ( inputType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) - { - *hMasaExtOutMeta = hIvasRend->inputsMc->hMcMasa->hMasaOut; - } - else if ( inputType == IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) - { - *hMasaExtOutMeta = hIvasRend->inputsSba->hDirAC->hMasaOut; - } - else + hSpatParamRendCom = inputMasa->hMasaExtRend->hSpatParamRendCom; + + /*-----------------------------------------------------------------* + * prepare library opening + *-----------------------------------------------------------------*/ + + if ( ( hDirACRend = (DIRAC_REND_HANDLE) malloc( sizeof( DIRAC_REND_DATA ) ) ) == NULL ) { - return IVAS_ERR_NOT_SUPPORTED_OPTION; + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC renderer\n" ) ); } - return IVAS_ERR_OK; -} - + nchan_transport = inputMasa->base.inConfig == IVAS_AUDIO_CONFIG_MASA2 ? 2 : 1; -/*---------------------------------------------------------------------* - * IVAS_REND_MergeMasaMetadata( ) - * - * Merge MASA metadata from two formats - *---------------------------------------------------------------------*/ + /*-----------------------------------------------------------------* + * output setup: for parametric binaural renderer, use output setup, otherwise internal setup + *-----------------------------------------------------------------*/ -ivas_error IVAS_REND_MergeMasaMetadata( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ - MASA_DECODER_EXT_OUT_META_HANDLE *hMasaExtOutMeta, /* o : pointer to handle, which will be set to point to merged metadata */ - const IVAS_REND_AudioConfigType inputType1, /* i : Input type 1 */ - const IVAS_REND_AudioConfigType inputType2 /* i : Input type 2 */ -) -{ - MASA_DECODER_EXT_OUT_META_HANDLE inMeta2; - float( *inEne1 )[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; - float( *inEne2 )[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; + ivas_output_init( &hDirACRend->hOutSetup, *inputMasa->base.ctx.pOutConfig ); - if ( hIvasRend == NULL ) + if ( hDirACRend->hOutSetup.output_config == IVAS_AUDIO_CONFIG_LS_CUSTOM ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + /* Copy from ivas_ls_custom_setup */ + hDirACRend->hOutSetup.nchan_out_woLFE = inputMasa->base.ctx.pCustomLsOut->num_spk; + hDirACRend->hOutSetup.ls_azimuth = inputMasa->base.ctx.pCustomLsOut->ls_azimuth; + hDirACRend->hOutSetup.ls_elevation = inputMasa->base.ctx.pCustomLsOut->ls_elevation; + + hDirACRend->hOutSetup.num_lfe = inputMasa->base.ctx.pCustomLsOut->num_lfe; + hDirACRend->hOutSetup.index_lfe[0] = inputMasa->base.ctx.pCustomLsOut->lfe_idx[0]; + + hDirACRend->hOutSetup.is_loudspeaker_setup = TRUE; + hDirACRend->hOutSetup.is_planar_setup = (int8_t) inputMasa->base.ctx.pCustomLsOut->is_planar_setup; } - /* Input1 metadata and energy */ - if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) + nchan_out_woLFE = hDirACRend->hOutSetup.nchan_out_woLFE; + + if ( hDirACRend->hOutSetup.ls_azimuth != NULL && hDirACRend->hOutSetup.ls_elevation != NULL ) { - *hMasaExtOutMeta = hIvasRend->inputsIsm->hOMasa->hMasaOut; - inEne1 = &( hIvasRend->inputsIsm->hOMasa->energy ); + mvr2r( hDirACRend->hOutSetup.ls_azimuth, ls_azimuth, nchan_out_woLFE ); + mvr2r( hDirACRend->hOutSetup.ls_elevation, ls_elevation, nchan_out_woLFE ); } - else if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + + if ( hDirACRend->hOutSetup.ambisonics_order == -1 ) { - *hMasaExtOutMeta = hIvasRend->inputsMc->hMcMasa->hMasaOut; - inEne1 = &( hIvasRend->inputsMc->hMcMasa->energy ); + hDirACRend->hOutSetup.ambisonics_order = SBA_HOA3_ORDER; /* Order 3 is used by default in DirAC for SHD processing */ + if ( hDirACRend->hOutSetup.output_config == IVAS_AUDIO_CONFIG_MONO || hDirACRend->hOutSetup.output_config == IVAS_AUDIO_CONFIG_STEREO ) + { + hDirACRend->hOutSetup.ambisonics_order = SBA_FOA_ORDER; + } } - else if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) + else if ( hDirACRend->hOutSetup.ambisonics_order >= SBA_FOA_ORDER ) { - *hMasaExtOutMeta = hIvasRend->inputsSba->hDirAC->hMasaOut; - inEne1 = &( hIvasRend->inputsSba->hDirAC->energy ); + mvr2r( ls_azimuth_4d4, ls_azimuth, DIRAC_HOA_RENDERING_NUM_VIRT_DECORR_LS ); + mvr2r( ls_elevation_4d4, ls_elevation, DIRAC_HOA_RENDERING_NUM_VIRT_DECORR_LS ); } - else if ( inputType1 == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + + /*-----------------------------------------------------------------* + * set input parameters + *-----------------------------------------------------------------*/ + + if ( hDirACRend->hOutSetup.output_config == IVAS_AUDIO_CONFIG_MONO ) { - *hMasaExtOutMeta = hIvasRend->inputsMasa->hMasaPrerend->hMasaOut; - inEne1 = &( hIvasRend->inputsMasa->hMasaPrerend->energy ); + hDirACRend->synthesisConf = DIRAC_SYNTHESIS_MONO; + hDirACRend->panningConf = DIRAC_PANNING_HOA3; + nchan_out_woLFE = 1; } - else + else if ( hDirACRend->hOutSetup.is_loudspeaker_setup ) { - return IVAS_ERR_NOT_SUPPORTED_OPTION; + hDirACRend->synthesisConf = DIRAC_SYNTHESIS_PSD_LS; + hDirACRend->panningConf = DIRAC_PANNING_VBAP; } - - /* Input2 metadata and energy */ - if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) + else if ( !hDirACRend->hOutSetup.is_loudspeaker_setup && nchan_transport > 1 ) { - inMeta2 = hIvasRend->inputsIsm->hOMasa->hMasaOut; - inEne2 = &( hIvasRend->inputsIsm->hOMasa->energy ); + hDirACRend->synthesisConf = DIRAC_SYNTHESIS_PSD_SHD; + hDirACRend->panningConf = DIRAC_PANNING_HOA3; } - else if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) + else { - inMeta2 = hIvasRend->inputsMc->hMcMasa->hMasaOut; - inEne2 = &( hIvasRend->inputsMc->hMcMasa->energy ); + hDirACRend->synthesisConf = DIRAC_SYNTHESIS_GAIN_SHD; + hDirACRend->panningConf = DIRAC_PANNING_HOA3; } - else if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) + + if ( ( hDirACRend->frequency_axis = (float *) malloc( hSpatParamRendCom->num_freq_bands * sizeof( float ) ) ) == NULL ) { - inMeta2 = hIvasRend->inputsSba->hDirAC->hMasaOut; - inEne2 = &( hIvasRend->inputsSba->hDirAC->energy ); + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); } - else if ( inputType2 == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + set_f( hDirACRend->frequency_axis, 0.0f, hSpatParamRendCom->num_freq_bands ); + + ivas_dirac_dec_get_frequency_axis( hDirACRend->frequency_axis, output_Fs, hSpatParamRendCom->num_freq_bands ); + + if ( hDirACRend->panningConf == DIRAC_PANNING_HOA3 && nchan_transport == 2 ) { - inMeta2 = hIvasRend->inputsMasa->hMasaPrerend->hMasaOut; - inEne2 = &( hIvasRend->inputsMasa->hMasaPrerend->energy ); + if ( ( hDirACRend->masa_stereo_type_detect = (MASA_STEREO_TYPE_DETECT *) malloc( sizeof( MASA_STEREO_TYPE_DETECT ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); + } + ivas_masa_init_stereotype_detection( hDirACRend->masa_stereo_type_detect ); } else { - return IVAS_ERR_NOT_SUPPORTED_OPTION; + hDirACRend->masa_stereo_type_detect = NULL; } - /* Merge metadata */ - ivas_prerend_merge_masa_metadata( *hMasaExtOutMeta, *hMasaExtOutMeta, inputType1, *inEne1, inMeta2, inputType2, *inEne2 ); - ( *hMasaExtOutMeta )->descriptiveMeta.numberOfChannels = hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_MASA1 ? 0u : 1u; - - return IVAS_ERR_OK; -} - + hSpatParamRendCom->numIsmDirections = 0; -/*---------------------------------------------------------------------* - * IVAS_REND_SetTotalNumberOfObjects( ) - * - * Set the total number of objects to the first object data - *---------------------------------------------------------------------*/ + /*-----------------------------------------------------------------* + * (re)configure sub-modules + *-----------------------------------------------------------------*/ -ivas_error IVAS_REND_SetTotalNumberOfObjects( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ - const uint16_t total_num_objects /* i : total number of objects */ -) -{ - if ( hIvasRend == NULL ) + /* prototype signal computation */ + /* allocate output setup related arrays */ + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_LS ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + /* Directional and diffuses components in output LS format */ + hDirACRend->num_outputs_diff = nchan_out_woLFE; + hDirACRend->num_outputs_dir = nchan_out_woLFE; } - - hIvasRend->inputsIsm[0].total_num_objects = total_num_objects; - - return IVAS_ERR_OK; -} - - -/*---------------------------------------------------------------------* - * IVAS_REND_SetIsmMetadataDelay( ) - * - * Set the Metadata Delay in ms in order to sync with audio delay - *---------------------------------------------------------------------*/ - -ivas_error IVAS_REND_SetIsmMetadataDelay( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ - const float sync_md_delay /* i : ISM Metadata Delay in ms to sync with audio delay */ -) -{ - int16_t i; - - if ( hIvasRend == NULL ) + else if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + /* Directional and diffuses components in SHD */ + /* Diffuseness components up to 1st order */ + hDirACRend->num_outputs_diff = ( min( hDirACRend->hOutSetup.ambisonics_order, 1 ) + 1 ) * ( min( hDirACRend->hOutSetup.ambisonics_order, 1 ) + 1 ); + hDirACRend->num_outputs_dir = ivas_sba_get_nchan( hDirACRend->hOutSetup.ambisonics_order, 0 ); } - - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) + else if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD ) { - hIvasRend->inputsIsm[i].ism_metadata_delay_ms = sync_md_delay; + hDirACRend->num_outputs_diff = DIRAC_HOA_RENDERING_NUM_VIRT_DECORR_LS; + hDirACRend->num_outputs_dir = nchan_out_woLFE; } - - return IVAS_ERR_OK; -} - - -/*-------------------------------------------------------------------* - * getSamplesInternal() - * - * - *-------------------------------------------------------------------*/ - -static ivas_error getSamplesInternal( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ -#ifdef SPLIT_REND_WITH_HEAD_ROT - IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */, - IVAS_REND_BitstreamBuffer *hBits /*i/o: buffer for input/output bitstream. Needed in split rendering mode*/ -#else - IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ -#endif -) -{ - ivas_error error; - int16_t numOutChannels; -#ifdef SPLIT_REND_WITH_HEAD_ROT - int16_t cldfb2tdSampleFact; - IVAS_REND_AudioBuffer outAudioOrig; -#endif - - /* Validate function arguments */ - if ( hIvasRend == NULL || outAudio.data == NULL ) + else if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) { - return IVAS_ERR_UNEXPECTED_NULL_POINTER; + hDirACRend->num_outputs_diff = 1; /* There is one output channel in mono */ + hDirACRend->num_outputs_dir = 2; /* Two channels are pre-rendered for stereo type detection */ } - -#ifdef SPLIT_REND_WITH_HEAD_ROT - cldfb2tdSampleFact = ( outAudio.config.is_cldfb ) ? 2 : 1; - - if ( outAudio.config.numSamplesPerChannel <= 0 || ( MAX_BUFFER_LENGTH_PER_CHANNEL < outAudio.config.numSamplesPerChannel && outAudio.config.is_cldfb == 0 ) || - ( ( MAX_BUFFER_LENGTH_PER_CHANNEL * cldfb2tdSampleFact ) < outAudio.config.numSamplesPerChannel && outAudio.config.is_cldfb == 1 ) ) -#else - if ( outAudio.config.numSamplesPerChannel <= 0 || MAX_BUFFER_LENGTH_PER_CHANNEL < outAudio.config.numSamplesPerChannel ) -#endif + else { - return IVAS_ERR_INVALID_BUFFER_SIZE; + assert( 0 && "DirAC: not existing synthesis methods!" ); } - if ( outAudio.config.numChannels <= 0 || MAX_OUTPUT_CHANNELS < outAudio.config.numChannels ) + if ( ( hDirACRend->proto_index_dir = (int16_t *) malloc( sizeof( int16_t ) * hDirACRend->num_outputs_dir ) ) == NULL ) { - return IVAS_ERR_WRONG_NUM_CHANNELS; + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); } -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && - hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && - hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM && - ( outAudio.config.numSamplesPerChannel * 1000 / cldfb2tdSampleFact ) != ( hIvasRend->num_subframes * BINAURAL_RENDERING_FRAME_SIZE_MS ) * hIvasRend->sampleRateOut ) -#else - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && - outAudio.config.numSamplesPerChannel * 1000 != ( hIvasRend->num_subframes * BINAURAL_RENDERING_FRAME_SIZE_MS ) * hIvasRend->sampleRateOut ) -#endif + if ( ( hDirACRend->proto_index_diff = (int16_t *) malloc( sizeof( int16_t ) * hDirACRend->num_outputs_diff ) ) == NULL ) { - return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Binaural rendering requires specific frame size" ); + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); } - /* Check that there is allowed configuration for MASA format output */ - if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) - { - int16_t i; - int16_t numMasaInputs = 0; - int16_t numOtherInputs = 0; - - for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) - { - numMasaInputs += hIvasRend->inputsMasa[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; - } + set_s( hDirACRend->proto_index_dir, 0, hDirACRend->num_outputs_dir ); + set_s( hDirACRend->proto_index_diff, 0, hDirACRend->num_outputs_diff ); - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; i++ ) + hDirACRend->sba_map_tc = sba_map_tc; + + if ( nchan_transport == 1 ) + { + hDirACRend->num_protos_ambi = 1; + hDirACRend->num_protos_dir = 1; + hDirACRend->num_protos_diff = 1; + } + else if ( nchan_transport == 2 ) + { + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) { - numOtherInputs += hIvasRend->inputsMc[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; + hDirACRend->num_protos_ambi = 2; + hDirACRend->num_protos_diff = 1; + hDirACRend->num_protos_dir = 2; + hDirACRend->proto_index_dir[1] = 1; } - - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; i++ ) + else if ( hDirACRend->hOutSetup.output_config == IVAS_AUDIO_CONFIG_MONO ) { - numOtherInputs += hIvasRend->inputsSba[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; + /* Following the foa rendering for code compatibility */ + hDirACRend->num_protos_ambi = 2; + hDirACRend->num_protos_dir = 2; + hDirACRend->num_protos_diff = 3; + hDirACRend->proto_index_dir[0] = 0; + hDirACRend->proto_index_diff[0] = 0; } + else + { + hDirACRend->num_protos_ambi = 2; + hDirACRend->num_protos_diff = 3; - /* For ISM, we check only first as all ISMs are handled together via OMASA when merging to MASA. */ - numOtherInputs += hIvasRend->inputsIsm[0].base.inConfig == IVAS_AUDIO_CONFIG_INVALID ? 0 : 1; + for ( k = 0; k < hDirACRend->num_outputs_diff; k++ ) + { + if ( ls_azimuth[k] > 0.0f ) + { + hDirACRend->proto_index_diff[k] = 1; + } + else if ( ls_azimuth[k] < 0.0f ) + { + hDirACRend->proto_index_diff[k] = 2; + } + else + { + hDirACRend->proto_index_diff[k] = 0; + } + } - if ( numMasaInputs == 0 || numOtherInputs == 0 ) - { - return IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED; + if ( hDirACRend->hOutSetup.is_loudspeaker_setup ) + { + hDirACRend->num_protos_dir = 3; + mvs2s( hDirACRend->proto_index_diff, hDirACRend->proto_index_dir, nchan_out_woLFE ); + } + else + { + hDirACRend->num_protos_dir = 2; + hDirACRend->proto_index_dir[1] = 1; + } } } - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) + /* direct/diffuse responses */ + if ( ( hDirACRend->diffuse_response_function = (float *) malloc( sizeof( float ) * hDirACRend->num_outputs_dir ) ) == NULL ) { - return error; + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); } -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( numOutChannels != outAudio.config.numChannels && hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) -#else - if ( numOutChannels != outAudio.config.numChannels ) -#endif + if ( ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_LS ) || ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD ) || ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) ) { - return IVAS_ERR_WRONG_NUM_CHANNELS; + initDiffuseResponses( hDirACRend->diffuse_response_function, nchan_out_woLFE, hDirACRend->hOutSetup.output_config, + hDirACRend->hOutSetup, hDirACRend->hOutSetup.ambisonics_order, MASA_FORMAT, &hDirACRend->num_ele_spk_no_diffuse_rendering, IVAS_AUDIO_CONFIG_INVALID ); + } + else + { + initDiffuseResponses( hDirACRend->diffuse_response_function, hDirACRend->num_outputs_dir, IVAS_AUDIO_CONFIG_FOA, + hDirACRend->hOutSetup, hDirACRend->hOutSetup.ambisonics_order, MASA_FORMAT, &hDirACRend->num_ele_spk_no_diffuse_rendering, IVAS_AUDIO_CONFIG_INVALID ); } - /* Clear original output buffer */ - set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); - -#ifdef SPLIT_REND_WITH_HEAD_ROT - outAudioOrig = outAudio; - /* Use internal buffer if outputting split rendering bitstream */ - if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || - ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + hDirACRend->hoa_encoder = NULL; + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD ) { - int16_t num_poses_orig; - num_poses_orig = hIvasRend->splitRendWrapper.multiBinPoseData.num_poses; - outAudio = hIvasRend->splitRendEncBuffer; - ivas_renderSplitGetMultiBinPoseData( &hIvasRend->hRendererConfig->split_rend_config, &hIvasRend->splitRendWrapper.multiBinPoseData, hIvasRend->headRotData.sr_pose_pred_axis ); - assert( num_poses_orig == hIvasRend->splitRendWrapper.multiBinPoseData.num_poses && "number of poses should not change dynamically" ); + if ( ( hDirACRend->hoa_encoder = (float *) malloc( nchan_out_woLFE * hDirACRend->num_outputs_diff * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); + } - /* Clear output buffer for split rendering bitstream */ - set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); + set_f( hDirACRend->hoa_encoder, 0.0f, nchan_out_woLFE * hDirACRend->num_outputs_diff ); + compute_hoa_encoder_mtx( ls_azimuth, ls_elevation, hDirACRend->hoa_encoder, hDirACRend->num_outputs_diff, hDirACRend->hOutSetup.ambisonics_order ); } -#endif - if ( ( error = renderActiveInputsIsm( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + /* VBAP */ + inputMasa->hMasaExtRend->hVBAPdata = NULL; + + if ( hDirACRend->panningConf == DIRAC_PANNING_VBAP ) { - return error; + if ( ( error = vbap_init_data( &( inputMasa->hMasaExtRend->hVBAPdata ), ls_azimuth, ls_elevation, nchan_out_woLFE, MASA_FORMAT ) ) != IVAS_ERR_OK ) + { + return error; + } } - if ( ( error = renderActiveInputsMc( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + /* HOA panning/dec */ + hDirACRend->hoa_decoder = NULL; + if ( hDirACRend->panningConf == DIRAC_PANNING_HOA3 ) { - return error; + if ( hDirACRend->hOutSetup.is_loudspeaker_setup ) + { + if ( ( error = ivas_sba_get_hoa_dec_matrix( hDirACRend->hOutSetup, &inputMasa->hMasaExtRend->hoa_dec_mtx, hDirACRend->hOutSetup.ambisonics_order ) ) != IVAS_ERR_OK ) + { + return error; + } + + hDirACRend->hoa_decoder = inputMasa->hMasaExtRend->hoa_dec_mtx; + } } - if ( ( error = renderActiveInputsSba( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + /* decorrelation */ + hDirACRend->proto_signal_decorr_on = 1; + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) { - return error; + hDirACRend->proto_signal_decorr_on = 0; } - if ( ( error = renderActiveInputsMasa( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + if ( hDirACRend->proto_signal_decorr_on ) { - return error; + if ( ( error = ivas_dirac_dec_decorr_open( &( hDirACRend->h_freq_domain_decorr_ap_params ), + &( hDirACRend->h_freq_domain_decorr_ap_state ), + hSpatParamRendCom->num_freq_bands, + hDirACRend->num_outputs_diff, + hDirACRend->num_protos_diff, + hDirACRend->synthesisConf, + hDirACRend->frequency_axis, + nchan_transport, + output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } } -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = renderActiveInputsSplitBin( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + /* output synthesis */ + if ( ( ivas_dirac_dec_output_synthesis_open( hSpatParamRendCom, hDirACRend, RENDERER_DIRAC, nchan_transport, output_Fs, 0 ) ) != IVAS_ERR_OK ) { return error; } + hDirACRend->h_output_synthesis_psd_params.use_onset_filters = hDirACRend->proto_signal_decorr_on; - if ( outAudio.config.is_cldfb == 0 ) + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) { -#ifndef DISABLE_LIMITER -#ifdef DEBUGGING - hIvasRend->numClipping += -#endif - limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); -#endif + hDirACRend->h_output_synthesis_psd_params.use_onset_filters = 0; } -#else -#ifndef DISABLE_LIMITER -#ifdef DEBUGGING - hIvasRend->numClipping += -#endif - limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); -#endif -#endif + /*-----------------------------------------------------------------* + * memory allocation + *-----------------------------------------------------------------*/ -#ifdef SPLIT_REND_WITH_HEAD_ROT - if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) { - IVAS_SPLIT_REND_BITS_DATA bits; - int16_t cldfb_in_flag; - float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - int16_t ch; - float *tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS], tmpBinaural_buff[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; - - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + hDirACRend->proto_frame_f = NULL; + } + else + { + if ( ( hDirACRend->proto_frame_f = (float *) malloc( sizeof( float ) * 2 * hDirACRend->num_protos_diff * hSpatParamRendCom->num_freq_bands ) ) == NULL ) { - tmpBinaural[ch] = tmpBinaural_buff[ch]; + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC\n" ) ); } + } - if ( outAudio.config.is_cldfb == 1 ) - { - cldfb_in_flag = 1; - copyBufferToCLDFBarray( outAudio, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural ); - } - else - { - cldfb_in_flag = 0; - copyBufferTo2dArray( outAudio, tmpBinaural_buff ); - } - /* Encode split rendering bitstream */ - convertBitsBufferToInternalBitsBuff( *hBits, &bits ); + hDirACRend->buffer_energy = NULL; - if ( ( error = ivas_renderMultiBinToSplitBinaural( &hIvasRend->splitRendWrapper, hIvasRend->headRotData.headPositions[0], hIvasRend->hRendererConfig->split_rend_config.splitRendBitRate, hIvasRend->hRendererConfig->split_rend_config.codec, hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, - &bits, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, ( const int16_t )( ( BINAURAL_MAXBANDS * hIvasRend->sampleRateOut ) / 48000 ), tmpBinaural, 1, cldfb_in_flag, ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + for ( i = 0; i < DIRAC_NUM_DIMS; i++ ) + { + for ( j = 0; j < DIRAC_NO_COL_AVG_DIFF; j++ ) { - return error; + hDirACRend->buffer_intensity_real[i][j] = NULL; } + } - convertInternalBitsBuffToBitsBuffer( hBits, bits ); - - /* reset to outAudioOrig in case of PCM output */ - outAudio = outAudioOrig; + /* output synthesis */ + ivas_dirac_dec_output_synthesis_init( hSpatParamRendCom, hDirACRend, nchan_out_woLFE, 0 ); - if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) - { - accumulate2dArrayToBuffer( tmpBinaural_buff, &outAudio ); - } + /* Allocate stack memory */ + if ( ( error = ivas_dirac_alloc_mem( hDirACRend, RENDERER_DIRAC, hSpatParamRendCom->num_freq_bands, &( hDirACRend->stack_mem ), 0 ) ) != IVAS_ERR_OK ) + { + return error; } -#endif - return IVAS_ERR_OK; -} + inputMasa->hMasaExtRend->hDirACRend = hDirACRend; + return error; +} -/*-------------------------------------------------------------------* - * IVAS_REND_GetSamples() - * - * - *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_GetSamples( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ +// Todo split rend MASA ext: Add split renderer data here somehow. Probably through hMasaExtRend +static ivas_error ivas_masa_ext_rend_parambin_init( + input_masa *inputMasa /* i/o: MASA external renderer structure */ ) { - + DIRAC_DEC_BIN_HANDLE hDiracDecBin; + HRTFS_PARAMBIN_HANDLE hHrtfParambin; + int16_t nBins; + int32_t output_Fs; + RENDERER_TYPE renderer_type; + int16_t j, k, bin; + float binCenterFreq, tmpFloat; + ivas_error error; + float frequency_axis[CLDFB_NO_CHANNELS_MAX]; #ifdef SPLIT_REND_WITH_HEAD_ROT - return getSamplesInternal( hIvasRend, outAudio, NULL ); -#else - return getSamplesInternal( hIvasRend, outAudio ); + int16_t pos_idx; #endif -} + error = IVAS_ERR_OK; -#ifdef SPLIT_REND_WITH_HEAD_ROT -/*-------------------------------------------------------------------* - * IVAS_REND_GetSplitBinauralBitstream() - * - * - *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_GetSplitBinauralBitstream( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ - IVAS_REND_BitstreamBuffer *hBits /* o : buffer for output bitstream */ -) -{ - int16_t cldfb_in_flag; + hHrtfParambin = inputMasa->hMasaExtRend->hHrtfParambin; - cldfb_in_flag = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); - hIvasRend->splitRendEncBuffer.config.is_cldfb = cldfb_in_flag; + /* Set common variables and defaults */ + output_Fs = *( inputMasa->base.ctx.pOutSampleRate ); + nBins = inputMasa->hMasaExtRend->hSpatParamRendCom->num_freq_bands; + renderer_type = inputMasa->hMasaExtRend->renderer_type; - if ( hIvasRend->hRendererConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && - ( hIvasRend->hRendererConfig->split_rend_config.dof == 0 || hIvasRend->hRendererConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) ) - { - hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms * hIvasRend->num_subframes * (int16_t) ( hIvasRend->sampleRateOut / 1000 ); - } - else +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( pos_idx = 0; pos_idx < 1 /*st_ivas->hSplitBinRend.splitrend.multiBinPoseData.num_poses*/; pos_idx++ ) // Todo split rend MASA ext: Need to get split rend data here { - hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = (int16_t) ( hIvasRend->sampleRateOut / FRAMES_PER_SEC ); - } - hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel *= cldfb_in_flag ? 2 : 1; + hDiracDecBin = inputMasa->hMasaExtRend->hDiracDecBin[pos_idx]; +#else + hDiracDecBin = inputMasa->hMasaExtRend->hDiracDecBin; +#endif - /* hIvasRend->splitRendEncBuffer used for BINAURAL_SPLIT_CODED output - outAudio used for BINAURAL_SPLIT_PCM output */ - return getSamplesInternal( hIvasRend, outAudio, hBits ); -} + /* Init assumes that no reconfiguration is required in external renderer. Instead, free and rebuild whole rendering. */ + if ( ( hDiracDecBin = (DIRAC_DEC_BIN_HANDLE) malloc( sizeof( DIRAC_DEC_BIN_DATA ) ) ) == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for DirAC binaural handle " ); + } + hDiracDecBin->hTdDecorr = NULL; + hDiracDecBin->hReverb = NULL; + hDiracDecBin->h_freq_domain_decorr_ap_params = NULL; + hDiracDecBin->h_freq_domain_decorr_ap_state = NULL; + hDiracDecBin->hDiffuseDist = NULL; /* Not used in external renderer */ + hDiracDecBin->useTdDecorr = 0; /* Always use frequency domain decorrelator in external renderer */ -/*-------------------------------------------------------------------* - * IVAS_REND_GetSplitBinauralSamples() - * - * - *-------------------------------------------------------------------*/ + for ( j = 0; j < BINAURAL_CHANNELS; j++ ) + { + for ( k = 0; k < BINAURAL_CHANNELS + MAX_NUM_OBJECTS; k++ ) + { + set_zero( hDiracDecBin->processMtxRe[j][k], nBins ); + set_zero( hDiracDecBin->processMtxIm[j][k], nBins ); + } -ivas_error IVAS_REND_GetSplitBinauralSamples( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ - bool *needNewFrame ) -{ - ivas_error error; + for ( k = 0; k < BINAURAL_CHANNELS; k++ ) + { + set_zero( hDiracDecBin->processMtxDecRe[j][k], nBins ); + set_zero( hDiracDecBin->processMtxDecIm[j][k], nBins ); + } + set_zero( hDiracDecBin->ChEnePrev[j], nBins ); + set_zero( hDiracDecBin->ChEneOutPrev[j], nBins ); + } + set_zero( hDiracDecBin->ChCrossRePrev, nBins ); + set_zero( hDiracDecBin->ChCrossImPrev, nBins ); + set_zero( hDiracDecBin->ChCrossReOutPrev, nBins ); + set_zero( hDiracDecBin->ChCrossImOutPrev, nBins ); + hDiracDecBin->renderStereoOutputInsteadOfBinaural = 0; - if ( ( error = getSamplesInternal( hIvasRend, outAudio, NULL ) ) != IVAS_ERR_OK ) - { - return error; - } - *needNewFrame = hIvasRend->inputsSplitPost[0].numCachedSamples == 0; + for ( bin = 0; bin < nBins; bin++ ) + { + binCenterFreq = ( (float) bin + 0.5f ) / (float) nBins * ( (float) output_Fs / 2.0f ); + /* These formulas and values are from Christian Borss's publication for binaural diffuse field coherence */ + tmpFloat = max( 0.0f, 1.0f - binCenterFreq / 2700.0f ); + hDiracDecBin->diffuseFieldCoherence[bin] = tmpFloat * sinf( binCenterFreq * EVS_PI / 550.0f ) / ( binCenterFreq * EVS_PI / 550.0f ); + } - return IVAS_ERR_OK; -} + /* No SPAR in external renderer so set directive diffuse field coherence tables to zero */ + set_zero( hDiracDecBin->diffuseFieldCoherenceX, BINAURAL_COHERENCE_DIFFERENCE_BINS ); + set_zero( hDiracDecBin->diffuseFieldCoherenceY, BINAURAL_COHERENCE_DIFFERENCE_BINS ); + set_zero( hDiracDecBin->diffuseFieldCoherenceZ, BINAURAL_COHERENCE_DIFFERENCE_BINS ); + + if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC ) /* Indication of binaural rendering without room effect */ + { + set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); + hDiracDecBin->hReverb = NULL; + } + else if ( renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) /* Indication of binaural rendering with room effect */ + { + mvr2r( hHrtfParambin->parametricEarlyPartEneCorrection, hDiracDecBin->earlyPartEneCorrection, nBins ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( hDiracDecBin->hReverb == NULL && pos_idx == 0 ) /* open reverb only for the main direction */ +#else + if ( hDiracDecBin->hReverb == NULL ) #endif + { + /* Todo Philips: Room acoustics should be passed here once the underlying part works. In this case, it probably should come from render context or somewhere else suitable. */ + if ( ( error = ivas_binaural_reverb_open_parambin( &hDiracDecBin->hReverb, nBins, CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, NULL, output_Fs, hHrtfParambin ) ) != IVAS_ERR_OK ) + { + return error; + } + } + } + else if ( renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + set_f( hDiracDecBin->earlyPartEneCorrection, 1.0f, CLDFB_NO_CHANNELS_MAX ); + hDiracDecBin->hReverb = NULL; + hDiracDecBin->renderStereoOutputInsteadOfBinaural = 1; + } + else /* Not valid renderer type for this renderer */ + { + assert( false ); + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( pos_idx == 0 ) /* open decorrelator only for the main direction */ + { +#endif + /* Always open frequency domain decorrelator */ + ivas_dirac_dec_get_frequency_axis( frequency_axis, output_Fs, nBins ); + if ( ( error = ivas_dirac_dec_decorr_open( &( hDiracDecBin->h_freq_domain_decorr_ap_params ), + &( hDiracDecBin->h_freq_domain_decorr_ap_state ), + nBins, + BINAURAL_CHANNELS, + BINAURAL_CHANNELS, + DIRAC_SYNTHESIS_PSD_LS, + frequency_axis, + BINAURAL_CHANNELS, + output_Fs ) ) != IVAS_ERR_OK ) + { + return error; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif + /* External renderer uses constant regularization factor */ + hDiracDecBin->reqularizationFactor = 0.4f; -/*-------------------------------------------------------------------* - * IVAS_REND_Close() - * - * - *-------------------------------------------------------------------*/ +#ifdef SPLIT_REND_WITH_HEAD_ROT + inputMasa->hMasaExtRend->hDiracDecBin[pos_idx] = hDiracDecBin; + } +#else + inputMasa->hMasaExtRend->hDiracDecBin = hDiracDecBin; +#endif -void IVAS_REND_Close( - IVAS_REND_HANDLE *phIvasRend /* i/o: Pointer to renderer handle */ -) + return error; +} + +static ivas_error initMasaExtRenderer( + input_masa *inputMasa, + const AUDIO_CONFIG outConfig ) { - uint16_t i; - IVAS_REND_HANDLE hIvasRend; + int16_t i; + ivas_error error; + MASA_EXT_REND_HANDLE hMasaExtRend; - /* Validate function arguments */ - if ( phIvasRend == NULL || *phIvasRend == NULL ) - { - return; - } - hIvasRend = *phIvasRend; + error = IVAS_ERR_OK; - if ( hIvasRend->efapOutWrapper.hEfap != NULL ) + if ( ( hMasaExtRend = (MASA_EXT_REND_HANDLE) malloc( sizeof( MASA_EXT_REND_DATA ) ) ) == NULL ) { - efap_free_data( &hIvasRend->efapOutWrapper.hEfap ); + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for MASA external renderer structure\n" ) ); } - /* clear inputs */ - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) - { - clearInputIsm( &hIvasRend->inputsIsm[i] ); - } - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) + inputMasa->hMasaExtRend = hMasaExtRend; + + /* Default init */ + hMasaExtRend->renderer_type = RENDERER_DISABLE; + hMasaExtRend->hDirACRend = NULL; + hMasaExtRend->hSpatParamRendCom = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) { - clearInputMc( &hIvasRend->inputsMc[i] ); + hMasaExtRend->hDiracDecBin[i] = NULL; } - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; ++i ) +#else + hMasaExtRend->hDiracDecBin = NULL; +#endif + hMasaExtRend->hReverb = NULL; + hMasaExtRend->hHrtfParambin = NULL; + hMasaExtRend->hVBAPdata = NULL; + hMasaExtRend->hoa_dec_mtx = NULL; + hMasaExtRend->current_subframe = 0; + + if ( ( error = getAudioConfigNumChannels( inputMasa->base.inConfig, &hMasaExtRend->nchan_input ) ) != IVAS_ERR_OK ) { - clearInputSba( &hIvasRend->inputsSba[i] ); + return error; } - for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + + if ( ( error = getAudioConfigNumChannels( outConfig, &hMasaExtRend->nchan_output ) ) != IVAS_ERR_OK ) { - clearInputMasa( &hIvasRend->inputsMasa[i] ); + return error; } -#ifdef SPLIT_REND_WITH_HEAD_ROT - for ( i = 0; i < RENDERER_MAX_BIN_INPUTS; ++i ) + + switch ( outConfig ) { - clearInputSplitRend( &hIvasRend->inputsSplitPost[i] ); - } -#endif + case IVAS_AUDIO_CONFIG_MONO: + if ( inputMasa->base.inConfig == IVAS_AUDIO_CONFIG_MASA2 ) + { + hMasaExtRend->renderer_type = RENDERER_DIRAC; + } + else + { + /* 1TC MASA to mono does not need rendering. */ + hMasaExtRend->renderer_type = RENDERER_DISABLE; + } + break; - /* clear Config. Renderer */ - ivas_render_config_close( &( hIvasRend->hRendererConfig ) ); + case IVAS_AUDIO_CONFIG_STEREO: + hMasaExtRend->renderer_type = RENDERER_STEREO_PARAMETRIC; + break; - ivas_limiter_close( &hIvasRend->hLimiter ); + case IVAS_AUDIO_CONFIG_5_1: + case IVAS_AUDIO_CONFIG_7_1: + case IVAS_AUDIO_CONFIG_5_1_2: + case IVAS_AUDIO_CONFIG_5_1_4: + case IVAS_AUDIO_CONFIG_7_1_4: + case IVAS_AUDIO_CONFIG_LS_CUSTOM: + case IVAS_AUDIO_CONFIG_FOA: + case IVAS_AUDIO_CONFIG_HOA2: + case IVAS_AUDIO_CONFIG_HOA3: + hMasaExtRend->renderer_type = RENDERER_DIRAC; + break; + case IVAS_AUDIO_CONFIG_BINAURAL: #ifdef SPLIT_REND_WITH_HEAD_ROT - /* Split binaural rendering */ - closeSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer ); + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: #endif + hMasaExtRend->renderer_type = RENDERER_BINAURAL_PARAMETRIC; + break; - closeHeadRotation( hIvasRend ); - - ivas_external_orientation_close( &hIvasRend->hExternalOrientationData ); - ivas_combined_orientation_close( &hIvasRend->hCombinedOrientationData ); - - free( hIvasRend ); - *phIvasRend = NULL; + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: + case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: + hMasaExtRend->renderer_type = RENDERER_BINAURAL_PARAMETRIC_ROOM; + break; - return; -} + default: + return ( IVAS_ERROR( IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED, "Wrong output config for MASA input in external renderer\n" ) ); + } + if ( hMasaExtRend->renderer_type != RENDERER_DISABLE ) + { + int16_t subframe; -#ifdef SPLIT_REND_WITH_HEAD_ROT -/*-------------------------------------------------------------------* - * IVAS_REND_openCldfb() - * - * - *-------------------------------------------------------------------*/ + if ( ( error = ivas_spat_hSpatParamRendCom_config( &hMasaExtRend->hSpatParamRendCom, DIRAC_OPEN, 0, MASA_FORMAT, MC_MODE_NONE, *( inputMasa->base.ctx.pOutSampleRate ), 0, 1 ) ) != IVAS_ERR_OK ) + { + return error; + } -ivas_error IVAS_REND_openCldfb( - IVAS_CLDFB_FILTER_BANK_HANDLE cldfbAna[IVAS_MAX_INPUT_CHANNELS], - IVAS_CLDFB_FILTER_BANK_HANDLE cldfbSyn[IVAS_MAX_OUTPUT_CHANNELS], - const int16_t num_in_chs, - const int16_t num_out_chs, - const int32_t output_Fs ) -{ - int16_t n; - ivas_error error; + /* Simple population of the metadata index map as no adaptation is present */ + set_s( hMasaExtRend->hSpatParamRendCom->render_to_md_map, 0, MAX_JBM_SUBFRAMES_5MS * JBM_CLDFB_SLOTS_IN_SUBFRAME ); + for ( subframe = 0; subframe < MAX_PARAM_SPATIAL_SUBFRAMES; subframe++ ) + { + hMasaExtRend->hSpatParamRendCom->render_to_md_map[subframe] = subframe; + } + hMasaExtRend->hSpatParamRendCom->subframes_rendered = 0; + } - for ( n = 0; n < num_in_chs; n++ ) + if ( hMasaExtRend->renderer_type == RENDERER_DIRAC ) { - if ( ( error = openCldfb( &( cldfbAna[n] ), CLDFB_ANALYSIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_masa_ext_rend_dirac_rend_init( inputMasa ) ) != IVAS_ERR_OK ) { return error; } } - for ( ; n < IVAS_MAX_INPUT_CHANNELS; n++ ) - { - cldfbAna[n] = NULL; - } - for ( n = 0; n < num_out_chs; n++ ) + if ( hMasaExtRend->renderer_type == RENDERER_BINAURAL_PARAMETRIC || hMasaExtRend->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || hMasaExtRend->renderer_type == RENDERER_STEREO_PARAMETRIC ) { - if ( ( error = openCldfb( &( cldfbSyn[n] ), CLDFB_SYNTHESIS, output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + if ( hMasaExtRend->renderer_type != RENDERER_STEREO_PARAMETRIC ) + { + if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &inputMasa->hMasaExtRend->hHrtfParambin ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + if ( ( error = ivas_masa_ext_rend_parambin_init( inputMasa ) ) != IVAS_ERR_OK ) { return error; } } - for ( ; n < IVAS_MAX_OUTPUT_CHANNELS; n++ ) - { - cldfbSyn[n] = NULL; - } - - return IVAS_ERR_OK; -} - - -/*-------------------------------------------------------------------* - * IVAS_REND_closeCldfb() - * - * - *-------------------------------------------------------------------*/ - -void IVAS_REND_closeCldfb( - IVAS_CLDFB_FILTER_BANK_HANDLE cldfbAna[IVAS_MAX_INPUT_CHANNELS], - IVAS_CLDFB_FILTER_BANK_HANDLE cldfbSyn[IVAS_MAX_OUTPUT_CHANNELS] ) -{ - int16_t n; - for ( n = 0; n < IVAS_MAX_INPUT_CHANNELS; n++ ) + /* Init CLDFB for analysis & synthesis if renderer is used */ + if ( hMasaExtRend->renderer_type != RENDERER_DISABLE ) { - if ( cldfbAna[n] != NULL ) + for ( i = 0; i < hMasaExtRend->nchan_input; i++ ) { - deleteCldfb( &( cldfbAna[n] ) ); - cldfbAna[n] = NULL; + if ( ( error = openCldfb( &( hMasaExtRend->cldfbAnaRend[i] ), CLDFB_ANALYSIS, *inputMasa->base.ctx.pOutSampleRate, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MASA_MAX_TRANSPORT_CHANNELS; i++ ) + { + hMasaExtRend->cldfbAnaRend[i] = NULL; } - } - for ( n = 0; n < IVAS_MAX_OUTPUT_CHANNELS; n++ ) - { - if ( cldfbSyn[n] != NULL ) + for ( i = 0; i < hMasaExtRend->nchan_output; i++ ) { - deleteCldfb( &( cldfbSyn[n] ) ); - cldfbSyn[n] = NULL; + if ( ( error = openCldfb( &( hMasaExtRend->cldfbSynRend[i] ), CLDFB_SYNTHESIS, *inputMasa->base.ctx.pOutSampleRate, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + hMasaExtRend->cldfbSynRend[i] = NULL; } } - return; -} + inputMasa->hMasaExtRend = hMasaExtRend; + return IVAS_ERR_OK; +} -/*-------------------------------------------------------------------* - * IVAS_REND_cldfbSynthesis_wrapper() - * - * - *-------------------------------------------------------------------*/ -void IVAS_REND_cldfbAnalysis_ts_wrapper( - const float *timeIn, /* i : time buffer */ - float realBuffer[IVAS_CLDFB_NO_CHANNELS_MAX], /* o : real value buffer */ - float imagBuffer[IVAS_CLDFB_NO_CHANNELS_MAX], /* o : imag value buffer */ - const int16_t samplesToProcess, /* i : samples to process */ - IVAS_CLDFB_FILTER_BANK_HANDLE h_cldfb /* i : filterbank state */ -) +static void freeMasaExtRenderer( + MASA_EXT_REND_HANDLE *hMasaExtRendOut ) { - cldfbAnalysis_ts( timeIn, realBuffer, imagBuffer, samplesToProcess, h_cldfb ); + MASA_EXT_REND_HANDLE hMasaExtRend; + int16_t i; - return; -} + if ( hMasaExtRendOut == NULL || *hMasaExtRendOut == NULL ) + { + return; + } + hMasaExtRend = *hMasaExtRendOut; -/*-------------------------------------------------------------------* - * IVAS_REND_cldfbSynthesis_wrapper() - * - * - *-------------------------------------------------------------------*/ + if ( hMasaExtRend->hDirACRend != NULL ) + { + ivas_dirac_rend_close( &hMasaExtRend->hDirACRend ); + } -void IVAS_REND_cldfbSynthesis_wrapper( - float **realBuffer, /* i : real values */ - float **imagBuffer, /* i : imag values */ - float *timeOut, /* o : output time domain samples */ - const int16_t samplesToProcess, /* i : number of processed samples */ - IVAS_CLDFB_FILTER_BANK_HANDLE h_cldfb /* i : filter bank state */ -) -{ - cldfbSynthesis( realBuffer, imagBuffer, timeOut, samplesToProcess, h_cldfb ); + if ( hMasaExtRend->hSpatParamRendCom != NULL ) + { + ivas_spat_hSpatParamRendCom_close( &hMasaExtRend->hSpatParamRendCom ); + } - return; -} +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) + { + if ( hMasaExtRend->hDiracDecBin[i] != NULL ) + { + ivas_dirac_dec_close_binaural_data( &hMasaExtRend->hDiracDecBin[i] ); + } + } + +#else + if ( hMasaExtRend->hDiracDecBin != NULL ) + { + ivas_dirac_dec_close_binaural_data( &hMasaExtRend->hDiracDecBin ); + } #endif + if ( hMasaExtRend->hReverb != NULL ) + { + ivas_binaural_reverb_close( &hMasaExtRend->hReverb ); + } -#ifdef DEBUGGING -/*-------------------------------------------------------------------* - * IVAS_REND_GetNoCLipping() - * - * - *-------------------------------------------------------------------*/ + if ( hMasaExtRend->hHrtfParambin != NULL ) + { + ivas_HRTF_parambin_binary_close( &hMasaExtRend->hHrtfParambin ); + } -int32_t IVAS_REND_GetNoCLipping( - IVAS_REND_CONST_HANDLE hIvasRend ) -{ - return hIvasRend->numClipping; -} + if ( hMasaExtRend->hVBAPdata != NULL ) + { + vbap_free_data( &hMasaExtRend->hVBAPdata ); + } -int32_t IVAS_REND_GetCntFramesLimited( - IVAS_REND_CONST_HANDLE hIvasRend ) -{ - if ( hIvasRend->hLimiter == NULL ) + if ( hMasaExtRend->hoa_dec_mtx != NULL ) { - return 0; + free( hMasaExtRend->hoa_dec_mtx ); } - return hIvasRend->hLimiter->cnt_frames_limited; + for ( i = 0; i < MASA_MAX_TRANSPORT_CHANNELS; i++ ) + { + if ( hMasaExtRend->cldfbAnaRend[i] != NULL ) + { + deleteCldfb( &hMasaExtRend->cldfbAnaRend[i] ); + } + } + + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + if ( hMasaExtRend->cldfbSynRend[i] != NULL ) + { + deleteCldfb( &hMasaExtRend->cldfbSynRend[i] ); + } + } + + free( hMasaExtRend ); + *hMasaExtRendOut = NULL; + + return; } -#endif -- GitLab From ec0c2c3efa6e4eaa5698ff84c3abd4c037ecb3f1 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Thu, 12 Oct 2023 08:47:27 +0300 Subject: [PATCH 2/9] Fix various warnings. --- lib_rend/ivas_dirac_dec_binaural_functions.c | 2 +- lib_rend/ivas_dirac_rend.c | 7 +++---- lib_rend/ivas_prot_rend.h | 1 - lib_rend/lib_rend.c | 4 ++-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index b80467ace5..1b912d686f 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -3657,7 +3657,7 @@ static void ivas_masa_ext_rend_parambin_internal( ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, - nchanSeparateChannels, NULL ); + 0, NULL ); /* re-use reverb and decorr from main direction for the sides */ diff --git a/lib_rend/ivas_dirac_rend.c b/lib_rend/ivas_dirac_rend.c index b3cf98df4a..8ac646a93b 100644 --- a/lib_rend/ivas_dirac_rend.c +++ b/lib_rend/ivas_dirac_rend.c @@ -2462,10 +2462,9 @@ static void ivas_masa_ext_dirac_render_sf( } void ivas_masa_ext_dirac_render( - MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: IVAS decoder structure */ - float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ - const int16_t nchan_transport, /* i : number of transport channels */ - const int16_t num_subframes /* i : number of subframes to render */ + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA renderer structure */ + float *output_f[], /* i/o: input/output signals in time domain */ + const int16_t num_subframes /* i : number of subframes to render */ ) { int16_t subframe_idx; diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 8880d2a5cc..b1063ccf33 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -517,7 +517,6 @@ void ivas_dirac_deallocate_parameters( void ivas_masa_ext_dirac_render( MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA renderer structure */ float *output_f[], /* i/o: input/output signals in time domain */ - const int16_t nchan_transport, /* i : number of transport channels */ const int16_t num_subframes /* i : number of subframes to render */ ); diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index aef5c7b237..b9ea0bae43 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -7779,13 +7779,13 @@ static ivas_error renderInputMasa( copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); - num_subframes = masaInput->base.inputBuffer.config.numSamplesPerChannel / ( *masaInput->base.ctx.pOutSampleRate / ( IVAS_NUM_FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES ) ); + num_subframes = (int16_t) ( masaInput->base.inputBuffer.config.numSamplesPerChannel / ( *masaInput->base.ctx.pOutSampleRate / ( IVAS_NUM_FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES ) ) ); switch ( masaInput->hMasaExtRend->renderer_type ) { case RENDERER_DIRAC: copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom ); - ivas_masa_ext_dirac_render( masaInput->hMasaExtRend, tmpBuffer, masaInput->hMasaExtRend->nchan_input, num_subframes ); + ivas_masa_ext_dirac_render( masaInput->hMasaExtRend, tmpBuffer, num_subframes ); break; case RENDERER_STEREO_PARAMETRIC: case RENDERER_BINAURAL_PARAMETRIC: -- GitLab From 4f264d2c1f1c536a23f9d5a475fe3f999c9637a5 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 20 Oct 2023 15:46:59 +0300 Subject: [PATCH 3/9] Fix various sanitizer problems. --- lib_rend/ivas_dirac_rend.c | 8 ++++++++ lib_rend/lib_rend.c | 31 ++++++++++++++++++------------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/lib_rend/ivas_dirac_rend.c b/lib_rend/ivas_dirac_rend.c index 8ac646a93b..43681502d6 100644 --- a/lib_rend/ivas_dirac_rend.c +++ b/lib_rend/ivas_dirac_rend.c @@ -2238,6 +2238,14 @@ static void ivas_masa_ext_dirac_render_sf( hMasaExtRend->cldfbAnaRend[ch] ); } + + if ( nchan_transport == 1 ) + { + /* Need to set second CLDFB channel to zero as further processing assumes CNA content in it */ + set_zero( Cldfb_RealBuffer[1][0], hSpatParamRendCom->num_freq_bands ); + set_zero( Cldfb_ImagBuffer[1][0], hSpatParamRendCom->num_freq_bands ); + } + /*-----------------------------------------------------------------* * protoype signal computation *-----------------------------------------------------------------*/ diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index b9ea0bae43..7b186e68c0 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -7557,7 +7557,8 @@ static ivas_error renderActiveInputsSba( static void copyMasaMetadataToDiracRenderer( MASA_METADATA_FRAME *meta, - SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom ) + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, + const int16_t maxBin ) { int16_t band, sf, bin; int16_t meta_write_index; @@ -7571,7 +7572,7 @@ static void copyMasaMetadataToDiracRenderer( for ( band = 0; band < MASA_MAXIMUM_CODING_SUBBANDS; band++ ) { - for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1]; bin++ ) + for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1] && bin < maxBin; bin++ ) { hSpatParamRendCom->azimuth[meta_write_index][bin] = (int16_t) meta->directional_meta[0].azimuth[sf][band]; hSpatParamRendCom->elevation[meta_write_index][bin] = (int16_t) meta->directional_meta[0].elevation[sf][band]; @@ -7732,6 +7733,7 @@ static ivas_error renderInputMasa( { IVAS_REND_AudioBuffer inAudio; int16_t ch; + int16_t maxBin; float *tmpBuffer[MAX_OUTPUT_CHANNELS]; float tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -7759,6 +7761,7 @@ static ivas_error renderInputMasa( /* Apply input gain to new audio */ v_multc( inAudio.data, masaInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + maxBin = (int16_t) ( *masaInput->base.ctx.pOutSampleRate * INV_CLDFB_BANDWIDTH ); /* Todo split rend MASA ext: Need to add split renderer path to below. Options are: * - Join together with binaural renderer path @@ -7784,13 +7787,13 @@ static ivas_error renderInputMasa( switch ( masaInput->hMasaExtRend->renderer_type ) { case RENDERER_DIRAC: - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); ivas_masa_ext_dirac_render( masaInput->hMasaExtRend, tmpBuffer, num_subframes ); break; case RENDERER_STEREO_PARAMETRIC: case RENDERER_BINAURAL_PARAMETRIC: case RENDERER_BINAURAL_PARAMETRIC_ROOM: - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes ); break; case RENDERER_DISABLE: @@ -9161,7 +9164,17 @@ static ivas_error initMasaExtRenderer( } } - /* Init CLDFB for analysis & synthesis if renderer is used */ + /* Init CLDFB for analysis & synthesis if renderer is used. Otherwise, NULL. */ + for ( i = 0; i < MASA_MAX_TRANSPORT_CHANNELS; i++ ) + { + hMasaExtRend->cldfbAnaRend[i] = NULL; + } + + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + hMasaExtRend->cldfbSynRend[i] = NULL; + } + if ( hMasaExtRend->renderer_type != RENDERER_DISABLE ) { for ( i = 0; i < hMasaExtRend->nchan_input; i++ ) @@ -9171,10 +9184,6 @@ static ivas_error initMasaExtRenderer( return error; } } - for ( ; i < MASA_MAX_TRANSPORT_CHANNELS; i++ ) - { - hMasaExtRend->cldfbAnaRend[i] = NULL; - } for ( i = 0; i < hMasaExtRend->nchan_output; i++ ) { @@ -9183,10 +9192,6 @@ static ivas_error initMasaExtRenderer( return error; } } - for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) - { - hMasaExtRend->cldfbSynRend[i] = NULL; - } } inputMasa->hMasaExtRend = hMasaExtRend; -- GitLab From 401c8e7bc000d21e48b1e33f0ca8180d3ecbdeb4 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Tue, 24 Oct 2023 14:16:21 +0200 Subject: [PATCH 4/9] split rendering support for MASA external renderer --- lib_rend/ivas_dirac_dec_binaural_functions.c | 51 ++++++++----- lib_rend/ivas_prot_rend.h | 15 +++- lib_rend/lib_rend.c | 78 ++++++++++++++------ 3 files changed, 98 insertions(+), 46 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 20f6dbabc5..ff753a2b89 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -3476,7 +3476,14 @@ static void ivas_masa_ext_rend_parambin_internal( MASA_EXT_REND_HANDLE hMasaExtRend, COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, float *output_f[], +#ifdef SPLIT_REND_WITH_HEAD_ROT + const int16_t subframe, + const SPLIT_REND_WRAPPER *hSplitRendWrapper, + float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX] ) +#else const int16_t subframe ) +#endif { DIRAC_DEC_BIN_HANDLE hDiracDecBin; SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; @@ -3492,7 +3499,7 @@ static void ivas_masa_ext_rend_parambin_internal( #ifdef SPLIT_REND_WITH_HEAD_ROT int16_t pos_idx; - MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; + const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; float tmp_Cldfb_out_re[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float tmp_Cldfb_out_im[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; /* these allow re-using the reverb and freq-domain decorrelator signals from ivas_dirac_dec_binaural_process_output() in split rendering for the side renderings */ @@ -3624,14 +3631,11 @@ static void ivas_masa_ext_rend_parambin_internal( #endif #ifdef SPLIT_REND_WITH_HEAD_ROT - // pMultiBinPoseData = &st_ivas->hSplitBinRend.splitrend.multiBinPoseData; // Todo split rend MASA ext: Need to get split rend structure here - -#ifdef FIX_658_SPLIT_REND_MASA - if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) -#else - if ( pMultiBinPoseData != NULL && pMultiBinPoseData->num_poses > 1 ) -#endif + pMultiBinPoseData = NULL; + if ( hSplitRendWrapper != NULL ) { + pMultiBinPoseData = &(hSplitRendWrapper->multiBinPoseData); + ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, hMasaExtRend->cldfbSynRend, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, reverbRe, reverbIm, decorrRe, decorrIm, 1 ); @@ -3640,9 +3644,8 @@ static void ivas_masa_ext_rend_parambin_internal( { for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) { - // Todo split rend MASA ext: Need to get split rend CLDFB output structure here - // mvr2r( tmp_Cldfb_out_re[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); - // mvr2r( tmp_Cldfb_out_im[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + mvr2r( tmp_Cldfb_out_re[ch][i], Cldfb_Out_Real[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + mvr2r( tmp_Cldfb_out_im[ch][i], Cldfb_Out_Imag[ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); } } } @@ -3710,9 +3713,8 @@ static void ivas_masa_ext_rend_parambin_internal( { for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++ ) { - // Todo split rend MASA ext: Need to get split rend structure here - // mvr2r( tmp_Cldfb_out_re[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); - // mvr2r( tmp_Cldfb_out_im[ch][i], st_ivas->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + mvr2r( tmp_Cldfb_out_re[ch][i], Cldfb_Out_Real[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); + mvr2r( tmp_Cldfb_out_im[ch][i], Cldfb_Out_Imag[pos_idx * BINAURAL_CHANNELS + ch][subframe * CLDFB_SLOTS_PER_SUBFRAME + i], CLDFB_NO_CHANNELS_MAX ); } } @@ -3730,13 +3732,18 @@ static void ivas_masa_ext_rend_parambin_internal( } -// Todo split rend MASA ext: If needed, make a copy of this function for split rendering purposes void ivas_masa_ext_rend_parambin_render( - MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ - float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ - const int16_t num_subframes /* i : number of subframes to render */ -) + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ + float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + const int16_t num_subframes, /* i : number of subframes to render */ + const SPLIT_REND_WRAPPER *hSplitRendWrapper, /* i : split rendering orientation data */ + float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* o : rendered orientations for split rend. real part of cldfb */ + float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX] ) /* o : rendered orientations for split rend. imag part of cldfb */ +#else + const int16_t num_subframes ) /* i : number of subframes to render */ +#endif { int16_t subframe; SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; @@ -3757,7 +3764,11 @@ void ivas_masa_ext_rend_parambin_render( int16_t n_samples_sf = hSpatParamRendCom->slot_size * CLDFB_SLOTS_PER_SUBFRAME; hSpatParamRendCom->slots_rendered = 0; +#ifdef SPLIT_REND_WITH_HEAD_ROT + ivas_masa_ext_rend_parambin_internal( hMasaExtRend, hCombinedOrientationData, p_output, hSpatParamRendCom->dirac_read_idx, hSplitRendWrapper, Cldfb_Out_Real, Cldfb_Out_Imag ); +#else ivas_masa_ext_rend_parambin_internal( hMasaExtRend, hCombinedOrientationData, p_output, hSpatParamRendCom->dirac_read_idx ); +#endif for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 3995dbd359..ac3999c442 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -201,10 +201,17 @@ void ivas_dirac_dec_binaural_render( ); void ivas_masa_ext_rend_parambin_render( - MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ - float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ - const int16_t num_subframes /* i : number of subframes to render */ + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ + float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + const int16_t num_subframes, /* i : number of subframes to render */ + const SPLIT_REND_WRAPPER *hSplitRendWrapper, /* i : split rendering orientation data */ + float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* o : rendered orientations for split rend. real part of cldfb */ + float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX] /* o : rendered orientations for split rend. imag part of cldfb */ +#else + const int16_t num_subframes /* i : number of subframes to render */ +#endif ); ivas_error ivas_dirac_dec_init_binaural_data( diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index fa44278b8c..53d7f77079 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -7737,6 +7737,8 @@ static ivas_error renderInputMasa( float tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; #ifdef SPLIT_REND_WITH_HEAD_ROT int16_t cldfb2tdSampleFact; + float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; #endif if ( !masaInput->metadataHasBeenFed ) @@ -7767,9 +7769,6 @@ static ivas_error renderInputMasa( ivas_combined_orientation_set_to_start_index( *( masaInput->base.ctx.pCombinedOrientationData ) ); #endif - /* Todo split rend MASA ext: Need to add split renderer path to below. Options are: - * - Join together with binaural renderer path - * - Add a separate path for split renderer processing that is tuned for it */ if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) { /* MASA prerendering path for MASA -> MASA */ @@ -7777,8 +7776,8 @@ static ivas_error renderInputMasa( } else { - int16_t num_subframes; /* MASA external renderer -> other formats */ + int16_t num_subframes; for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) { tmpBuffer[ch] = tmpBuffer_buff[ch]; @@ -7788,25 +7787,61 @@ static ivas_error renderInputMasa( num_subframes = (int16_t) ( masaInput->base.inputBuffer.config.numSamplesPerChannel / ( *masaInput->base.ctx.pOutSampleRate / ( IVAS_NUM_FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES ) ) ); - switch ( masaInput->hMasaExtRend->renderer_type ) +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) { - case RENDERER_DIRAC: - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); - ivas_masa_ext_dirac_render( masaInput->hMasaExtRend, tmpBuffer, num_subframes ); - break; - case RENDERER_STEREO_PARAMETRIC: - case RENDERER_BINAURAL_PARAMETRIC: - case RENDERER_BINAURAL_PARAMETRIC_ROOM: - copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); - ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes ); - break; - case RENDERER_DISABLE: - break; /* This happens for 1TC MASA to MONO where we just copy input transport to output */ - default: - return ( IVAS_ERROR( IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED, "Wrong output config for MASA input in external renderer\n" ) ); + /* split rendering. use the combined of the first subframe in all subframes */ + int16_t sf, i, j; + COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; + pCombinedOrientationData = *masaInput->base.ctx.pCombinedOrientationData; + for ( sf = 1; sf < pCombinedOrientationData->num_subframes; sf++ ) + { + pCombinedOrientationData->Quaternions[sf] = pCombinedOrientationData->Quaternions[0]; + for ( i = 0; i < 3; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + pCombinedOrientationData->Rmat[sf][i][j] = pCombinedOrientationData->Rmat[0][i][j]; + } + } + } + + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); + + ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes, masaInput->base.ctx.pSplitRendWrapper, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural ); + + accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); } + else + { + /* non-split path */ +#endif + switch ( masaInput->hMasaExtRend->renderer_type ) + { + case RENDERER_DIRAC: + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); + ivas_masa_ext_dirac_render( masaInput->hMasaExtRend, tmpBuffer, num_subframes ); + break; + case RENDERER_STEREO_PARAMETRIC: + case RENDERER_BINAURAL_PARAMETRIC: + case RENDERER_BINAURAL_PARAMETRIC_ROOM: + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->hMasaExtRend->hSpatParamRendCom, maxBin ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes, NULL, NULL, NULL ); +#else + ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes ); +#endif + break; + case RENDERER_DISABLE: + break; /* This happens for 1TC MASA to MONO where we just copy input transport to output */ + default: + return ( IVAS_ERROR( IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED, "Wrong output config for MASA input in external renderer\n" ) ); + } - accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); + accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif } return IVAS_ERR_OK; @@ -8889,7 +8924,6 @@ static ivas_error ivas_masa_ext_rend_dirac_rend_init( return error; } -// Todo split rend MASA ext: Add split renderer data here somehow. Probably through hMasaExtRend static ivas_error ivas_masa_ext_rend_parambin_init( input_masa *inputMasa /* i/o: MASA external renderer structure */ ) @@ -8917,7 +8951,7 @@ static ivas_error ivas_masa_ext_rend_parambin_init( renderer_type = inputMasa->hMasaExtRend->renderer_type; #ifdef SPLIT_REND_WITH_HEAD_ROT - for ( pos_idx = 0; pos_idx < 1 /*st_ivas->hSplitBinRend.splitrend.multiBinPoseData.num_poses*/; pos_idx++ ) // Todo split rend MASA ext: Need to get split rend data here + for ( pos_idx = 0; pos_idx < inputMasa->base.ctx.pSplitRendWrapper->multiBinPoseData.num_poses; pos_idx++ ) { hDiracDecBin = inputMasa->hMasaExtRend->hDiracDecBin[pos_idx]; #else -- GitLab From a104dd11d61990540560b4127e64923e315be6ee Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Tue, 24 Oct 2023 14:33:46 +0200 Subject: [PATCH 5/9] clang formatting --- lib_rend/ivas_dirac_dec_binaural_functions.c | 10 +++++----- lib_rend/lib_rend.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index ff753a2b89..21f2c86654 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -3634,7 +3634,7 @@ static void ivas_masa_ext_rend_parambin_internal( pMultiBinPoseData = NULL; if ( hSplitRendWrapper != NULL ) { - pMultiBinPoseData = &(hSplitRendWrapper->multiBinPoseData); + pMultiBinPoseData = &( hSplitRendWrapper->multiBinPoseData ); ivas_dirac_dec_binaural_process_output( hDiracDecBin, hSpatParamRendCom, hMasaExtRend->cldfbSynRend, output_f, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, max_band_decorr, numInChannels, config_data.processReverb, subframe, tmp_Cldfb_out_re, tmp_Cldfb_out_im, @@ -3733,16 +3733,16 @@ static void ivas_masa_ext_rend_parambin_internal( void ivas_masa_ext_rend_parambin_render( - MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ - float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ + MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ + float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ #ifdef SPLIT_REND_WITH_HEAD_ROT const int16_t num_subframes, /* i : number of subframes to render */ const SPLIT_REND_WRAPPER *hSplitRendWrapper, /* i : split rendering orientation data */ float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* o : rendered orientations for split rend. real part of cldfb */ float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX] ) /* o : rendered orientations for split rend. imag part of cldfb */ #else - const int16_t num_subframes ) /* i : number of subframes to render */ + const int16_t num_subframes ) /* i : number of subframes to render */ #endif { int16_t subframe; diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 53d7f77079..cabe81444f 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -7829,7 +7829,7 @@ static ivas_error renderInputMasa( #ifdef SPLIT_REND_WITH_HEAD_ROT ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes, NULL, NULL, NULL ); #else - ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes ); + ivas_masa_ext_rend_parambin_render( masaInput->hMasaExtRend, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, num_subframes ); #endif break; case RENDERER_DISABLE: -- GitLab From 9139be351b4231d5fce5f8bdbf84a8664a91ef4a Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Wed, 25 Oct 2023 08:51:12 +0200 Subject: [PATCH 6/9] output sampling rate verification for split rendering in general --- lib_rend/lib_rend.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index cabe81444f..e918c21609 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -640,6 +640,14 @@ static ivas_error validateOutputSampleRate( /* If no binaural rendering, any sampling rate is supported */ return IVAS_ERR_OK; } +#ifdef SPLIT_REND_WITH_HEAD_ROT + else if ( ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) && sampleRate != 48000 ) + { + return IVAS_ERROR( IVAS_ERR_INVALID_SAMPLING_RATE, "Error: Only 48kHz output sampling rate is supported for split rendering." ); + } + else + { +#endif /* Otherwise rendering to binaural, support the same set as IVAS decoder */ switch ( sampleRate ) @@ -652,6 +660,9 @@ static ivas_error validateOutputSampleRate( } return IVAS_ERR_INVALID_SAMPLING_RATE; +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif } -- GitLab From 0ed039fa99d3774686884036405f58b8bebce1a2 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Wed, 25 Oct 2023 09:21:17 +0200 Subject: [PATCH 7/9] remove unused masa_ext_rend_data structure member. comment clean-up. --- lib_rend/ivas_dirac_dec_binaural_functions.c | 1 + lib_rend/ivas_dirac_rend.c | 6 +++--- lib_rend/ivas_stat_rend.h | 2 -- lib_rend/lib_rend.c | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 21f2c86654..0bbfb2c85b 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -3502,6 +3502,7 @@ static void ivas_masa_ext_rend_parambin_internal( const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; float tmp_Cldfb_out_re[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float tmp_Cldfb_out_im[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + /* these allow re-using the reverb and freq-domain decorrelator signals from ivas_dirac_dec_binaural_process_output() in split rendering for the side renderings */ float reverbRe[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; float reverbIm[BINAURAL_CHANNELS][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; diff --git a/lib_rend/ivas_dirac_rend.c b/lib_rend/ivas_dirac_rend.c index 43681502d6..888432139d 100644 --- a/lib_rend/ivas_dirac_rend.c +++ b/lib_rend/ivas_dirac_rend.c @@ -2104,7 +2104,7 @@ static void ivas_masa_ext_dirac_render_sf( int16_t nchan_transport; int16_t masa_band_mapping[MASA_FREQUENCY_BANDS + 1]; - /*CLDFB: last output channels reserved to LFT for CICPx*/ + /* CLDFB: last output channels reserved to LFT for CICPx */ float Cldfb_RealBuffer[MAX_OUTPUT_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer[MAX_OUTPUT_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; @@ -2247,7 +2247,7 @@ static void ivas_masa_ext_dirac_render_sf( } /*-----------------------------------------------------------------* - * protoype signal computation + * prototype signal computation *-----------------------------------------------------------------*/ if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) @@ -2363,7 +2363,7 @@ static void ivas_masa_ext_dirac_render_sf( if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_LS || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD ) { - /*Compute diffuse prototypes*/ + /* Compute diffuse prototypes */ ivas_dirac_dec_compute_diffuse_proto( hDirACRend, hSpatParamRendCom->num_freq_bands, slot_idx ); } diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index 4044dafec2..3aad8c0fec 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -1608,8 +1608,6 @@ typedef struct ivas_masa_external_rendering_struct HANDLE_CLDFB_FILTER_BANK cldfbAnaRend[MASA_MAX_TRANSPORT_CHANNELS]; HANDLE_CLDFB_FILTER_BANK cldfbSynRend[MAX_OUTPUT_CHANNELS]; - - int16_t current_subframe; } MASA_EXT_REND_DATA, *MASA_EXT_REND_HANDLE; /*----------------------------------------------------------------------------------* diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index e918c21609..87067e4efa 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2973,7 +2973,6 @@ static ivas_error setRendInputActiveMasa( } else { - // Todo split rend MASA ext: Maybe something should go in here if ( ( error = initMasaExtRenderer( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { return error; @@ -9117,7 +9116,6 @@ static ivas_error initMasaExtRenderer( hMasaExtRend->hHrtfParambin = NULL; hMasaExtRend->hVBAPdata = NULL; hMasaExtRend->hoa_dec_mtx = NULL; - hMasaExtRend->current_subframe = 0; if ( ( error = getAudioConfigNumChannels( inputMasa->base.inConfig, &hMasaExtRend->nchan_input ) ) != IVAS_ERR_OK ) { -- GitLab From 04fbf981e6dccfffaa91982b22a5cf1cb95fd250 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Wed, 25 Oct 2023 10:33:00 +0200 Subject: [PATCH 8/9] wrapped MASA external renderer rewrite changes under switch NONBE_FIX_225_MASA_EXT_REND --- apps/renderer.c | 20 + lib_com/options.h | 1 + lib_dec/ivas_dirac_dec.c | 16 + lib_dec/ivas_rom_dec.c | 144 ++++ lib_dec/ivas_rom_dec.h | 23 + lib_rend/ivas_dirac_dec_binaural_functions.c | 10 +- lib_rend/ivas_dirac_decorr_dec.c | 7 + lib_rend/ivas_dirac_output_synthesis_dec.c | 27 + lib_rend/ivas_dirac_rend.c | 16 +- lib_rend/ivas_prot_rend.h | 27 +- lib_rend/ivas_rom_rend.c | 2 + lib_rend/ivas_rom_rend.h | 2 + lib_rend/ivas_stat_rend.h | 2 + lib_rend/lib_rend.c | 729 ++++++++++++++++++- 14 files changed, 1020 insertions(+), 6 deletions(-) diff --git a/apps/renderer.c b/apps/renderer.c index 5b667d0629..9b13f77bca 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1793,6 +1793,7 @@ int main( exit( -1 ); } +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( isCurrentFrameMultipleOf20ms ) { if ( masaReaders[i] != NULL ) @@ -1811,6 +1812,19 @@ int main( } } } +#else + if ( masaReaders[i] != NULL ) + { + /* This will update data in hMasaMetadata[i] */ + MasaFileReader_readNextFrame( masaReaders[i] ); + + if ( ( error = IVAS_REND_FeedInputMasaMetadata( hIvasRend, masaIds[i], hMasaMetadata[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } +#endif } @@ -2515,7 +2529,13 @@ static IVAS_AUDIO_CONFIG parseAudioConfig( switch ( charBuf[4] ) { case '1': +#ifdef NONBE_FIX_225_MASA_EXT_REND return IVAS_AUDIO_CONFIG_MASA1; +#else + fprintf( stderr, "1TC MASA support is not functional and is pending on DirAC renderer refactoring.\n" ); + exit( EXIT_FAILURE ); + /*return IVAS_AUDIO_CONFIG_MASA1;*/ // ToDo: temporarily disabled to avoid compilation warnings +#endif case '2': return IVAS_AUDIO_CONFIG_MASA2; default: diff --git a/lib_com/options.h b/lib_com/options.h index 20d8ccce14..012a014b00 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -162,6 +162,7 @@ #define NONBE_FIX_856_TCX_LTP_SYNTH_FILTER /* FhG: issue 856: correct filtering length for tcx-ltp synth filtering*/ #define NONBE_UNIFIED_DECODING_PATHS /* FhG: unify decoding paths */ #define NONBE_FIX_836_PARAMUPMIX_HEADROT /* Dlb: issue #836: Resolve "ParamUpmix MC to SBA conversion done on the already binaurlized output" */ +#define NONBE_FIX_225_MASA_EXT_REND /* Nokia: Resolve #225: Complete MASA external renderer implementation */ /* ##################### End NON-BE switches ########################### */ diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index b4ac8e08c8..2f7c925d5e 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -41,7 +41,9 @@ #include "ivas_cnst.h" #include "ivas_rom_com.h" #include "ivas_rom_dec.h" +#ifdef NONBE_FIX_225_MASA_EXT_REND #include "ivas_rom_rend.h" +#endif #ifdef DEBUGGING #include "debug.h" #endif @@ -841,8 +843,14 @@ ivas_error ivas_dirac_dec_config( if ( !sparfoa_flag ) { common_rend_config_flag = st_ivas->hSpatParamRendCom == NULL ? DIRAC_OPEN : flag_config_inp; +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( ( error = ivas_spat_hSpatParamRendCom_config( &st_ivas->hSpatParamRendCom, common_rend_config_flag, dec_param_estim_new, st_ivas->ivas_format, st_ivas->mc_mode, output_Fs, hodirac_flag, 0 ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_spat_hSpatParamRendCom_config( &st_ivas->hSpatParamRendCom, common_rend_config_flag, dec_param_estim_new, + st_ivas->ivas_format, st_ivas->mc_mode, output_Fs, hodirac_flag ) ) != IVAS_ERR_OK ) + +#endif { return error; } @@ -1965,7 +1973,11 @@ void ivas_dirac_dec_render_sf( ivas_dirac_dec_compute_directional_responses( hSpatParamRendCom, hDirACRend, st_ivas->hVBAPdata, +#ifdef NONBE_FIX_225_MASA_EXT_REND st_ivas->hMasa == NULL ? NULL : st_ivas->hMasa->data.band_mapping, +#else + st_ivas->hMasa, +#endif st_ivas->hMasaIsmData, azimuth, elevation, @@ -1980,7 +1992,11 @@ void ivas_dirac_dec_render_sf( ivas_dirac_dec_compute_directional_responses( hSpatParamRendCom, hDirACRend, st_ivas->hVBAPdata, +#ifdef NONBE_FIX_225_MASA_EXT_REND st_ivas->hMasa == NULL ? NULL : st_ivas->hMasa->data.band_mapping, +#else + st_ivas->hMasa, +#endif st_ivas->hMasaIsmData, azimuth, elevation, diff --git a/lib_dec/ivas_rom_dec.c b/lib_dec/ivas_rom_dec.c index 8f0f1cca60..26846d6254 100644 --- a/lib_dec/ivas_rom_dec.c +++ b/lib_dec/ivas_rom_dec.c @@ -379,6 +379,19 @@ const uint16_t * const sym_freq_ECSQ_tab_abs_lsbs[1 + 4] = * DirAC ROM tables *----------------------------------------------------------------------------------*/ +#ifndef NONBE_FIX_225_MASA_EXT_REND +const float ls_azimuth_4d4[8] = { 45.0f, -45.0f, 135.0f, -135.0f, 45.0f, -45.0f, 135.0f, -135.0f }; + +const float ls_elevation_4d4[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 35.0f, 35.0f, 35.0f, 35.0f }; + +const float diffuse_response_CICP6[5] = { 0.4219f, 0.4219f, 0.3704f, 0.5034f, 0.5034f }; + +const float diffuse_response_CICP14[7] = { 0.3817f, 0.3817f, 0.2819f, 0.5399f, 0.5399f, 0.1519f, 0.1519f }; + +const float diffuse_response_CICP16[9] = { 0.3456f, 0.3456f, 0.3035f, 0.4124f, 0.4124f, 0.2702f, 0.2702f, 0.3023f, 0.3023f }; + +#endif + const float dirac_dithering_azi_scale[DIRAC_DIFFUSE_LEVELS] = { 6.716062e-01f, 1.011837e+00f, 1.799065e+00f, 2.824915e+00f, 4.800879e+00f, 9.206031e+00f, 1.469832e+01f, 2.566224e+01f @@ -389,7 +402,138 @@ const float dirac_dithering_ele_scale[DIRAC_DIFFUSE_LEVELS] = 6.716062e-01f, 1.011804e+00f, 1.796875e+00f, 2.804382e+00f, 4.623130e+00f, 7.802667e+00f, 1.045446e+01f, 1.379538e+01f }; +#ifndef NONBE_FIX_225_MASA_EXT_REND +const int16_t ap_pre_delay[DIRAC_DECORR_NUM_SPLIT_BANDS] = { 7, 2, 1 }; + +const int16_t ap_filter_length[DIRAC_DECORR_NUM_SPLIT_BANDS] = { 15, 6, 3 }; + +const float ap_lattice_delta_phi[DIRAC_MAX_NUM_DECORR_FILTERS*DIRAC_MAX_DECORR_FILTER_LEN] = +{ + 1.802519f, 0.922986f, 1.813685f, 1.272828f, 0.856928f, 0.366571f, 1.531249f, 1.318158f, 0.123812f, 0.897173f, 0.958696f, 1.256384f, 0.179677f, 0.668918f, 1.440292f, 1.573058f, 1.396481f, 1.191463f, 0.444143f, 1.666942f, + 1.273955f, 1.747171f, 1.408330f, 1.002782f, 1.559302f, 1.782992f, 1.474896f, 0.813181f, 1.457724f, 0.588531f, 1.384302f, 0.156493f, 0.600048f, 1.661632f, 0.538958f, 0.645429f, 0.565237f, 0.024684f, 0.264229f, 0.062140f, + 1.235343f, 0.851725f, 1.820211f, 0.116148f, 0.972111f, 0.488703f, 1.777672f, 1.452170f, 0.814134f, 1.272649f, 1.281416f, 0.101871f, 0.897888f, 0.199760f, 0.085732f, 1.686579f, 0.964558f, 0.057281f, 0.910252f, 1.662302f, + 0.955234f, 0.834348f, 1.672478f, 1.324896f, 0.444544f, 1.721172f, 0.153356f, 1.602240f, 0.171880f, 1.169774f, 0.543628f, 1.409581f, 1.763724f, 1.686754f, 1.210390f, 0.402691f, 0.983618f, 0.862997f, 1.220409f, 0.890061f, + 0.031641f, 0.461590f, 1.719550f, 1.357698f, 1.112262f, 1.166531f, 0.246097f, 1.387325f, 0.177485f, 1.446268f, 0.799476f, 1.667227f, 1.723465f, 1.505920f, 0.245874f, 1.155854f, 0.831394f, 0.677194f, 0.568871f, 1.652070f, + 0.019803f, 1.197794f, 0.635553f, 0.531682f, 0.878194f, 0.048050f, 0.080480f, 1.566743f, 0.724210f, 0.853668f, 1.741191f, 0.698465f, 1.553550f, 0.130290f, 0.688346f, 1.331091f, 0.599759f, 1.125466f, 1.764818f, 1.042879f, + 1.486589f, 1.627971f, 1.871181f, 0.102359f, 0.035021f, 1.403176f, 1.468675f, 0.190347f, 0.553282f, 1.031227f, 1.232390f, 1.255724f, 1.504443f, 0.683526f, 0.600958f, 1.746936f, 1.529243f, 1.448196f, 0.646850f, 0.116053f, + 1.283295f, 0.355220f, 1.380620f, 1.858453f, 0.818804f, 0.219006f, 0.476292f, 0.420029f, 1.291187f, 0.568738f, 1.174088f, 0.628805f, 1.753154f, 1.459582f, 1.354449f, 1.755790f, 0.441757f, 0.856240f, 1.647962f, 0.686353f, + 1.395289f, 0.699934f, 0.239310f, 0.239535f, 0.958190f, 0.748780f, 0.513784f, 1.666344f, 1.461995f, 1.599060f, 0.893107f, 0.341873f, 1.387703f, 1.808363f, 0.676542f, 1.424958f, 0.310574f, 0.836247f, 1.011101f, 1.686200f, + 1.823432f, 0.994827f, 1.635555f, 0.684380f, 1.017029f, 1.440371f, 1.694641f, 0.607132f, 1.197331f, 0.862161f, 0.666449f, 1.047956f, 0.159627f, 0.043131f, 1.251515f, 1.618724f, 0.216906f, 0.152250f, 0.471610f, 0.744260f, + 0.576810f, 1.632177f, 1.556912f, 1.866317f, 0.568088f, 1.541817f, 1.726725f, 0.275154f, 0.814958f, 0.863399f, 1.333040f, 0.148277f, 0.197893f, 1.048665f, 1.158090f, 1.692225f, 0.884294f, 0.289619f, 0.380633f, 1.728234f, + 1.433213f, 1.749505f, 1.533837f, 0.669701f, 0.372580f, 1.052390f, 1.116645f, 0.181320f, 1.139126f, 0.222671f, 0.604393f, 1.811797f, 1.743315f, 1.368792f, 1.861434f, 0.751908f, 0.159811f, 1.566503f, 0.443273f, 1.667530f, + 1.083060f, 1.243136f, 0.717777f, 0.675019f, 0.690490f, 0.672228f, 1.060789f, 0.423566f, 1.198457f, 0.485768f, 0.993953f, 0.443540f, 0.361702f, 1.552042f, 0.863562f, 1.517677f, 1.061899f, 0.691413f, 1.642818f, 1.756590f, + 0.278323f, 0.790363f, 0.172303f, 0.417138f, 0.009343f, 0.783325f, 1.369303f, 1.041067f, 0.467102f, 0.992773f, 1.525170f, 0.871213f, 0.243906f, 1.542036f, 0.449148f, 0.843633f, 0.191800f, 1.614246f, 1.038188f, 1.415620f, + 0.551081f, 0.382599f, 1.410121f, 0.102084f, 0.137286f, 0.671081f, 0.254860f, 1.758068f, 1.079013f, 0.129143f, 1.410873f, 0.150485f, 0.601119f, 0.760737f, 0.975905f, 0.223261f, 0.710162f, 1.677048f, 0.996836f, 1.849865f, + 1.536222f, 0.089016f, 0.960881f, 0.388690f, 0.379955f, 1.002223f, 1.271420f, 1.410632f, 0.254397f, 1.535559f, 1.133703f, 1.305280f, 1.466565f, 0.274167f, 0.399688f, 1.359638f, 1.766289f, 1.401348f, 1.310883f, 0.261030f, + 1.314825f, 1.538635f, 1.317986f, 1.243167f, 1.749461f, 1.689706f, 0.024853f, 0.634754f, 1.036317f, 1.828101f, 1.676951f, 0.023606f, 0.857000f, 0.076471f, 1.622198f, 0.254469f, 1.451625f, 1.720881f, 0.763812f, 0.186982f, + 0.056994f, 0.590507f, 0.375291f, 1.609261f, 0.607721f, 0.026355f, 0.483366f, 0.823931f, 0.792878f, 0.163577f, 0.753588f, 0.730789f, 0.135991f, 1.031660f, 1.554135f, 1.192863f, 0.016693f, 0.125796f, 1.017920f, 1.591773f, + 0.575956f, 0.112943f, 0.249506f, 1.399570f, 0.053241f, 1.410759f, 0.251638f, 1.059086f, 0.025315f, 1.422914f, 1.030412f, 0.848758f, 0.317396f, 1.375456f, 1.116858f, 1.682310f, 0.279550f, 0.325974f, 0.937704f, 1.744329f, + 0.447773f, 1.024286f, 1.001528f, 1.863684f, 1.278323f, 0.860699f, 1.346331f, 1.692596f, 0.022627f, 1.033613f, 0.546354f, 0.395804f, 1.486546f, 1.381045f, 1.312260f, 0.245976f, 1.607429f, 1.818793f, 0.964359f, 1.496598f, + 0.669967f, 1.535929f, 1.841878f, 0.979127f, 0.614002f, 1.879218f, 0.512531f, 1.167061f, 0.081697f, 1.773427f, 1.535668f, 0.757729f, 0.220395f, 1.538243f, 1.281162f, 0.302159f, 0.889871f, 0.798522f, 1.476288f, 1.665941f, + 0.915365f, 1.394094f, 0.757041f, 0.350064f, 1.199679f, 1.319499f, 1.128405f, 0.632337f, 0.790673f, 0.461582f, 1.693343f, 1.537442f, 0.346527f, 0.433782f, 1.754552f, 0.550903f, 0.686724f, 0.764433f, 1.792750f, 1.489998f +}; + +const float ap_lattice_coeffs_1[DIRAC_DECORR_FILTER_LEN_1*DIRAC_MAX_NUM_DECORR_FILTERS] = +{ + 0.795329f, 0.502700f, 0.204456f, 0.416566f, 0.459648f, 0.270454f, -0.201944f, 0.027997f, 0.067811f, -0.052627f, -0.038779f, -0.057387f, 0.020480f, 0.367697f, -0.593705f, + 0.533667f, 0.202500f, -0.001953f, 0.195081f, -0.184458f, -0.233867f, 0.228100f, -0.329293f, -0.338714f, -0.079700f, 0.052389f, -0.009468f, 0.178807f, 0.190843f, -0.478877f, + 0.044022f, 0.788498f, 0.133529f, -0.173657f, 0.545391f, 0.681431f, 0.332868f, 0.294685f, 0.325261f, 0.047617f, 0.157401f, 0.116272f, 0.218980f, -0.189248f, -0.317401f, + -0.753681f, 0.772385f, -0.507384f, 0.276980f, -0.692775f, 0.232302f, -0.354759f, -0.015620f, -0.223059f, 0.310506f, -0.035034f, 0.085783f, -0.499998f, 0.215260f, 0.201415f, + -0.495551f, -0.301660f, 0.196510f, 0.326147f, -0.594364f, 0.314921f, 0.668671f, 0.076643f, 0.045711f, -0.124790f, -0.203272f, -0.297190f, 0.125806f, -0.179483f, -0.201757f, + 0.161128f, 0.396050f, 0.266897f, 0.356586f, 0.488145f, -0.056254f, 0.139280f, -0.296405f, -0.112844f, 0.037405f, -0.367425f, -0.216292f, -0.277360f, 0.389420f, 0.115115f, + -0.240449f, -0.271015f, 0.426720f, -0.011059f, 0.151813f, 0.253490f, 0.225764f, 0.498716f, -0.136377f, 0.443004f, -0.305017f, -0.031310f, -0.010765f, 0.170349f, 0.496478f, + -0.139875f, -0.241998f, -0.104850f, 0.294343f, -0.067728f, -0.492202f, -0.487610f, 0.036395f, 0.109393f, 0.396155f, -0.352845f, -0.205913f, -0.082999f, -0.463033f, -0.309296f, + 0.025886f, -0.092456f, -0.125139f, -0.156117f, -0.004887f, 0.178440f, 0.089586f, 0.044827f, 0.238219f, -0.312120f, -0.390688f, -0.178543f, 0.454418f, 0.387012f, -0.388874f, + -0.197797f, 0.035540f, 0.455388f, -0.054410f, 0.380035f, 0.290964f, 0.048804f, 0.078637f, 0.221740f, -0.217548f, 0.121289f, -0.396681f, -0.218482f, -0.127265f, -0.269507f, + -0.344218f, -0.465038f, -0.421415f, -0.026031f, 0.221547f, 0.361993f, -0.348243f, 0.294983f, 0.366175f, 0.070663f, -0.086050f, 0.252129f, 0.156066f, -0.062800f, 0.408972f, + 0.242461f, -0.301764f, -0.066160f, 0.388651f, -0.462227f, -0.158880f, 0.230796f, -0.093179f, 0.047076f, 0.073402f, -0.335018f, 0.022940f, 0.354611f, 0.072391f, 0.019473f, + 0.336900f, -0.480534f, 0.170267f, 0.259663f, -0.393576f, -0.348588f, -0.108962f, 0.278842f, 0.385490f, 0.362249f, -0.318739f, 0.362305f, 0.288936f, 0.291204f, -0.278019f, + -0.449302f, -0.288513f, -0.413973f, -0.405279f, -0.295152f, -0.245110f, 0.002530f, 0.287890f, 0.348229f, -0.178354f, -0.206517f, 0.351081f, -0.482205f, 0.021360f, -0.492207f, + 0.048642f, 0.453282f, 0.109160f, 0.232599f, -0.473781f, 0.140502f, 0.352527f, -0.098606f, 0.147172f, -0.055797f, 0.107739f, -0.231026f, 0.357310f, 0.348031f, 0.232404f, + -0.324788f, 0.162480f, 0.057647f, -0.060734f, -0.009742f, -0.224185f, -0.282355f, -0.065443f, 0.064697f, 0.280370f, -0.284906f, -0.470501f, 0.019484f, -0.442308f, 0.377214f, + 0.447384f, -0.290498f, 0.345528f, 0.370207f, -0.313120f, 0.119592f, 0.300014f, 0.406995f, -0.277922f, 0.447039f, 0.194824f, 0.157703f, -0.223402f, -0.147167f, 0.379073f, + -0.216342f, 0.137967f, -0.397180f, 0.073905f, -0.273110f, -0.443037f, -0.168327f, 0.346264f, 0.037543f, 0.065387f, 0.163901f, -0.122523f, 0.365477f, -0.316321f, 0.117273f, + -0.301282f, 0.169625f, -0.336466f, 0.269914f, -0.420160f, -0.331296f, 0.498523f, -0.393009f, -0.462184f, -0.323097f, 0.470977f, -0.359463f, 0.264315f, 0.216797f, 0.493400f, + 0.151489f, -0.321653f, 0.464413f, -0.355673f, 0.420401f, 0.184297f, -0.302128f, 0.136536f, -0.252849f, 0.193906f, 0.298775f, 0.238808f, -0.386298f, -0.181999f, -0.077326f, + 0.124840f, -0.263394f, 0.389606f, 0.004502f, 0.339804f, 0.415204f, 0.377751f, 0.400221f, 0.352426f, -0.003808f, -0.184530f, 0.433348f, -0.237554f, 0.147684f, 0.407210f, + 0.067616f, -0.249313f, -0.354010f, 0.320937f, 0.426000f, 0.309576f, -0.189689f, -0.209447f, 0.158967f, -0.081929f, 0.386828f, 0.178582f, -0.407143f, 0.222189f, -0.149097f +}; + +const float ap_lattice_coeffs_2[DIRAC_DECORR_FILTER_LEN_2*DIRAC_MAX_NUM_DECORR_FILTERS] = +{ + 0.633692f, 0.681207f, -0.049418f, 0.286715f, 0.146022f, 0.135402f, + -0.410145f, -0.206766f, -0.656968f, -0.101746f, 0.436299f, 0.339818f, + -0.131383f, -0.773746f, -0.301627f, 0.327561f, 0.332227f, 0.205858f, + 0.717602f, -0.552370f, -0.150136f, 0.054556f, 0.239519f, -0.648477f, + -0.721848f, 0.189377f, 0.068185f, 0.006216f, 0.077025f, 0.038678f, + 0.423415f, -0.108658f, 0.432050f, -0.414641f, 0.277840f, 0.418486f, + 0.077811f, -0.283063f, -0.357008f, 0.004634f, -0.442640f, -0.372315f, + 0.403336f, 0.243023f, 0.314367f, 0.444513f, 0.347517f, 0.496043f, + 0.108408f, 0.469382f, -0.170286f, 0.326310f, 0.172157f, 0.440334f, + 0.058417f, 0.339673f, -0.194965f, 0.491219f, -0.281296f, -0.043120f, + -0.482487f, -0.335005f, -0.336159f, 0.196469f, -0.164623f, 0.442491f, + -0.135190f, -0.354385f, 0.452133f, -0.311221f, -0.347640f, 0.498342f, + -0.288999f, 0.376431f, -0.177924f, 0.195542f, 0.333872f, -0.152697f, + 0.053230f, 0.457767f, 0.442476f, 0.235254f, -0.345159f, -0.286098f, + 0.155438f, 0.405393f, 0.217073f, -0.107960f, 0.010698f, 0.368987f, + -0.498358f, -0.495816f, -0.215479f, -0.093869f, 0.320276f, -0.013842f, + 0.489406f, 0.470814f, -0.065834f, 0.346871f, 0.027279f, 0.150086f, + 0.484013f, -0.497391f, 0.168796f, -0.493841f, -0.173528f, 0.334676f, + 0.264235f, -0.424651f, -0.314926f, 0.253086f, 0.397381f, -0.491565f, + -0.453727f, -0.463358f, -0.019128f, 0.000344f, 0.315432f, 0.472345f, + 0.095139f, 0.283375f, -0.225088f, -0.119762f, -0.476871f, 0.037525f, + 0.336951f, 0.494511f, -0.062603f, 0.177652f, 0.463892f, 0.489286f +}; + +const float ap_lattice_coeffs_3[DIRAC_DECORR_FILTER_LEN_3*DIRAC_MAX_NUM_DECORR_FILTERS] = +{ + 0.018977f, -0.212205f, 0.422719f, + -0.400657f, -0.106890f, -0.024589f, + 0.140005f, 0.279582f, 0.032357f, + 0.632535f, 0.578535f, -0.734606f, + 0.017182f, 0.013244f, -0.027715f, + -0.353356f, -0.482160f, -0.491265f, + 0.457024f, 0.165122f, 0.469723f, + -0.195705f, 0.440105f, -0.477366f, + 0.360186f, -0.490565f, 0.484623f, + -0.173791f, 0.007543f, 0.278186f, + 0.434416f, 0.060363f, -0.193717f, + -0.033709f, 0.496222f, 0.002939f, + -0.480848f, -0.109552f, -0.023198f, + 0.324679f, -0.292075f, -0.356148f, + -0.366595f, 0.380917f, -0.301741f, + 0.110318f, 0.383789f, 0.303984f, + -0.499685f, -0.349584f, 0.334749f, + -0.020224f, -0.430078f, -0.154705f, + -0.371129f, 0.334080f, 0.346913f, + -0.166781f, -0.229089f, 0.117956f, + 0.341292f, 0.490463f, 0.493655f, + -0.367726f, 0.426528f, -0.045774f +}; + +const float * const ap_lattice_coeffs[DIRAC_DECORR_NUM_SPLIT_BANDS] = +{ + &ap_lattice_coeffs_1[0], + &ap_lattice_coeffs_2[0], + &ap_lattice_coeffs_3[0], +}; + +const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1] = +{ + 0.0f, 0.125f, 0.375f, 1.0f +}; + +const int16_t sba_map_tc[11] = +{ + 0, 1, 2, 3, 4, 8, 9, 15, 5, 6, 7 +}; +const int16_t sba_map_tc_512[11] = +{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15 +}; + +#endif /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables *----------------------------------------------------------------------------------*/ diff --git a/lib_dec/ivas_rom_dec.h b/lib_dec/ivas_rom_dec.h index 993f6abb0f..5c22186afd 100644 --- a/lib_dec/ivas_rom_dec.h +++ b/lib_dec/ivas_rom_dec.h @@ -92,9 +92,32 @@ extern const uint16_t *const sym_freq_ECSQ_tab_abs_lsbs[1 + 4]; * DirAC ROM tables *----------------------------------------------------------------------------------*/ +#ifndef NONBE_FIX_225_MASA_EXT_REND +extern const float ls_azimuth_4d4[8]; +extern const float ls_elevation_4d4[8]; +extern const float diffuse_response_CICP6[5]; +extern const float diffuse_response_CICP14[7]; +extern const float diffuse_response_CICP16[9]; + +#endif + extern const float dirac_dithering_azi_scale[DIRAC_DIFFUSE_LEVELS]; extern const float dirac_dithering_ele_scale[DIRAC_DIFFUSE_LEVELS]; +#ifndef NONBE_FIX_225_MASA_EXT_REND +extern const int16_t ap_pre_delay[DIRAC_DECORR_NUM_SPLIT_BANDS]; +extern const int16_t ap_filter_length[DIRAC_DECORR_NUM_SPLIT_BANDS]; +extern const float ap_lattice_delta_phi[DIRAC_MAX_NUM_DECORR_FILTERS * DIRAC_MAX_DECORR_FILTER_LEN]; +extern const float ap_lattice_coeffs_1[DIRAC_DECORR_FILTER_LEN_1 * DIRAC_MAX_NUM_DECORR_FILTERS]; +extern const float ap_lattice_coeffs_2[DIRAC_DECORR_FILTER_LEN_2 * DIRAC_MAX_NUM_DECORR_FILTERS]; +extern const float ap_lattice_coeffs_3[DIRAC_DECORR_FILTER_LEN_3 * DIRAC_MAX_NUM_DECORR_FILTERS]; +extern const float *const ap_lattice_coeffs[DIRAC_DECORR_NUM_SPLIT_BANDS]; +extern const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1]; + +extern const int16_t sba_map_tc[11]; +extern const int16_t sba_map_tc_512[11]; +#endif + /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables *----------------------------------------------------------------------------------*/ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 0bbfb2c85b..6636f08060 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -127,6 +127,13 @@ static void matrixMul( float Are[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Ai static void matrixTransp2Mul( float Are[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Aim[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Bre[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Bim[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float outRe[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float outIm[BINAURAL_CHANNELS][BINAURAL_CHANNELS] ); +#ifdef NONBE_FIX_225_MASA_EXT_REND +#ifdef SPLIT_REND_WITH_HEAD_ROT +static void ivas_masa_ext_rend_parambin_internal( MASA_EXT_REND_HANDLE hMasaExtRend, COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, float *output_f[], const int16_t subframe, const SPLIT_REND_WRAPPER *hSplitRendWrapper, float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX] ); +#else +static void ivas_masa_ext_rend_parambin_internal( MASA_EXT_REND_HANDLE hMasaExtRend, COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, float *output_f[], const int16_t subframe ); +#endif +#endif /*------------------------------------------------------------------------- * ivas_dirac_dec_init_binaural_data() @@ -3471,7 +3478,7 @@ void ivas_omasa_preProcessStereoTransportsForMovedObjects( return; } - +#ifdef NONBE_FIX_225_MASA_EXT_REND static void ivas_masa_ext_rend_parambin_internal( MASA_EXT_REND_HANDLE hMasaExtRend, COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, @@ -3785,3 +3792,4 @@ void ivas_masa_ext_rend_parambin_render( return; } +#endif diff --git a/lib_rend/ivas_dirac_decorr_dec.c b/lib_rend/ivas_dirac_decorr_dec.c index 89c2c4a086..36ac883e87 100644 --- a/lib_rend/ivas_dirac_decorr_dec.c +++ b/lib_rend/ivas_dirac_decorr_dec.c @@ -38,8 +38,15 @@ #include "prot.h" #include "ivas_prot.h" #include "ivas_prot_rend.h" +#ifndef NONBE_FIX_225_MASA_EXT_REND +#include "ivas_stat_dec.h" +#endif #include "ivas_cnst.h" +#ifdef NONBE_FIX_225_MASA_EXT_REND #include "ivas_rom_rend.h" +#else +#include "ivas_rom_dec.h" +#endif #ifdef DEBUGGING #include "debug.h" #endif diff --git a/lib_rend/ivas_dirac_output_synthesis_dec.c b/lib_rend/ivas_dirac_output_synthesis_dec.c index 8ee3b1e4f5..30d102440b 100644 --- a/lib_rend/ivas_dirac_output_synthesis_dec.c +++ b/lib_rend/ivas_dirac_output_synthesis_dec.c @@ -1741,11 +1741,19 @@ static void ivas_dirac_dec_get_response_split_order( *------------------------------------------------------------------------*/ void ivas_dirac_dec_compute_directional_responses( +#ifdef NONBE_FIX_225_MASA_EXT_REND SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ const int16_t *masa_band_mapping, /* i : Band mapping for MASA, NULL assumes not using MASA in any form */ MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ +#else + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ + DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ + const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ + const MASA_DECODER_HANDLE hMasa, /* i : MASA decoder structure */ + MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ +#endif const int16_t *azimuth, const int16_t *elevation, const int16_t md_idx, @@ -1793,14 +1801,24 @@ void ivas_dirac_dec_compute_directional_responses( for ( k = 0; k < hSpatParamRendCom->num_freq_bands; ++k ) { +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( masa_band_mapping != NULL && k == MASA_band_grouping_24[masa_band_mapping[codingBand + 1]] ) +#else + if ( hMasa != NULL && k == MASA_band_grouping_24[hMasa->data.band_mapping[codingBand + 1]] ) +#endif { codingBand++; } +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( masa_band_mapping != NULL && k > MASA_band_grouping_24[masa_band_mapping[codingBand]] && k < MASA_band_grouping_24[masa_band_mapping[codingBand + 1]] && k != hDirACRend->h_output_synthesis_psd_params.max_band_decorr ) +#else + if ( hMasa != NULL && k > MASA_band_grouping_24[hMasa->data.band_mapping[codingBand]] && + k < MASA_band_grouping_24[hMasa->data.band_mapping[codingBand + 1]] && + k != hDirACRend->h_output_synthesis_psd_params.max_band_decorr ) +#endif { /* Panning gains have to be computed only for the first bin of the coding band in MASA, for other bins the previous values can be used */ if ( hDirACRend->synthesisConf != DIRAC_SYNTHESIS_GAIN_SHD ) @@ -1841,7 +1859,11 @@ void ivas_dirac_dec_compute_directional_responses( } } +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( masa_band_mapping == NULL && hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) +#else + if ( hMasa == NULL && hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) +#endif { mvr2r_inc( direct_response_hoa, 1, &hDirACRend->h_output_synthesis_psd_state.direct_responses[k], hSpatParamRendCom->num_freq_bands, num_channels_dir ); @@ -1850,8 +1872,13 @@ void ivas_dirac_dec_compute_directional_responses( mvr2r_inc( direct_response_dir2, 1, &hDirACRend->h_output_synthesis_psd_state.direct_responses[k + hSpatParamRendCom->num_freq_bands * num_channels_dir], hSpatParamRendCom->num_freq_bands, num_channels_dir ); } } +#ifdef NONBE_FIX_225_MASA_EXT_REND else if ( ( ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) && ( masa_band_mapping != NULL ) ) || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) +#else + else if ( ( ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) && ( hMasa != NULL ) ) || + hDirACRend->synthesisConf == DIRAC_SYNTHESIS_PSD_SHD || hDirACRend->synthesisConf == DIRAC_SYNTHESIS_MONO ) +#endif { /* Synthesize the first direction */ spreadCoherencePanningHoa( azimuth[k], elevation[k], hSpatParamRendCom->spreadCoherence[md_idx][k], direct_response_hoa, num_channels_dir, hDirACRend->hOutSetup.ambisonics_order ); diff --git a/lib_rend/ivas_dirac_rend.c b/lib_rend/ivas_dirac_rend.c index 888432139d..98cd33401e 100644 --- a/lib_rend/ivas_dirac_rend.c +++ b/lib_rend/ivas_dirac_rend.c @@ -39,7 +39,11 @@ #include "ivas_prot.h" #include "ivas_prot_rend.h" #include "ivas_cnst.h" +#ifdef NONBE_FIX_225_MASA_EXT_REND #include "ivas_rom_rend.h" +#else +#include "ivas_rom_dec.h" +#endif #ifdef DEBUGGING #include "debug.h" #endif @@ -197,8 +201,12 @@ ivas_error ivas_spat_hSpatParamRendCom_config( const IVAS_FORMAT ivas_format, const MC_MODE mc_mode, const int32_t output_Fs, +#ifdef NONBE_FIX_225_MASA_EXT_REND const int16_t hodirac_flag, const int16_t masa_ext_rend_flag ) +#else + const int16_t hodirac_flag ) +#endif { ivas_error error; int16_t map_idx; @@ -267,7 +275,11 @@ ivas_error ivas_spat_hSpatParamRendCom_config( hSpatParamRendCom->dirac_md_buffer_length = 0; hSpatParamRendCom->dirac_bs_md_write_idx = 0; hSpatParamRendCom->dirac_read_idx = 0; +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( mc_mode == MC_MODE_MCMASA || masa_ext_rend_flag == 1 ) +#else + if ( mc_mode == MC_MODE_MCMASA ) +#endif { hSpatParamRendCom->dirac_md_buffer_length = MAX_PARAM_SPATIAL_SUBFRAMES; @@ -2086,7 +2098,7 @@ void rotateAziEle_DirAC( return; } - +#ifdef NONBE_FIX_225_MASA_EXT_REND /* A reduced rewrite of the corresponding decoder side function */ static void ivas_masa_ext_dirac_render_sf( MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: IVAS decoder structure */ @@ -2503,6 +2515,6 @@ void ivas_masa_ext_dirac_render( hSpatParamRendCom->dirac_read_idx = ( hSpatParamRendCom->dirac_read_idx + 1 ) % hSpatParamRendCom->dirac_md_buffer_length; } - return; } +#endif diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index f796348c46..1cb9e5adfc 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -200,6 +200,7 @@ void ivas_dirac_dec_binaural_render( float *output_f[] /* o : rendered time signal */ ); +#ifdef NONBE_FIX_225_MASA_EXT_REND void ivas_masa_ext_rend_parambin_render( MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA ext rend structure */ COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ @@ -213,7 +214,7 @@ void ivas_masa_ext_rend_parambin_render( const int16_t num_subframes /* i : number of subframes to render */ #endif ); - +#endif ivas_error ivas_dirac_dec_init_binaural_data( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ HRTFS_PARAMBIN_HANDLE hHrtfParambin /* i : HRTF structure for rendering */ @@ -495,6 +496,7 @@ void ivas_dirac_dec_compute_power_factors( float *diffuse_power_factor ); +#ifdef NONBE_FIX_225_MASA_EXT_REND void ivas_dirac_dec_compute_directional_responses( SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ @@ -509,6 +511,22 @@ void ivas_dirac_dec_compute_directional_responses( const float *p_Rmat, /* i : rotation matrix */ const int16_t hodirac_flag /* i : flag to indicate HO-DirAC mode */ ); +#else +void ivas_dirac_dec_compute_directional_responses( + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, /* i/o: common spatial renderer data handle */ + DIRAC_REND_HANDLE hDirACRend, /* i/o: DirAC renderer handle */ + const VBAP_HANDLE hVBAPdata, /* i : VBAP structure */ + const MASA_DECODER_HANDLE hMasa, /* i : MASA decoder structure */ + MASA_ISM_DATA_HANDLE hMasaIsm, /* i : MASA_ISM data structure */ + const int16_t *azimuth, + const int16_t *elevation, + const int16_t md_idx, + const float *surCohRatio, + const int16_t shd_rot_max_order, /* i : split-order rotation method */ + const float *p_Rmat, /* i : rotation matrix */ + const int16_t hodirac_flag /* i : flag to indicate HO-DirAC mode */ +); +#endif void ivas_dirac_dec_get_frequency_axis( float *frequency_axis, /* o : array of center frequencies of a real filter bank */ @@ -523,8 +541,12 @@ ivas_error ivas_spat_hSpatParamRendCom_config( const IVAS_FORMAT ivas_format, const MC_MODE mc_mode, const int32_t output_Fs, +#ifdef NONBE_FIX_225_MASA_EXT_REND const int16_t hodirac_flag, const int16_t masa_ext_rend_flag +#else + const int16_t hodirac_flag +#endif ); void ivas_spat_hSpatParamRendCom_close( @@ -545,12 +567,13 @@ void ivas_dirac_deallocate_parameters( const int16_t params_flag /* i : set of parameters flag */ ); +#ifdef NONBE_FIX_225_MASA_EXT_REND void ivas_masa_ext_dirac_render( MASA_EXT_REND_HANDLE hMasaExtRend, /* i/o: MASA renderer structure */ float *output_f[], /* i/o: input/output signals in time domain */ const int16_t num_subframes /* i : number of subframes to render */ ); - +#endif /*----------------------------------------------------------------------------------* * HRTF diff --git a/lib_rend/ivas_rom_rend.c b/lib_rend/ivas_rom_rend.c index 99ffe44285..4aef137a33 100644 --- a/lib_rend/ivas_rom_rend.c +++ b/lib_rend/ivas_rom_rend.c @@ -39,6 +39,7 @@ /* clang-format off */ +#ifdef NONBE_FIX_225_MASA_EXT_REND /*----------------------------------------------------------------------------------* * DirAC renderer ROM tables *----------------------------------------------------------------------------------*/ @@ -181,6 +182,7 @@ const int16_t sba_map_tc_512[11] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15 }; +#endif /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables diff --git a/lib_rend/ivas_rom_rend.h b/lib_rend/ivas_rom_rend.h index 165f4791c6..db0c7d1c33 100644 --- a/lib_rend/ivas_rom_rend.h +++ b/lib_rend/ivas_rom_rend.h @@ -41,6 +41,7 @@ #include "ivas_cnst.h" #include "ivas_stat_rend.h" +#ifdef NONBE_FIX_225_MASA_EXT_REND /*----------------------------------------------------------------------------------* * DirAC renderer ROM tables *----------------------------------------------------------------------------------*/ @@ -62,6 +63,7 @@ extern const float ap_split_frequencies[DIRAC_DECORR_NUM_SPLIT_BANDS + 1]; extern const int16_t sba_map_tc[11]; extern const int16_t sba_map_tc_512[11]; +#endif /*----------------------------------------------------------------------------------* * FASTCONV and PARAMETRIC binaural renderer ROM tables diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index 3aad8c0fec..634025e11e 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -1584,6 +1584,7 @@ typedef enum } ChannelType; +#ifdef NONBE_FIX_225_MASA_EXT_REND /*----------------------------------------------------------------------------------* * MASA external renderer structure *----------------------------------------------------------------------------------*/ @@ -1609,6 +1610,7 @@ typedef struct ivas_masa_external_rendering_struct HANDLE_CLDFB_FILTER_BANK cldfbAnaRend[MASA_MAX_TRANSPORT_CHANNELS]; HANDLE_CLDFB_FILTER_BANK cldfbSynRend[MAX_OUTPUT_CHANNELS]; } MASA_EXT_REND_DATA, *MASA_EXT_REND_HANDLE; +#endif /*----------------------------------------------------------------------------------* * Multichannel MASA (McMASA) analysis structure diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 87067e4efa..791ef0cff0 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -212,14 +212,29 @@ typedef struct } input_split_post_rend; #endif +#ifndef NONBE_FIX_225_MASA_EXT_REND +/* Due to API of some rendering methods, the renderer has to use the decoder struct. + Only struct members relevant for rendering will be initialized, therefore typedef as "dummy" decoder struct */ +typedef Decoder_Struct DecoderDummy; +#endif + typedef struct { +#ifdef NONBE_FIX_225_MASA_EXT_REND input_base base; MASA_METADATA_FRAME masaMetadata; bool metadataHasBeenFed; float *bufferData; MASA_EXT_REND_HANDLE hMasaExtRend; MASA_PREREND_HANDLE hMasaPrerend; +#else + input_base base; + DecoderDummy *decDummy; + MASA_METADATA_FRAME masaMetadata; + bool metadataHasBeenFed; + float *bufferData; + MASA_PREREND_HANDLE hMasaPrerend; +#endif } input_masa; struct IVAS_REND @@ -263,13 +278,14 @@ struct IVAS_REND }; +#ifdef NONBE_FIX_225_MASA_EXT_REND /*-------------------------------------------------------------------* * Local function prototypes *-------------------------------------------------------------------*/ static ivas_error initMasaExtRenderer( input_masa *inputMasa, const AUDIO_CONFIG outConfig ); static void freeMasaExtRenderer( MASA_EXT_REND_HANDLE *hMasaExtRendOut ); - +#endif /*-------------------------------------------------------------------* * Local functions @@ -640,6 +656,7 @@ static ivas_error validateOutputSampleRate( /* If no binaural rendering, any sampling rate is supported */ return IVAS_ERR_OK; } +#ifdef NONBE_FIX_225_MASA_EXT_REND #ifdef SPLIT_REND_WITH_HEAD_ROT else if ( ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) && sampleRate != 48000 ) { @@ -647,6 +664,7 @@ static ivas_error validateOutputSampleRate( } else { +#endif #endif /* Otherwise rendering to binaural, support the same set as IVAS decoder */ @@ -660,9 +678,11 @@ static ivas_error validateOutputSampleRate( } return IVAS_ERR_INVALID_SAMPLING_RATE; +#ifdef NONBE_FIX_225_MASA_EXT_REND #ifdef SPLIT_REND_WITH_HEAD_ROT } #endif +#endif } @@ -2930,6 +2950,392 @@ static void clearInputSba( } +#ifndef NONBE_FIX_225_MASA_EXT_REND +static ivas_error initMasaDummyDecForMcOut( + input_masa *inputMasa, + const AUDIO_CONFIG outConfig ) +{ + ivas_error error; + int16_t numCldfbAnalyses; + int16_t numCldfbSyntheses; + int16_t i; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + decDummy->hDecoderConfig->output_config = outConfig; + decDummy->sba_analysis_order = 1; + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + + ivas_output_init( &( decDummy->hOutSetup ), outConfig ); + ivas_output_init( &( decDummy->hIntSetup ), outConfig ); + + decDummy->renderer_type = RENDERER_DIRAC; + if ( outConfig == IVAS_AUDIO_CONFIG_STEREO ) + { + decDummy->renderer_type = RENDERER_STEREO_PARAMETRIC; + } + + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = IVAS_AUDIO_CONFIG_INVALID; + + /* Todo refactor: Access to qmetadata is not required by the algorithm. */ + if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hQMetaData->coherence_flag = 1; + + if ( ( error = ivas_dirac_dec_config( decDummy, DIRAC_OPEN ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hSpatParamRendCom->dirac_bs_md_write_idx = 0; + + if ( decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + numCldfbAnalyses = decDummy->nchan_transport; + numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; + + for ( i = 0; i < numCldfbAnalyses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + } + + for ( i = 0; i < numCldfbSyntheses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + + +static ivas_error initMasaDummyDecForSbaOut( + input_masa *inputMasa, + const AUDIO_CONFIG outConfig ) +{ + ivas_error error; + int16_t numCldfbAnalyses; + int16_t numCldfbSyntheses; + int16_t i; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + decDummy->hDecoderConfig->output_config = outConfig; + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + decDummy->sba_analysis_order = 1; + + ivas_output_init( &( decDummy->hOutSetup ), outConfig ); + ivas_output_init( &( decDummy->hIntSetup ), outConfig ); + + decDummy->renderer_type = RENDERER_DIRAC; + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = IVAS_AUDIO_CONFIG_INVALID; + + /* Todo refactor: Access to qmetadata is not required by the algorithm. */ + if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hQMetaData->coherence_flag = 1; + + if ( ( error = ivas_dirac_dec_config( decDummy, DIRAC_OPEN ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hSpatParamRendCom->dirac_bs_md_write_idx = 0; + + numCldfbAnalyses = decDummy->nchan_transport; + numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; + + for ( i = 0; i < numCldfbAnalyses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + } + + for ( i = 0; i < numCldfbSyntheses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + + +static ivas_error initMasaDummyDecForBinauralOut( + input_masa *inputMasa, + const AUDIO_CONFIG outConfig ) +{ + ivas_error error; + int16_t i; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + decDummy->hDecoderConfig->output_config = outConfig; + decDummy->sba_analysis_order = 1; + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + + ivas_output_init( &( decDummy->hOutSetup ), outConfig ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) +#else + if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL ) +#endif + { + decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC; + } + else + { + decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC_ROOM; + } + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = IVAS_AUDIO_CONFIG_INVALID; + + if ( ( error = ivas_dirac_dec_config( decDummy, DIRAC_OPEN ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hSpatParamRendCom->dirac_bs_md_write_idx = 0; + + if ( ( error = ivas_dirac_dec_binaural_copy_hrtfs( &decDummy->hHrtfParambin ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy, decDummy->hHrtfParambin ) ) != IVAS_ERR_OK ) + { + return error; + } + + for ( i = 0; i < BINAURAL_CHANNELS; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + + +static ivas_error updateMasaDummyDec( + input_masa *inputMasa, + const AUDIO_CONFIG outConfig ) +{ + ivas_error error; + + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + error = initMasaDummyDecForMcOut( inputMasa, outConfig ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + error = initMasaDummyDecForSbaOut( inputMasa, outConfig ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + error = initMasaDummyDecForBinauralOut( inputMasa, outConfig ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + + return error; +} + + +static DecoderDummy *initDecoderDummy( + const int32_t sampleRate, + const int16_t numTransChannels, + const AUDIO_CONFIG outConfig, +#ifdef SPLIT_REND_WITH_HEAD_ROT + const uint8_t enableRenderConfig, + const SPLIT_REND_WRAPPER *pSplitRendWrapper +#else + const uint8_t enableRenderConfig +#endif +) +{ + ivas_error error; + int16_t i; + int16_t numOutChannels; + DecoderDummy *decDummy; + int16_t num_subframes; + num_subframes = MAX_PARAM_SPATIAL_SUBFRAMES; + + if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) + { + /* Checking error with assert is enough, this function is only temporary anyway */ + assert( error == IVAS_ERR_OK ); + } + + decDummy = malloc( sizeof( DecoderDummy ) ); + decDummy->hDecoderConfig = malloc( sizeof( DECODER_CONFIG ) ); + decDummy->hDecoderConfig->output_Fs = sampleRate; + decDummy->hDecoderConfig->nchan_out = numOutChannels; + decDummy->hDecoderConfig->Opt_Headrotation = 0; + decDummy->hDecoderConfig->Opt_tsm = 0; +#ifndef NONBE_UNIFIED_DECODING_PATHS + decDummy->hDecoderConfig->Opt_5ms = 0; +#endif + decDummy->hBinRenderer = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT + decDummy->hSplitBinRend.hSplitRendBits = NULL; + decDummy->hSplitBinRend.hMultiBinCldfbData = NULL; + ivas_init_split_rend_handles( &decDummy->hSplitBinRend.splitrend ); + decDummy->hSplitBinRend.splitrend = *pSplitRendWrapper; + + if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + if ( ( decDummy->hSplitBinRend.hMultiBinCldfbData = (IVAS_DEC_SPLIT_REND_MULTI_BIN_CLDFB_DATA_HANDLE) malloc( sizeof( IVAS_DEC_SPLIT_REND_MULTI_BIN_CLDFB_DATA ) ) ) == NULL ) + { + assert( error == IVAS_ERR_OK ); + } + } +#endif + + decDummy->hEFAPdata = NULL; + decDummy->hCrendWrapper = NULL; + decDummy->hHrtfTD = NULL; + decDummy->hSpar = NULL; + decDummy->hoa_dec_mtx = NULL; + decDummy->hVBAPdata = NULL; // note: not used at the moment + decDummy->hMasa = NULL; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) + { + decDummy->hDiracDecBin[i] = NULL; + } +#else + decDummy->hDiracDecBin = NULL; +#endif + + decDummy->hDirACRend = NULL; + decDummy->hSpatParamRendCom = NULL; + decDummy->hQMetaData = NULL; + decDummy->hHrtfParambin = NULL; + decDummy->hHeadTrackData = NULL; + decDummy->hDirAC = NULL; + decDummy->hTcBuffer = NULL; + decDummy->hDecoderConfig->output_config = outConfig; + decDummy->nchan_transport = numTransChannels; + + if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL ) + { + decDummy->hHeadTrackData = malloc( sizeof( HEAD_TRACK_DATA ) ); + /* Initialise Rmat_prev to I, Rmat will be computed later */ + for ( i = 0; i < 3; i++ ) + { + set_zero( decDummy->hHeadTrackData->Rmat_prev[i], 3 ); + decDummy->hHeadTrackData->Rmat_prev[i][i] = 1.0f; + } + + set_zero( decDummy->hHeadTrackData->chEneIIR[0], MASA_FREQUENCY_BANDS ); + set_zero( decDummy->hHeadTrackData->chEneIIR[1], MASA_FREQUENCY_BANDS ); + set_zero( decDummy->hHeadTrackData->procChEneIIR[0], MASA_FREQUENCY_BANDS ); + set_zero( decDummy->hHeadTrackData->procChEneIIR[1], MASA_FREQUENCY_BANDS ); + for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) + { + decDummy->hHeadTrackData->Quaternions[i].w = 1.0f; + decDummy->hHeadTrackData->Quaternions[i].x = 0.0f; + decDummy->hHeadTrackData->Quaternions[i].y = 0.0f; + decDummy->hHeadTrackData->Quaternions[i].z = 0.0f; + } + decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; + decDummy->hHeadTrackData->lrSwitchedCurrent = 0; + decDummy->hHeadTrackData->lrSwitchedNext = 0; + decDummy->hHeadTrackData->OrientationTracker = (ivas_orient_trk_state_t *) malloc( sizeof( ivas_orient_trk_state_t ) ); + ivas_orient_trk_Init( decDummy->hHeadTrackData->OrientationTracker ); + + /* External orientations */ + if ( ( error = ivas_external_orientation_open( &( decDummy->hExtOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) + { + assert( error == IVAS_ERR_OK ); + } + + /* Combined orientations */ +#ifdef NONBE_UNIFIED_DECODING_PATHS + if ( ( error = ivas_combined_orientation_open( &( decDummy->hCombinedOrientationData ), sampleRate, num_subframes ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_combined_orientation_open( &( decDummy->hCombinedOrientationData ), num_subframes ) ) != IVAS_ERR_OK ) +#endif + { + assert( error == IVAS_ERR_OK ); + } + } + else + { + decDummy->hHeadTrackData = NULL; + decDummy->hExtOrientationData = NULL; + decDummy->hCombinedOrientationData = NULL; + } + + if ( enableRenderConfig ) + { + ivas_render_config_open( &decDummy->hRenderConfig ); + } + else + { + decDummy->hRenderConfig = NULL; + } + + /* get a default TC buffer, needed for some renderers */ + ivas_jbm_dec_tc_buffer_open( decDummy, TC_BUFFER_MODE_NONE, 0, 0, 0, 1 ); + + decDummy->renderer_type = RENDERER_DISABLE; + + return decDummy; +} +#endif + static ivas_error setRendInputActiveMasa( void *input, const AUDIO_CONFIG inConfig, @@ -2973,17 +3379,138 @@ static ivas_error setRendInputActiveMasa( } else { +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( ( error = initMasaExtRenderer( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { return error; } inputMasa->metadataHasBeenFed = false; +#else +#ifdef SPLIT_REND_WITH_HEAD_ROT + inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, numInChannels, outConfig, 0, rendCtx.pSplitRendWrapper ); +#else + inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, numInChannels, outConfig, 0 ); +#endif + + inputMasa->metadataHasBeenFed = false; + + if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif } return IVAS_ERR_OK; } +#ifndef NONBE_FIX_225_MASA_EXT_REND +static void freeDecoderDummy( + DecoderDummy **ppDecDummy ) +{ + int16_t i; + DecoderDummy *pDecDummy; + + if ( ppDecDummy == NULL || *ppDecDummy == NULL ) + { + return; + } + pDecDummy = *ppDecDummy; + + if ( pDecDummy->hDecoderConfig != NULL ) + { + free( pDecDummy->hDecoderConfig ); + } + + if ( pDecDummy->hHeadTrackData != NULL ) + { + if ( pDecDummy->hHeadTrackData->OrientationTracker != NULL ) + { + free( pDecDummy->hHeadTrackData->OrientationTracker ); + } + free( pDecDummy->hHeadTrackData ); + } + + if ( pDecDummy->hExtOrientationData != NULL ) + { + free( pDecDummy->hExtOrientationData ); + } + + if ( pDecDummy->hCombinedOrientationData != NULL ) + { + free( pDecDummy->hCombinedOrientationData ); + } + + ivas_render_config_close( &pDecDummy->hRenderConfig ); + + /* CLDFB handles */ + for ( i = 0; i < MAX_INTERN_CHANNELS; i++ ) + { + if ( pDecDummy->cldfbAnaDec[i] != NULL ) + { + deleteCldfb( &( pDecDummy->cldfbAnaDec[i] ) ); + } + } + + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + if ( pDecDummy->cldfbSynDec[i] != NULL ) + { + deleteCldfb( &( pDecDummy->cldfbSynDec[i] ) ); + } + } + + /* DirAC handle */ + ivas_dirac_rend_close( &( pDecDummy->hDirACRend ) ); + ivas_spat_hSpatParamRendCom_close( &( pDecDummy->hSpatParamRendCom ) ); + ivas_dirac_dec_close( &( pDecDummy->hDirAC ) ); + + /* Qmetadata handle */ + ivas_qmetadata_close( &pDecDummy->hQMetaData ); + + /* VBAP handle */ + vbap_free_data( &( pDecDummy->hVBAPdata ) ); + + /* HOA decoder matrix */ + if ( pDecDummy->hoa_dec_mtx != NULL ) + { + free( pDecDummy->hoa_dec_mtx ); + pDecDummy->hoa_dec_mtx = NULL; + } + + /* Parametric binaural renderer HRTF structure */ + if ( pDecDummy->hHrtfParambin != NULL ) + { + free( pDecDummy->hHrtfParambin ); + pDecDummy->hHrtfParambin = NULL; + } + +#ifdef SPLIT_REND_WITH_HEAD_ROT + /* Split binaural renderr handle */ + if ( pDecDummy->hSplitBinRend.hMultiBinCldfbData != NULL ) + { + free( pDecDummy->hSplitBinRend.hMultiBinCldfbData ); + pDecDummy->hSplitBinRend.hMultiBinCldfbData = NULL; + } +#endif + + /* Parametric binaural renderer handle */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + ivas_dirac_dec_close_binaural_data( pDecDummy->hDiracDecBin ); +#else + ivas_dirac_dec_close_binaural_data( &pDecDummy->hDiracDecBin ); +#endif + + /* TC buffer */ + ivas_jbm_dec_tc_buffer_close( &pDecDummy->hTcBuffer ); + + free( pDecDummy ); + pDecDummy = NULL; + + return; +} +#endif static void clearInputMasa( input_masa *inputMasa ) @@ -2995,9 +3522,14 @@ static void clearInputMasa( freeInputBaseBufferData( &inputMasa->bufferData ); masaPrerendClose( &inputMasa->hMasaPrerend ); +#ifdef NONBE_FIX_225_MASA_EXT_REND freeMasaExtRenderer( &inputMasa->hMasaExtRend ); +#endif initRendInputBase( &inputMasa->base, IVAS_AUDIO_CONFIG_INVALID, 0, rendCtx, NULL, 0 ); +#ifndef NONBE_FIX_225_MASA_EXT_REND + freeDecoderDummy( &inputMasa->decDummy ); +#endif return; } @@ -3220,10 +3752,15 @@ ivas_error IVAS_REND_Open( { initRendInputBase( &hIvasRend->inputsMasa[i].base, IVAS_AUDIO_CONFIG_INVALID, 0, getRendCtx( hIvasRend ), NULL, 0 ); +#ifndef NONBE_FIX_225_MASA_EXT_REND + hIvasRend->inputsMasa[i].decDummy = NULL; +#endif hIvasRend->inputsMasa[i].metadataHasBeenFed = false; hIvasRend->inputsMasa[i].bufferData = NULL; hIvasRend->inputsMasa[i].hMasaPrerend = NULL; +#ifdef NONBE_FIX_225_MASA_EXT_REND hIvasRend->inputsMasa[i].hMasaExtRend = NULL; +#endif } #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -7566,8 +8103,12 @@ static ivas_error renderActiveInputsSba( static void copyMasaMetadataToDiracRenderer( MASA_METADATA_FRAME *meta, +#ifdef NONBE_FIX_225_MASA_EXT_REND SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, const int16_t maxBin ) +#else + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom ) +#endif { int16_t band, sf, bin; int16_t meta_write_index; @@ -7581,7 +8122,11 @@ static void copyMasaMetadataToDiracRenderer( for ( band = 0; band < MASA_MAXIMUM_CODING_SUBBANDS; band++ ) { +#ifdef NONBE_FIX_225_MASA_EXT_REND for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1] && bin < maxBin; bin++ ) +#else + for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1]; bin++ ) +#endif { hSpatParamRendCom->azimuth[meta_write_index][bin] = (int16_t) meta->directional_meta[0].azimuth[sf][band]; hSpatParamRendCom->elevation[meta_write_index][bin] = (int16_t) meta->directional_meta[0].elevation[sf][band]; @@ -7607,6 +8152,119 @@ static void copyMasaMetadataToDiracRenderer( return; } +#ifndef NONBE_FIX_225_MASA_EXT_REND +static void renderMasaToMc( + input_masa *masaInput, + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t ch; + float *tmpBuffer[MAX_OUTPUT_CHANNELS], tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + tmpBuffer[ch] = tmpBuffer_buff[ch]; + } + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); + + if ( masaInput->decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + } + else + { + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + } + + accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); + + return; +} + + +static void renderMasaToSba( + input_masa *masaInput, + IVAS_REND_AudioBuffer outAudio ) +{ + int16_t ch; + float *tmpBuffer[MAX_OUTPUT_CHANNELS], tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + tmpBuffer[ch] = tmpBuffer_buff[ch]; + } + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); + + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + + accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); + + return; +} + + +static void renderMasaToBinaural( + input_masa *masaInput, +#ifdef SPLIT_REND_WITH_HEAD_ROT + IVAS_REND_AudioBuffer outAudio, + const int16_t is_split_rend_mode +#else + IVAS_REND_AudioBuffer outAudio +#endif +) +{ + int16_t ch; + float *tmpBuffer[MAX_OUTPUT_CHANNELS], tmpBuffer_buff[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) + { + tmpBuffer[ch] = tmpBuffer_buff[ch]; + } + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( is_split_rend_mode ) + { + int16_t sf, i, j; + COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; + pCombinedOrientationData = *masaInput->base.ctx.pCombinedOrientationData; + for ( sf = 1; sf < pCombinedOrientationData->num_subframes; sf++ ) + { + pCombinedOrientationData->Quaternions[sf] = pCombinedOrientationData->Quaternions[0]; + for ( i = 0; i < 3; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + pCombinedOrientationData->Rmat[sf][i][j] = pCombinedOrientationData->Rmat[0][i][j]; + } + } + } + } +#endif + + ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) ); + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( is_split_rend_mode ) + { + accumulateCLDFBArrayToBuffer( masaInput->decDummy->hSplitBinRend.hMultiBinCldfbData->Cldfb_RealBuffer_Binaural, masaInput->decDummy->hSplitBinRend.hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural, &outAudio ); + } + else + { +#endif + accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif + + return; +} +#endif static void renderMasaToMasa( input_masa *masaInput, @@ -7741,6 +8399,7 @@ static ivas_error renderInputMasa( IVAS_REND_AudioBuffer outAudio ) { IVAS_REND_AudioBuffer inAudio; +#ifdef NONBE_FIX_225_MASA_EXT_REND int16_t ch; int16_t maxBin; float *tmpBuffer[MAX_OUTPUT_CHANNELS]; @@ -7749,6 +8408,11 @@ static ivas_error renderInputMasa( int16_t cldfb2tdSampleFact; float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; +#endif +#else +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; +#endif #endif if ( !masaInput->metadataHasBeenFed ) @@ -7772,13 +8436,16 @@ static ivas_error renderInputMasa( /* Apply input gain to new audio */ v_multc( inAudio.data, masaInput->base.gain, inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); +#ifdef NONBE_FIX_225_MASA_EXT_REND maxBin = (int16_t) ( *masaInput->base.ctx.pOutSampleRate * INV_CLDFB_BANDWIDTH ); +#endif #ifdef NONBE_UNIFIED_DECODING_PATHS /* set combined orientation subframe info to start info */ ivas_combined_orientation_set_to_start_index( *( masaInput->base.ctx.pCombinedOrientationData ) ); #endif +#ifdef NONBE_FIX_225_MASA_EXT_REND if ( getAudioConfigType( outConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) { /* MASA prerendering path for MASA -> MASA */ @@ -7851,6 +8518,51 @@ static ivas_error renderInputMasa( accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); #ifdef SPLIT_REND_WITH_HEAD_ROT } +#endif +#else + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + renderMasaToMc( masaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + renderMasaToSba( masaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + switch ( outConfig ) + { + case IVAS_AUDIO_CONFIG_BINAURAL: + +#ifdef SPLIT_REND_WITH_HEAD_ROT + renderMasaToBinaural( masaInput, outAudio, 0 ); +#else + renderMasaToBinaural( masaInput, outAudio ); +#endif + break; + /* ToDo */ + // #ifdef FIX_196_REFACTOR_RENDERER_OUTPUT_CONFIG + // case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR: + // case IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB: + // #else + // case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: + // #endif + // renderMasaToBinauralRoom( masaInput, outConfig, outAudio ); + // break; +#ifdef SPLIT_REND_WITH_HEAD_ROT + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED: + case IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM: + renderMasaToBinaural( masaInput, outAudio, 1 ); + break; +#endif + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + renderMasaToMasa( masaInput, outAudio ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; #endif } @@ -7865,6 +8577,9 @@ static ivas_error renderActiveInputsMasa( int16_t i; input_masa *pCurrentInput; ivas_error error; +#ifndef NONBE_FIX_225_MASA_EXT_REND + int16_t sf_idx; +#endif for ( i = 0, pCurrentInput = hIvasRend->inputsMasa; i < RENDERER_MAX_MASA_INPUTS; ++i, ++pCurrentInput ) { @@ -7874,6 +8589,16 @@ static ivas_error renderActiveInputsMasa( continue; } +#ifndef NONBE_FIX_225_MASA_EXT_REND + if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && pCurrentInput->decDummy->hHeadTrackData != NULL ) + { + for ( sf_idx = 0; sf_idx < hIvasRend->num_subframes; ++sf_idx ) + { + pCurrentInput->decDummy->hHeadTrackData->Quaternions[sf_idx] = hIvasRend->headRotData.headPositions[sf_idx]; + pCurrentInput->decDummy->hHeadTrackData->Pos[sf_idx] = hIvasRend->headRotData.Pos[sf_idx]; + } + } +#endif if ( ( error = renderInputMasa( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) { return error; @@ -8574,6 +9299,7 @@ int32_t IVAS_REND_GetCntFramesLimited( } #endif +#ifdef NONBE_FIX_225_MASA_EXT_REND static ivas_error ivas_masa_ext_rend_dirac_rend_init( input_masa *inputMasa ) { @@ -9332,3 +10058,4 @@ static void freeMasaExtRenderer( return; } +#endif -- GitLab From 7612d77f667d4935b01bf6115290333c8090b6b9 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Wed, 25 Oct 2023 10:35:55 +0200 Subject: [PATCH 9/9] apply clang formatting --- lib_rend/lib_rend.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 791ef0cff0..7451583b65 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -667,17 +667,17 @@ static ivas_error validateOutputSampleRate( #endif #endif - /* Otherwise rendering to binaural, support the same set as IVAS decoder */ - switch ( sampleRate ) - { - case 8000: - case 16000: - case 32000: - case 48000: - return IVAS_ERR_OK; - } + /* Otherwise rendering to binaural, support the same set as IVAS decoder */ + switch ( sampleRate ) + { + case 8000: + case 16000: + case 32000: + case 48000: + return IVAS_ERR_OK; + } - return IVAS_ERR_INVALID_SAMPLING_RATE; + return IVAS_ERR_INVALID_SAMPLING_RATE; #ifdef NONBE_FIX_225_MASA_EXT_REND #ifdef SPLIT_REND_WITH_HEAD_ROT } -- GitLab