diff --git a/Workspace_msvc/lib_com.vcxproj b/Workspace_msvc/lib_com.vcxproj index 223f837a2e1685afeedb952a5137dab7b902b3a6..c28b36a1454bdb9bd3245a0ce9050d258602a13a 100644 --- a/Workspace_msvc/lib_com.vcxproj +++ b/Workspace_msvc/lib_com.vcxproj @@ -223,7 +223,7 @@ - + diff --git a/Workspace_msvc/lib_rend.vcxproj b/Workspace_msvc/lib_rend.vcxproj index 27d4a19a693cf86f60b3ce654912c65b4a7cbaa6..854c99a979e4ae274d9c002b2ca6686a96e75870 100644 --- a/Workspace_msvc/lib_rend.vcxproj +++ b/Workspace_msvc/lib_rend.vcxproj @@ -175,6 +175,7 @@ + diff --git a/Workspace_msvc/lib_rend.vcxproj.filters b/Workspace_msvc/lib_rend.vcxproj.filters index 942c63712f66e57821488868f81981d3042c56b0..2d1d7d46c0b86e517f6f5a9a35d3c5a985a4caac 100644 --- a/Workspace_msvc/lib_rend.vcxproj.filters +++ b/Workspace_msvc/lib_rend.vcxproj.filters @@ -32,30 +32,12 @@ rend_c - - rend_c - - - rend_c - - - rend_c - - - rend_c - rend_c rend_c - - rend_c - - - rend_c - rend_c @@ -83,15 +65,6 @@ rend_c - - rend_c - - - rend_c - - - rend_c - rend_c @@ -113,9 +86,6 @@ rend_c - - rend_c - rend_c @@ -134,24 +104,6 @@ rend_c - - rend_c - - - rend_c - - - rend_c - - - rend_c - - - rend_c - - - rend_c - rend_c @@ -161,30 +113,15 @@ rend_c - - rend_c - - - rend_c - - + rend_c - + rend_c - - rend_h - - - rend_h - - - rend_h - rend_h @@ -203,12 +140,6 @@ rend_h - - rend_h - - - rend_h - diff --git a/apps/renderer.c b/apps/renderer.c index 5ad0f8eecc1f9dbd3eaadcf46b89b9831ed695fb..0a29d20fc559d4ef1aa55ac0db0dad96edc61ada 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -702,6 +702,7 @@ int main( SplitFileReadWrite *hSplitRendFileReadWrite; int16_t delayNumSamples_temp; int32_t delayTimeScale_temp; + bool flushRendererLastFrame = false; int16_t numSamplesRead; int16_t delayNumSamples = -1; int16_t delayNumSamples_orig = 0; @@ -746,8 +747,11 @@ int main( CmdlnArgs args = parseCmdlnArgs( argc, argv ); - if ( args.nonDiegeticPan && !( ( args.inConfig.numAudioObjects == 0 && args.inConfig.multiChannelBuses[0].audioConfig == IVAS_AUDIO_CONFIG_MONO ) || - ( args.inConfig.numAudioObjects > 0 && args.inConfig.audioObjects[0].audioConfig == IVAS_AUDIO_CONFIG_OBA && args.inConfig.numAudioObjects == 1 ) ) ) + if ( args.nonDiegeticPan && + !( ( args.inConfig.numAudioObjects == 0 && + args.inConfig.multiChannelBuses[0].audioConfig == IVAS_AUDIO_CONFIG_MONO ) || + ( args.inConfig.numAudioObjects > 0 && + args.inConfig.audioObjects[0].audioConfig == IVAS_AUDIO_CONFIG_OBA && args.inConfig.numAudioObjects == 1 ) ) ) { fprintf( stderr, "\ninvalid configuration - non-diegetic panning requires mono or ISM1 input\n" ); goto cleanup; @@ -1068,7 +1072,10 @@ int main( { /* sanity check */ - if ( ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL ) && ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR ) && ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && !is_split_pre_rend_mode( &args ) ) + if ( ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL ) && + ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR ) && + ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && + !is_split_pre_rend_mode( &args ) ) { fprintf( stderr, "\nExternal Renderer Config is supported only when binaural output configurations is used as output OR when Split pre-rendering mode is enabled. Exiting. \n" ); goto cleanup; @@ -1374,7 +1381,7 @@ int main( } int16_t numOutChannels; - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_GetNumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError in IVAS_REND_NumOutChannels(): %s\n", ivas_error_to_string( error ) ); goto cleanup; @@ -1389,8 +1396,16 @@ int main( } } - if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) + if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) { + char *outFile = args.outMetadataFilePath; + + if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) + { + outFile = args.outputFilePath; + audioWriter = NULL; + } + if ( ( error = IVAS_REND_GetSplitRendBitstreamHeader( hIvasRend, &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection, &bitsBuffer.config.codec_frame_size_ms, &bitsBuffer.config.isar_frame_size_ms ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError in IVAS_REND_GetSplitRendBitstreamHeader()!\n" ); @@ -1403,36 +1418,15 @@ int main( goto cleanup; } - if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, args.outputFilePath, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK ) + if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, outFile, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK ) { - fprintf( stderr, "\nCould not open split rend metadata file %s\n", args.outputFilePath ); + fprintf( stderr, "\nCould not open split rend metadata file %s\n", outFile ); goto cleanup; } - audioWriter = NULL; } - else - { - if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) - { - if ( ( error = IVAS_REND_GetSplitRendBitstreamHeader( hIvasRend, &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection, &bitsBuffer.config.codec_frame_size_ms, &bitsBuffer.config.isar_frame_size_ms ) ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nError in IVAS_REND_GetSplitRendBitstreamHeader()!\n" ); - goto cleanup; - } - - if ( IVAS_REND_GetDelay( hIvasRend, &delayNumSamples_temp, &delayTimeScale_temp ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nUnable to get delay of renderer!\n" ); - goto cleanup; - } - - if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, args.outMetadataFilePath, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nCould not open split rend metadata file %s\n", args.outMetadataFilePath ); - goto cleanup; - } - } + if ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) + { if ( AudioFileWriter_open( &audioWriter, args.outputFilePath, args.sampleRate, numOutChannels ) != IVAS_ERR_OK ) { fprintf( stderr, "\nFailed to open file: %s\n", args.outputFilePath ); @@ -1558,16 +1552,23 @@ int main( if ( numSamplesRead == 0 ) { /* end of input data */ - break; + flushRendererLastFrame = true; } /* Convert from int to float and from interleaved to packed */ - convertInputBuffer( inpInt16Buffer, numSamplesRead, inBuffer.config.numSamplesPerChannel, num_in_channels, inFloatBuffer, inBuffer.config.is_cldfb, cldfbAna ); + if ( !flushRendererLastFrame ) + { + convertInputBuffer( inpInt16Buffer, numSamplesRead, inBuffer.config.numSamplesPerChannel, num_in_channels, inFloatBuffer, inBuffer.config.is_cldfb, cldfbAna ); + } + else + { + memset( inBuffer.data, 0, inBuffer.config.numChannels * inBuffer.config.numSamplesPerChannel * sizeof( float ) ); + } int16_t num_subframes, sf_idx; num_subframes = (int16_t) args.render_framesize; - if ( isCurrentFrameMultipleOf20ms ) + if ( isCurrentFrameMultipleOf20ms && !flushRendererLastFrame ) { IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer ); @@ -1679,7 +1680,7 @@ int main( } IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.multiChannelBuses[i].inputChannelIndex, numChannels ); - if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, mcIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, mcIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) ); goto cleanup; @@ -1694,7 +1695,7 @@ int main( { IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.audioObjects[i].inputChannelIndex, args.inConfig.numAudioObjects ); - if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) ); goto cleanup; @@ -1711,7 +1712,7 @@ int main( { IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.audioObjects[i].inputChannelIndex, 1 ); - if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) ); goto cleanup; @@ -1734,7 +1735,7 @@ int main( } IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.ambisonicsBuses[i].inputChannelIndex, numChannels ); - if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, sbaIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, sbaIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) ); goto cleanup; @@ -1750,13 +1751,13 @@ int main( } IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.masaBuses[i].inputChannelIndex, numChannels ); - if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, masaIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, masaIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK ) { fprintf( stderr, "IVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) ); goto cleanup; } - if ( isCurrentFrameMultipleOf20ms ) + if ( isCurrentFrameMultipleOf20ms && !flushRendererLastFrame ) { if ( masaReaders[i] != NULL ) { @@ -1819,7 +1820,7 @@ int main( zeroPad = delayNumSamples; } - if ( is_split_pre_rend_mode( &args ) ) + if ( is_split_pre_rend_mode( &args ) && !flushRendererLastFrame ) { if ( split_rend_write_bitstream_to_file( hSplitRendFileReadWrite, bitsBuffer.bits, &bitsBuffer.config.bitsRead, &bitsBuffer.config.bitsWritten ) != IVAS_ERR_OK ) @@ -1829,7 +1830,7 @@ int main( } } - if ( audioWriter != NULL ) + if ( audioWriter != NULL && !flushRendererLastFrame ) { if ( delayNumSamples * num_out_channels < outBufferSize ) { @@ -1850,7 +1851,7 @@ int main( bitsBuffer.config.bitsWritten = 0; /* Write MASA metadata for MASA outputs */ - if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA1 || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA2 ) + if ( !flushRendererLastFrame && ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA1 || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA2 ) ) { IVAS_REND_AudioConfigType inputType1; IVAS_REND_AudioConfigType inputType2; @@ -1922,7 +1923,8 @@ int main( } } - if ( ( args.inConfig.numAmbisonicsBuses > 0 || args.inConfig.numMultiChannelBuses > 0 || args.inConfig.numMasaBuses > 0 ) && args.inConfig.numAudioObjects > 0 ) + if ( ( args.inConfig.numAmbisonicsBuses > 0 || args.inConfig.numMultiChannelBuses > 0 || args.inConfig.numMasaBuses > 0 ) && + args.inConfig.numAudioObjects > 0 ) { inputType2 = IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED; if ( ( error = IVAS_REND_MergeMasaMetadata( hIvasRend, &hMetaOutput, inputType1, inputType2 ) ) != IVAS_ERR_OK ) @@ -1939,6 +1941,13 @@ int main( } } + /* no new input was actually read, only delay buffers were flushed + * therefore this is not a real frame */ + if ( flushRendererLastFrame ) + { + break; + } + frame++; if ( !args.quietModeEnabled ) { @@ -1951,12 +1960,13 @@ int main( #endif } - /* add zeros at the end to have equal length of synthesized signals */ + /* add zeros at the end to have equal length of synthesized signals + * the output buffer will contain either leftover input samples from delay aligned inputs + * or zeros for padding */ if ( audioWriter != NULL ) { for ( zeroPadToWrite = zeroPad; zeroPadToWrite > frameSize_smpls; zeroPadToWrite -= frameSize_smpls ) { - memset( outInt16Buffer, 0, outBufferSize * sizeof( int16_t ) ); if ( ( error = AudioFileWriter_write( audioWriter, outInt16Buffer, outBufferSize ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nOutput audio file writer error\n" ); @@ -1964,7 +1974,6 @@ int main( } } - memset( outInt16Buffer, 0, zeroPadToWrite * outBuffer.config.numChannels * sizeof( int16_t ) ); if ( ( error = AudioFileWriter_write( audioWriter, outInt16Buffer, zeroPadToWrite * outBuffer.config.numChannels ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nOutput audio file writer error\n" ); @@ -1973,9 +1982,10 @@ int main( zeroPadToWrite = 0; } - if ( args.inConfig.numAudioObjects != 0 && ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) ) + if ( args.inConfig.numAudioObjects != 0 && + ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) ) { - fprintf( stdout, "\n\nMetadata delayed %d subframes\n\n", (int16_t) round( args.syncMdDelay / ( 1000 / IVAS_NUM_FRAMES_PER_SEC / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ) ) ); + fprintf( stdout, "\n\nMetadata delayed %d subframes\n\n", (int16_t) round( args.syncMdDelay / ( 1000.f / IVAS_NUM_FRAMES_PER_SEC / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ) ) ); } if ( !args.quietModeEnabled && args.delayCompensationEnabled ) @@ -3305,7 +3315,8 @@ static void parseObjectPosition( *positionDuration = (uint16_t) strtol( line, &endptr, 10 ); readNextMetadataChunk( line, "\n" ); - read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f", &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] ); + read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f", + &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] ); if ( read_values < 2 ) { diff --git a/lib_isar/lib_isar_pre_rend.c b/lib_isar/lib_isar_pre_rend.c index 09323f6eedf08d3900bde17cb70a27a71e8be582..d24bc94de7411889db57dd0757fbd50945d41dcf 100644 --- a/lib_isar/lib_isar_pre_rend.c +++ b/lib_isar/lib_isar_pre_rend.c @@ -53,8 +53,8 @@ *------------------------------------------------------------------------*/ ivas_error ISAR_PRE_REND_open( - SPLIT_REND_WRAPPER *hSplitBinRend, /* i/o: Split renderer pre-renerer handle */ - ISAR_SPLIT_REND_CONFIG_DATA *pSplitRendConfig, /* i/o: Split renderer pre-renerer config */ + SPLIT_REND_WRAPPER *hSplitBinRend, /* i/o: Split renderer pre-renderer handle */ + ISAR_SPLIT_REND_CONFIG_DATA *pSplitRendConfig, /* i/o: Split renderer pre-renderer config */ const int32_t output_Fs, /* i : output sampling rate */ const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time doamin input */ const int16_t pcm_out_flag, /* i : Flag to indicate PCM output */ @@ -94,9 +94,7 @@ ivas_error ISAR_PRE_REND_open( isCldfbNeeded = 1; } - hSplitBinRend->hCldfbHandles = NULL; - - if ( isCldfbNeeded ) + if ( isCldfbNeeded && hSplitBinRend->hCldfbHandles == NULL ) { if ( ( hSplitBinRend->hCldfbHandles = (CLDFB_HANDLES_WRAPPER_HANDLE) malloc( sizeof( CLDFB_HANDLES_WRAPPER ) ) ) == NULL ) { @@ -286,7 +284,7 @@ ivas_error ISAR_PRE_REND_MultiBinToSplitBinaural( const int16_t max_bands, /* i : CLDFB bands */ float *output[], /* i/o: PCM in/out buffer */ const int16_t low_res_pre_rend_rot, /* i : low time resolution pre-renderer flag */ - const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time doamin input */ + const int16_t cldfb_in_flag, /* i : Flag to indicate CLDFB or time domain input */ const int16_t pcm_out_flag, /* i : Flag to indicate PCM output */ const int16_t ro_md_flag /* i : Flag to indicate real only metadata for yaw */ ) diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index bb936d4b14b85a1eac27874a27c08884f57de35b..2268f62adda1a484ffe8b1edd58821426fbca016 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -1582,6 +1582,40 @@ void ivas_rend_closeCldfbRend( CLDFB_REND_WRAPPER *pCldfbRend ); +/*----------------------------------------------------------------------------------* + * Time domain ring buffer prototypes + *----------------------------------------------------------------------------------*/ + +ivas_error ivas_TD_RINGBUF_Open( + TD_RINGBUF_HANDLE *ph, /* i/o: Ring buffer handle */ + const uint32_t capacity_per_channel, /* i : Number of samples stored per channel */ + const uint16_t num_channels /* i : Number of channels */ +); + +void ivas_TD_RINGBUF_Close( + TD_RINGBUF_HANDLE *ph /* i/o: Ring buffer handle */ +); + +void ivas_TD_RINGBUF_Push( + TD_RINGBUF_HANDLE h, /* i/o: Ring buffer handle */ + const float *data, /* i : Input data */ + const uint32_t num_samples_per_channel /* i : Number of samples per channel to store */ +); + +void ivas_TD_RINGBUF_PushZeros( + TD_RINGBUF_HANDLE h, /* i/o: Ring buffer handle */ + const uint32_t num_samples_per_channel /* i : Number of zeros per channel to store */ +); + +void ivas_TD_RINGBUF_Pop( + TD_RINGBUF_HANDLE h, /* i/o: Ring buffer handle */ + float *data, /* i : Output data */ + const uint32_t num_samples_per_channel /* i : Number of samples per channel to retrieve*/ +); + +uint32_t ivas_TD_RINGBUF_Size( + const TD_RINGBUF_HANDLE h /* i : Ring buffer handle */ +); /* clang-format on */ diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index e990c336f69e0d8c2e45520c92754ff21f3a992d..69fcb30fb803f590f19fb50517892a98ecc97f9c 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -1371,6 +1371,22 @@ typedef struct } CLDFB_REND_WRAPPER; +/*----------------------------------------------------------------------------------* + * Time domain ring buffer structure + *----------------------------------------------------------------------------------*/ + +typedef struct +{ + float *data; /* samples in interleaved layout */ + uint32_t capacity; + uint16_t num_channels; + uint32_t write_pos; + uint32_t read_pos; + int16_t is_full; + +} TD_RINGBUF_DATA, *TD_RINGBUF_HANDLE; + + /*----------------------------------------------------------------------------------* * MASA external renderer structure *----------------------------------------------------------------------------------*/ diff --git a/lib_rend/ivas_td_ring_buffer.c b/lib_rend/ivas_td_ring_buffer.c new file mode 100644 index 0000000000000000000000000000000000000000..5c9c6089bbc41150c6469cd8240a1df3741b38ce --- /dev/null +++ b/lib_rend/ivas_td_ring_buffer.c @@ -0,0 +1,288 @@ +/****************************************************************************************************** + + (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository. All Rights Reserved. + + This software is protected by copyright law and by international treaties. + The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, + Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., + Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, + Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other + contributors to this repository retain full ownership rights in their respective contributions in + the software. This notice grants no license of any kind, including but not limited to patent + license, nor is any license granted by implication, estoppel or otherwise. + + Contributors are required to enter into the IVAS codec Public Collaboration agreement before making + contributions. + + This software is provided "AS IS", without any express or implied warranties. The software is in the + development stage. It is intended exclusively for experts who have experience with such software and + solely for the purpose of inspection. All implied warranties of non-infringement, merchantability + and fitness for a particular purpose are hereby disclaimed and excluded. + + Any dispute, controversy or claim arising under or in relation to providing this software shall be + submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in + accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and + the United Nations Convention on Contracts on the International Sales of Goods. + +*******************************************************************************************************/ + +#include +#include +#include "ivas_error_utils.h" +#include "ivas_prot_rend.h" +#include "options.h" +#include "wmc_auto.h" + + +/*-----------------------------------------------------------------------* + * Local function prototypes + *-----------------------------------------------------------------------*/ + +static uint32_t ivas_td_ringbuf_total_size( + TD_RINGBUF_HANDLE h ) +{ + if ( h->is_full ) + { + return h->capacity; + } + + if ( h->read_pos <= h->write_pos ) + { + return h->write_pos - h->read_pos; + } + /* else wrap around */ + return h->write_pos + h->capacity - h->read_pos; +} + + +static int16_t ivas_td_ringbuf_has_space_for_num_samples( + TD_RINGBUF_HANDLE h, + const uint32_t num_samples ) +{ + return (int16_t) ( ivas_td_ringbuf_total_size( h ) + num_samples <= h->capacity ); +} + + +/*-----------------------------------------------------------------------* + * Global function definitions + *-----------------------------------------------------------------------*/ + +/*---------------------------------------------------------------------* + * ivas_TD_RINGBUF_Open() + * + * Allocate a ring buffer for TD data with the given capacity of TD samples per channel. + * + * May return IVAS_ERR_FAILED_ALLOC on failed allocation, or IVAS_ERR_OK otherwise. + *---------------------------------------------------------------------*/ + +ivas_error ivas_TD_RINGBUF_Open( + TD_RINGBUF_HANDLE *ph, /* i/o: Ring buffer handle */ + const uint32_t capacity_per_channel, /* i : Number of samples stored per channel */ + const uint16_t num_channels /* i : Number of channels */ +) +{ + TD_RINGBUF_HANDLE h; + uint32_t capacity; + + capacity = capacity_per_channel * num_channels; + + h = malloc( sizeof( TD_RINGBUF_DATA ) ); + if ( h == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Failed to allocate memory for TD ring buffer\n" ); + } + h->data = NULL; + h->capacity = 0; + h->num_channels = num_channels; + h->write_pos = 0; + h->read_pos = 0; + h->is_full = 0; + *ph = h; + + h->data = malloc( capacity * sizeof( float ) ); + if ( h->data == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Failed to allocate memory for TD ring buffer\n" ); + } + h->capacity = capacity; + + return IVAS_ERR_OK; +} + + +/*---------------------------------------------------------------------* + * ivas_TD_RINGBUF_Close() + * + * Dellocate TD ring buffer. The given handle will be set to NULL. + + *---------------------------------------------------------------------*/ + +void ivas_TD_RINGBUF_Close( + TD_RINGBUF_HANDLE *ph /* i/o: Ring buffer handle */ +) +{ + TD_RINGBUF_HANDLE h; + + if ( ph == NULL ) + { + return; + } + h = *ph; + + if ( h == NULL ) + { + return; + } + + if ( h->data != NULL ) + { + free( h->data ); + } + + free( h ); + *ph = NULL; + + return; +} + + +/*---------------------------------------------------------------------* + * ivas_TD_RINGBUF_Push() + * + * Push samples onto the back of the TD ring buffer. + * Returns total number of buffered samples (includes number of channels) + *---------------------------------------------------------------------*/ + +void ivas_TD_RINGBUF_Push( + TD_RINGBUF_HANDLE h, /* i/o: Ring buffer handle */ + const float *data, /* i : Input data */ + const uint32_t num_samples_per_channel /* i : Number of samples per channel to store */ +) +{ + uint32_t s; + uint16_t c; + + assert( ivas_td_ringbuf_has_space_for_num_samples( h, num_samples_per_channel * h->num_channels ) ); + + for ( s = 0; s < num_samples_per_channel; ++s ) + { + for ( c = 0; c < h->num_channels; ++c ) + { + h->data[h->write_pos] = *( data + c * num_samples_per_channel + s ); + ++h->write_pos; + + if ( h->write_pos == h->capacity ) + { + h->write_pos = 0; + } + } + } + + if ( h->read_pos == h->write_pos ) + { + h->is_full = 1; + } + + return; +} + + +/*---------------------------------------------------------------------* + * ivas_TD_RINGBUF_PushZeros() + * + * Push zero samples onto the back of the TD ring buffer. + *---------------------------------------------------------------------*/ + +void ivas_TD_RINGBUF_PushZeros( + TD_RINGBUF_HANDLE h, /* i/o: Ring buffer handle */ + const uint32_t num_samples_per_channel /* i : Number of zeros per channel to store */ +) +{ + uint32_t s; + uint16_t c; + + assert( ivas_td_ringbuf_has_space_for_num_samples( h, num_samples_per_channel * h->num_channels ) ); + if ( !num_samples_per_channel ) + { + return; + } + + for ( s = 0; s < num_samples_per_channel; ++s ) + { + for ( c = 0; c < h->num_channels; ++c ) + { + h->data[h->write_pos] = 0.f; + ++h->write_pos; + + if ( h->write_pos == h->capacity ) + { + h->write_pos = 0; + } + } + } + + if ( h->read_pos == h->write_pos ) + { + h->is_full = 1; + } + + return; +} + + +/*---------------------------------------------------------------------* + * ivas_TD_RINGBUF_Pop() + * + * Pop samples from the front of the TD ring buffer. + *---------------------------------------------------------------------*/ + +void ivas_TD_RINGBUF_Pop( + TD_RINGBUF_HANDLE h, /* i/o: Ring buffer handle */ + float *data, /* i : Output data */ + const uint32_t num_samples_per_channel /* i : Number of samples per channel to retrieve */ +) +{ + uint32_t s; + uint16_t c; + + assert( ivas_td_ringbuf_total_size( h ) >= num_samples_per_channel * h->num_channels ); + + for ( s = 0; s < num_samples_per_channel; ++s ) + { + for ( c = 0; c < h->num_channels; ++c ) + { + *( data + c * num_samples_per_channel + s ) = h->data[h->read_pos]; + ++h->read_pos; + + if ( h->read_pos == h->capacity ) + { + h->read_pos = 0; + } + } + } + + if ( h->is_full ) + { + h->is_full = 0; + } + + return; +} + + +/*---------------------------------------------------------------------* + * ivas_TD_RINGBUF_Size() + * + * Returns number of buffered samples per channel. + *---------------------------------------------------------------------*/ + +uint32_t ivas_TD_RINGBUF_Size( + const TD_RINGBUF_HANDLE h /* i : Ring buffer handle */ +) +{ + return ivas_td_ringbuf_total_size( h ) / (uint32_t) h->num_channels; +} diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 214712487a239a0a3ebfb390e1abe5b1cef7459f..cdf0ce31b1333c34dcf3ef64ad24c5c615da2820 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -81,6 +81,7 @@ typedef struct typedef struct { const int32_t *pOutSampleRate; + const int32_t *pMaxGlobalDelayNs; const AUDIO_CONFIG *pOutConfig; const LSSETUP_CUSTOM_STRUCT *pCustomLsOut; const EFAP_WRAPPER *pEfapOutWrapper; @@ -97,9 +98,11 @@ typedef struct AUDIO_CONFIG inConfig; IVAS_REND_InputId id; IVAS_REND_AudioBuffer inputBuffer; + TD_RINGBUF_HANDLE delayBuffer; float gain; /* Linear, not in dB */ rendering_context ctx; int32_t numNewSamplesPerChannel; /* Used to keep track how much new audio was fed before rendering current frame */ + int32_t delayNumSamples; } input_base; typedef struct @@ -189,6 +192,7 @@ typedef struct hrtf_handles struct IVAS_REND { int32_t sampleRateOut; + int32_t maxGlobalDelayNs; IVAS_LIMITER_HANDLE hLimiter; #ifdef DEBUGGING @@ -204,11 +208,12 @@ struct IVAS_REND AUDIO_CONFIG outputConfig; EFAP_WRAPPER efapOutWrapper; IVAS_LSSETUP_CUSTOM_STRUCT customLsOut; + + int16_t splitRendBFI; SPLIT_REND_WRAPPER *splitRendWrapper; IVAS_REND_AudioBuffer splitRendEncBuffer; IVAS_REND_HeadRotData headRotData; - int16_t splitRendBFI; EXTERNAL_ORIENTATION_HANDLE hExternalOrientationData; COMBINED_ORIENTATION_HANDLE hCombinedOrientationData; @@ -262,6 +267,13 @@ static void freeInputBaseBufferData( return; } +static int16_t latencyNsToSamples( + int32_t sampleRate, + int32_t latency_ns ) +{ + return (int16_t) roundf( (float) ( latency_ns ) * ( sampleRate / 1000000000.f ) ); +} + static ivas_error allocateMcLfeDelayBuffer( float **lfeDelayBuffer, const int16_t data_size ) @@ -369,7 +381,7 @@ static void copyBufferToCLDFBarray( static void accumulateCLDFBArrayToBuffer( float re[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float im[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - IVAS_REND_AudioBuffer *buffer ) + const IVAS_REND_AudioBuffer *buffer ) { uint32_t smplIdx, slotIdx; uint32_t numCldfbSamples, num_bands; @@ -1162,6 +1174,8 @@ static void initRendInputBase( inputBase->gain = 1.0f; inputBase->ctx = rendCtx; inputBase->numNewSamplesPerChannel = 0; + inputBase->delayNumSamples = 0; + inputBase->delayBuffer = NULL; inputBase->inputBuffer.config.numSamplesPerChannel = 0; inputBase->inputBuffer.config.numChannels = 0; @@ -1210,6 +1224,7 @@ static rendering_context getRendCtx( /* Note: when refactoring this, always take the ADDRESS of a member of the * renderer struct, so that the context stores a POINTER to the member, even * if the member is a pointer or handle itself. */ + ctx.pMaxGlobalDelayNs = &hIvasRend->maxGlobalDelayNs; ctx.pOutConfig = &hIvasRend->outputConfig; ctx.pOutSampleRate = &hIvasRend->sampleRateOut; ctx.pCustomLsOut = &hIvasRend->customLsOut; @@ -1252,6 +1267,272 @@ static bool isIoConfigPairSupported( } +static int32_t getRendInputDelayIsm( + const input_ism *inputIsm, + bool splitPreRendCldfb ) +{ + int32_t latency_ns; + latency_ns = 0; + (void) ( splitPreRendCldfb ); /* unused */ + + /* set the rendering delay in InputBase */ + latency_ns = max( latency_ns, + inputIsm->tdRendWrapper.binaural_latency_ns ); + if ( inputIsm->crendWrapper != NULL ) + { + latency_ns = max( latency_ns, + inputIsm->crendWrapper->binaural_latency_ns ); + } + + return latency_ns; +} + + +static void setRendInputDelayIsm( + void *input, + bool splitPreRendCldfb ) +{ + input_ism *inputIsm; + inputIsm = (input_ism *) input; + + inputIsm->base.delayNumSamples = latencyNsToSamples( *inputIsm->base.ctx.pOutSampleRate, + getRendInputDelayIsm( inputIsm, splitPreRendCldfb ) ); +} + + +static int32_t getRendInputDelayMc( + const input_mc *inputMc, + bool splitPreRendCldfb ) +{ + int32_t latency_ns; + latency_ns = 0; + (void) ( splitPreRendCldfb ); /* unused */ + + latency_ns = max( latency_ns, + inputMc->tdRendWrapper.binaural_latency_ns ); + if ( inputMc->crendWrapper != NULL ) + { + latency_ns = max( latency_ns, + inputMc->crendWrapper->binaural_latency_ns ); + } + + return latency_ns; +} + + +static void setRendInputDelayMc( + void *input, + bool splitPreRendCldfb ) +{ + input_mc *inputMc; + inputMc = (input_mc *) input; + + inputMc->base.delayNumSamples = latencyNsToSamples( *inputMc->base.ctx.pOutSampleRate, + getRendInputDelayMc( inputMc, splitPreRendCldfb ) ); +} + + +static int32_t getRendInputDelaySba( + const input_sba *inputSba, + bool splitPreRendCldfb ) +{ + int32_t latency_ns; + latency_ns = 0; + + if ( inputSba->cldfbRendWrapper.hCldfbRend != NULL ) + { + latency_ns = max( latency_ns, + inputSba->cldfbRendWrapper.binaural_latency_ns + + ( splitPreRendCldfb ? 0 : IVAS_FB_DEC_DELAY_NS ) ); + } + if ( inputSba->crendWrapper != NULL ) + { + latency_ns = max( latency_ns, + inputSba->crendWrapper->binaural_latency_ns ); + } + + return latency_ns; +} + + +static void setRendInputDelaySba( + void *input, + bool splitPreRendCldfb ) +{ + input_sba *inputSba; + inputSba = (input_sba *) input; + + inputSba->base.delayNumSamples = latencyNsToSamples( *inputSba->base.ctx.pOutSampleRate, + getRendInputDelaySba( inputSba, splitPreRendCldfb ) ); +} + + +static int32_t getRendInputDelayMasa( + const input_masa *inputMasa, + bool splitPreRendCldfb ) +{ + int32_t latency_ns; + + latency_ns = 0; + + if ( ( inputMasa->base.inConfig == IVAS_AUDIO_CONFIG_MASA1 && *inputMasa->base.ctx.pOutConfig == IVAS_AUDIO_CONFIG_MONO ) || + ( getAudioConfigType( *inputMasa->base.ctx.pOutConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) ) + { + return 0; /* no delay */ + } + else + { + /* no delay applied for split rendering */ + latency_ns = max( latency_ns, + (int32_t) ( ( splitPreRendCldfb ? 0 : (float) IVAS_FB_DEC_DELAY_NS + 0.5f ) ) ); + } + return latency_ns; +} + + +static void setRendInputDelayMasa( + void *input, + bool splitPreRendCldfb ) +{ + input_masa *inputMasa; + inputMasa = (input_masa *) input; + + inputMasa->base.delayNumSamples = latencyNsToSamples( *inputMasa->base.ctx.pOutSampleRate, + getRendInputDelayMasa( inputMasa, splitPreRendCldfb ) ); +} + + +static int32_t getMaxGlobalDelayNs( IVAS_REND_CONST_HANDLE hIvasRend ) +{ + int16_t i; + int32_t latency_ns; + int32_t max_latency_ns; + bool splitPreRendCldfb; + + max_latency_ns = 0; + /*assumes that input has been added which means codec has been set to either lcld or lc3plus (even if render config specified default)*/ + if ( hIvasRend->hRendererConfig != NULL ) + { + splitPreRendCldfb = ( hIvasRend->hRendererConfig->split_rend_config.codec == ISAR_SPLIT_REND_CODEC_LCLD ); + } + else + { + splitPreRendCldfb = false; + } + + /* Compute the maximum delay across all inputs */ + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; i++ ) + { + if ( hIvasRend->inputsIsm[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + latency_ns = getRendInputDelayIsm( &hIvasRend->inputsIsm[i], splitPreRendCldfb ); + max_latency_ns = max( max_latency_ns, latency_ns ); + } + } + + for ( i = 0; i < RENDERER_MAX_MC_INPUTS; i++ ) + { + if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + latency_ns = getRendInputDelayMc( &hIvasRend->inputsMc[i], splitPreRendCldfb ); + max_latency_ns = max( max_latency_ns, latency_ns ); + } + } + + for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; i++ ) + { + if ( hIvasRend->inputsSba[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + latency_ns = getRendInputDelaySba( &hIvasRend->inputsSba[i], splitPreRendCldfb ); + max_latency_ns = max( max_latency_ns, latency_ns ); + } + } + + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) + { + if ( hIvasRend->inputsMasa[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + latency_ns = getRendInputDelayMasa( &hIvasRend->inputsMasa[i], splitPreRendCldfb ); + max_latency_ns = max( max_latency_ns, latency_ns ); + } + } + + return max_latency_ns; +} + + +static void setMaxGlobalDelayNs( IVAS_REND_HANDLE hIvasRend ) +{ + hIvasRend->maxGlobalDelayNs = getMaxGlobalDelayNs( (IVAS_REND_CONST_HANDLE) hIvasRend ); +} + +static ivas_error alignInputDelay( + input_base *inputBase, + const IVAS_REND_ReadOnlyAudioBuffer inputAudio, + const int32_t maxGlobalDelayNs, + const int32_t sampleRateOut, + const int16_t cldfb2tdSampleFact, + const bool flushInputs ) +{ + ivas_error error; + input_ism *inputIsm; + int16_t maxGlobalDelaySamples; + int32_t numSamplesToPush, numSamplesToPop; + uint32_t ringBufferSize, preDelay; + + maxGlobalDelaySamples = latencyNsToSamples( sampleRateOut, maxGlobalDelayNs ); + maxGlobalDelaySamples *= cldfb2tdSampleFact; + + /* check if we need to open the delay buffer */ + if ( inputBase->delayBuffer == NULL ) + { + /* buffer has to accomodate maxGlobalDelaySamples + 2 * frameSize */ + ringBufferSize = maxGlobalDelaySamples + 2 * inputAudio.config.numSamplesPerChannel; + + /* pre delay for this input is maximum delay - input delay */ + preDelay = maxGlobalDelaySamples - inputBase->delayNumSamples * cldfb2tdSampleFact; + + if ( preDelay > 0 ) + { + if ( ( error = ivas_TD_RINGBUF_Open( &inputBase->delayBuffer, ringBufferSize, inputAudio.config.numChannels ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* for the first frame we need to push zeros to align the input delay to the global delay + * and then push a frame of actual data */ + ivas_TD_RINGBUF_PushZeros( inputBase->delayBuffer, preDelay ); + + /* for ISM inputs, ensure the metadata sync delay is updated */ + if ( getAudioConfigType( inputBase->inConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) + { + inputIsm = (input_ism *) inputBase; + inputIsm->ism_metadata_delay_ms = maxGlobalDelayNs / 1e6f; + } + } + } + + if ( inputBase->delayBuffer != NULL ) + { + /* push in the new input data and pop to retrieve a complete input frame + * if we are flushing the inputs, we don't push in any new data */ + numSamplesToPush = flushInputs ? 0 : inputAudio.config.numSamplesPerChannel; + numSamplesToPop = flushInputs ? ivas_TD_RINGBUF_Size( inputBase->delayBuffer ) : (uint32_t) inputAudio.config.numSamplesPerChannel; + + ivas_TD_RINGBUF_Push( inputBase->delayBuffer, inputAudio.data, numSamplesToPush ); + ivas_TD_RINGBUF_Pop( inputBase->delayBuffer, inputBase->inputBuffer.data, numSamplesToPop ); + } + else + { + /* delay buffer isn't open - we don't need it */ + mvr2r( inputAudio.data, + inputBase->inputBuffer.data, + inputAudio.config.numSamplesPerChannel * inputAudio.config.numChannels ); + } + + return IVAS_ERR_OK; +} + static ivas_error initIsmMasaRendering( input_ism *inputIsm, const int32_t inSampleRate ) @@ -1387,6 +1668,8 @@ static void clearInputIsm( rendCtx = inputIsm->base.ctx; freeInputBaseBufferData( &inputIsm->base.inputBuffer.data ); + ivas_TD_RINGBUF_Close( &inputIsm->base.delayBuffer ); + initRendInputBase( &inputIsm->base, IVAS_AUDIO_CONFIG_INVALID, 0, rendCtx, NULL, 0 ); /* Free input's internal handles */ @@ -2157,7 +2440,7 @@ static ivas_error initMcBinauralRendering( /* determine binaural delay ( used for aligning LFE to output signal ) */ binauralDelayNs = max( ( inputMc->crendWrapper != NULL ) ? inputMc->crendWrapper->binaural_latency_ns : 0, inputMc->tdRendWrapper.binaural_latency_ns ); - inputMc->binauralDelaySmp = (int16_t) roundf( (float) binauralDelayNs * *inputMc->base.ctx.pOutSampleRate / 1000000000.f ); + inputMc->binauralDelaySmp = latencyNsToSamples( *inputMc->base.ctx.pOutSampleRate, binauralDelayNs ); if ( inputMc->binauralDelaySmp > MAX_BIN_DELAY_SAMPLES ) { @@ -2357,6 +2640,8 @@ static void clearInputMc( freeMcLfeDelayBuffer( &inputMc->lfeDelayBuffer ); freeInputBaseBufferData( &inputMc->bufferData ); + ivas_TD_RINGBUF_Close( &inputMc->base.delayBuffer ); + initRendInputBase( &inputMc->base, IVAS_AUDIO_CONFIG_INVALID, 0, rendCtx, NULL, 0 ); /* Free input's internal handles */ @@ -2644,7 +2929,7 @@ static ivas_error setRendInputActiveSba( return error; } - return error; + return IVAS_ERR_OK; } @@ -2656,6 +2941,7 @@ static void clearInputSba( rendCtx = inputSba->base.ctx; freeInputBaseBufferData( &inputSba->bufferData ); + ivas_TD_RINGBUF_Close( &inputSba->base.delayBuffer ); initRendInputBase( &inputSba->base, IVAS_AUDIO_CONFIG_INVALID, 0, rendCtx, NULL, 0 ); @@ -2735,6 +3021,7 @@ static void clearInputMasa( rendCtx = inputMasa->base.ctx; freeInputBaseBufferData( &inputMasa->bufferData ); + ivas_TD_RINGBUF_Close( &inputMasa->base.delayBuffer ); masaPrerendClose( &inputMasa->hMasaPrerend ); freeMasaExtRenderer( &inputMasa->hMasaExtRend ); @@ -2804,7 +3091,7 @@ ivas_error IVAS_REND_Open( hIvasRend->num_subframes = num_subframes; /* Initialize limiter */ - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_GetNumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { return error; } @@ -2863,6 +3150,9 @@ ivas_error IVAS_REND_Open( isar_init_split_rend_handles( hIvasRend->splitRendWrapper ); } hIvasRend->splitRendEncBuffer.data = NULL; + hIvasRend->splitRendEncBuffer.config.is_cldfb = 0; + hIvasRend->splitRendEncBuffer.config.numChannels = 0; + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = 0; for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) { @@ -3058,7 +3348,7 @@ ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( hIvasRend->customLsOut = makeCustomLsSetup( layout ); /* Re-initialize limiter - number of output channels may have changed */ - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_GetNumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { return error; } @@ -3116,12 +3406,12 @@ ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( /*-------------------------------------------------------------------* - * IVAS_REND_NumOutChannels() + * IVAS_REND_GetNumOutChannels() * * *-------------------------------------------------------------------*/ -ivas_error IVAS_REND_NumOutChannels( +ivas_error IVAS_REND_GetNumOutChannels( IVAS_REND_CONST_HANDLE hIvasRend, int16_t *numOutChannels ) { @@ -3396,10 +3686,15 @@ static int16_t getCldfbRendFlag( const IVAS_REND_AudioConfigType new_configType ) { int16_t i; - int16_t numMasaInputs = 0, numSbaInputs = 0, numIsmInputs = 0, numMcInputs = 0; + int16_t numMasaInputs = 0, numSbaInputs = 0; int16_t isCldfbRend; isCldfbRend = 0; + /* This function is called during three different phases of renderer processing: + * - IVAS_REND_AddInput() + * - IVAS_REND_FeedRenderConfig() + * - IVAS_REND_GetSplitBinauralBitstream() + * Only the last case can assume all inputs are present for the current frame to be rendered */ if ( hIvasRend->hRendererConfig != NULL ) { for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) @@ -3410,20 +3705,7 @@ static int16_t getCldfbRendFlag( { numSbaInputs += ( hIvasRend->inputsSba[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS ) ? 0 : 1; } - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) - { - numIsmInputs += ( hIvasRend->inputsIsm[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED ) ? 0 : 1; - } - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; ++i ) - { - numMcInputs += ( hIvasRend->inputsMc[i].base.inConfig == IVAS_AUDIO_CONFIG_INVALID && new_configType != IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED ) ? 0 : 1; - } - - if ( numIsmInputs > 0 || numMcInputs > 0 ) - { - isCldfbRend = 0; - } - else if ( ( numMasaInputs > 0 ) || ( numSbaInputs > 0 && hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_BIN_RENDERER_TYPE_FASTCONV ) ) + if ( ( numMasaInputs > 0 ) || ( numSbaInputs > 0 && hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_BIN_RENDERER_TYPE_FASTCONV ) ) { isCldfbRend = 1; } @@ -3433,12 +3715,12 @@ static int16_t getCldfbRendFlag( } /*------------------------------------------------------------------------- - * Function ivas_pre_rend_init() + * Function isar_pre_rend_init() * * *------------------------------------------------------------------------*/ -static ivas_error ivas_pre_rend_init( +static ivas_error isar_pre_rend_init( SPLIT_REND_WRAPPER *pSplitRendWrapper, IVAS_REND_AudioBuffer *pSplitRendEncBuffer, ISAR_SPLIT_REND_CONFIG_DATA *pSplit_rend_config, @@ -3448,10 +3730,20 @@ static ivas_error ivas_pre_rend_init( const int16_t cldfb_in_flag, const int16_t num_subframes ) { + bool realloc; ivas_error error; IVAS_REND_AudioBufferConfig bufConfig; - if ( outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + realloc = false; + + /* only perform init if split rendering output */ + if ( outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && outConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + return IVAS_ERR_OK; + } + + /* these functions should only be called once during initial allocation */ + if ( pSplitRendEncBuffer->data == NULL ) { if ( pSplit_rend_config->poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { @@ -3466,28 +3758,34 @@ static ivas_error ivas_pre_rend_init( { return error; } + } + + /* We may need to change the allocated buffer size if a new input is added. + * If the cldfb_in_flag is different from what was previously allocated for the buffer, change the size */ + if ( pSplitRendEncBuffer->data != NULL && ( cldfb_in_flag != pSplitRendEncBuffer->config.is_cldfb ) ) + { + realloc = true; + } - /*allocate for CLDFB in and change to TD during process if needed*/ - bufConfig.numSamplesPerChannel = MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL; + if ( pSplitRendEncBuffer->data == NULL || realloc ) + { + /* set buffer config */ + bufConfig.is_cldfb = cldfb_in_flag; + bufConfig.numSamplesPerChannel = cldfb_in_flag ? MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL : L_FRAME_MAX; bufConfig.numChannels = BINAURAL_CHANNELS * pSplitRendWrapper->multiBinPoseData.num_poses; - bufConfig.is_cldfb = 1; pSplitRendEncBuffer->config = bufConfig; + /* allocate memory */ + if ( realloc ) + { + free( pSplitRendEncBuffer->data ); + } + if ( ( pSplitRendEncBuffer->data = malloc( bufConfig.numChannels * bufConfig.numSamplesPerChannel * sizeof( float ) ) ) == NULL ) { return IVAS_ERR_FAILED_ALLOC; } } - else - { - IVAS_REND_AudioBufferConfig bufConfig2; - - bufConfig2.numSamplesPerChannel = 0; - bufConfig2.numChannels = 0; - bufConfig2.is_cldfb = 0; - pSplitRendEncBuffer->config = bufConfig2; - pSplitRendEncBuffer->data = NULL; - } return IVAS_ERR_OK; } @@ -3514,7 +3812,10 @@ ivas_error IVAS_REND_AddInput( int32_t inputStructSize; #endif ivas_error ( *activateInput )( void *, AUDIO_CONFIG, IVAS_REND_InputId, RENDER_CONFIG_DATA *, hrtf_handles * ); + void ( *setInputDelay )( void *, bool ); int32_t inputIndex; + bool splitPreRendCldfb; + splitPreRendCldfb = false; /* Validate function arguments */ if ( hIvasRend == NULL || inputId == NULL ) @@ -3522,17 +3823,28 @@ ivas_error IVAS_REND_AddInput( return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) && hIvasRend->splitRendEncBuffer.data == NULL && hIvasRend->hRendererConfig != NULL ) + if ( hIvasRend->hRendererConfig != NULL ) { int16_t cldfb_in_flag; cldfb_in_flag = getCldfbRendFlag( hIvasRend, getAudioConfigType( inConfig ) ); - if ( ( error = ivas_pre_rend_init( hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, hIvasRend->num_subframes ) ) != IVAS_ERR_OK ) + if ( ( error = isar_pre_rend_init( hIvasRend->splitRendWrapper, + &hIvasRend->splitRendEncBuffer, + &hIvasRend->hRendererConfig->split_rend_config, + hIvasRend->headRotData, + hIvasRend->sampleRateOut, + hIvasRend->outputConfig, + cldfb_in_flag, + hIvasRend->num_subframes ) ) != IVAS_ERR_OK ) { return error; } + + /*assumes that input has been added which means codec has been set to either lcld or lc3plus (even if render config specified default)*/ + splitPreRendCldfb = ( hIvasRend->hRendererConfig->split_rend_config.codec == ISAR_SPLIT_REND_CODEC_LCLD ); } + #ifdef CODE_IMPROVEMENTS inputType = getAudioConfigType( inConfig ); switch ( inputType ) @@ -3547,6 +3859,7 @@ ivas_error IVAS_REND_AddInput( inputStructSize = sizeof( *hIvasRend->inputsIsm ); #endif activateInput = setRendInputActiveIsm; + setInputDelay = setRendInputDelayIsm; break; case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: maxNumInputsOfType = RENDERER_MAX_MC_INPUTS; @@ -3555,6 +3868,7 @@ ivas_error IVAS_REND_AddInput( inputStructSize = sizeof( *hIvasRend->inputsMc ); #endif activateInput = setRendInputActiveMc; + setInputDelay = setRendInputDelayMc; break; case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: maxNumInputsOfType = RENDERER_MAX_SBA_INPUTS; @@ -3563,6 +3877,7 @@ ivas_error IVAS_REND_AddInput( inputStructSize = sizeof( *hIvasRend->inputsSba ); #endif activateInput = setRendInputActiveSba; + setInputDelay = setRendInputDelaySba; break; case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: maxNumInputsOfType = RENDERER_MAX_MASA_INPUTS; @@ -3571,6 +3886,7 @@ ivas_error IVAS_REND_AddInput( inputStructSize = sizeof( *hIvasRend->inputsMasa ); #endif activateInput = setRendInputActiveMasa; + setInputDelay = setRendInputDelayMasa; break; default: return IVAS_ERR_INVALID_INPUT_FORMAT; @@ -3595,6 +3911,14 @@ ivas_error IVAS_REND_AddInput( { return error; } +#ifdef CODE_IMPROVEMENTS + setInputDelay( getInputByIndex( inputsArray, inputIndex, inputType ), splitPreRendCldfb ); +#else + setInputDelay( (uint8_t *) inputsArray + inputStructSize * inputIndex, splitPreRendCldfb ); +#endif + + /* set global maximum delay after adding an input */ + setMaxGlobalDelayNs( hIvasRend ); return IVAS_ERR_OK; } @@ -3848,6 +4172,9 @@ ivas_error IVAS_REND_RemoveInput( return IVAS_ERR_INVALID_INPUT_FORMAT; } + /* set global maximum delay after removing an input */ + setMaxGlobalDelayNs( hIvasRend ); + return IVAS_ERR_OK; } @@ -3924,11 +4251,6 @@ ivas_error IVAS_REND_GetDelay( int32_t *timeScale /* o : Time scale of the delay, equal to renderer output sampling rate */ ) { - /* TODO tmu : this function only returns the maximum delay across all inputs - * Ideally each input has its own delay buffer and everything is aligned (binaural and LFE filtering delays are nonuniform) - */ - int16_t i; - int32_t latency_ns; int32_t max_latency_ns; /* Validate function arguments */ @@ -3937,76 +4259,16 @@ ivas_error IVAS_REND_GetDelay( return IVAS_ERR_UNEXPECTED_NULL_POINTER; } - *timeScale = hIvasRend->sampleRateOut; *nSamples = 0; - max_latency_ns = 0; - - /* Compute the maximum delay across all inputs */ - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; i++ ) - { - if ( hIvasRend->inputsIsm[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - latency_ns = max( ( hIvasRend->inputsIsm[i].crendWrapper != NULL ) ? hIvasRend->inputsIsm[i].crendWrapper->binaural_latency_ns : 0, - hIvasRend->inputsIsm[i].tdRendWrapper.binaural_latency_ns ); - max_latency_ns = max( max_latency_ns, latency_ns ); - } - } - - for ( i = 0; i < RENDERER_MAX_MC_INPUTS; i++ ) - { - if ( hIvasRend->inputsMc[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - latency_ns = max( ( hIvasRend->inputsMc[i].crendWrapper != NULL ) ? hIvasRend->inputsMc[i].crendWrapper->binaural_latency_ns : 0, - hIvasRend->inputsMc[i].tdRendWrapper.binaural_latency_ns ); - max_latency_ns = max( max_latency_ns, latency_ns ); - } - } - - for ( i = 0; i < RENDERER_MAX_SBA_INPUTS; i++ ) - { - if ( hIvasRend->inputsSba[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - if ( hIvasRend->splitRendWrapper != NULL && hIvasRend->splitRendWrapper->hBinHrSplitPreRend != NULL ) - { - if ( hIvasRend->hRendererConfig->split_rend_config.rendererSelection == IVAS_BIN_RENDERER_TYPE_FASTCONV ) - { - latency_ns = hIvasRend->inputsSba[i].cldfbRendWrapper.binaural_latency_ns; - } - else - { - latency_ns = ( hIvasRend->inputsSba[i].crendWrapper != NULL ) ? hIvasRend->inputsSba[i].crendWrapper->binaural_latency_ns : 0; - } - max_latency_ns = max( max_latency_ns, latency_ns ); - } - else if ( hIvasRend->inputsSba[i].cldfbRendWrapper.hCldfbRend != NULL ) - { - latency_ns = hIvasRend->inputsSba[i].cldfbRendWrapper.binaural_latency_ns; - latency_ns += IVAS_FB_DEC_DELAY_NS; - max_latency_ns = max( max_latency_ns, latency_ns ); - } - else - { - latency_ns = ( hIvasRend->inputsSba[i].crendWrapper != NULL ) ? hIvasRend->inputsSba[i].crendWrapper->binaural_latency_ns : 0; - max_latency_ns = max( max_latency_ns, latency_ns ); - } - } - } + *timeScale = hIvasRend->sampleRateOut; - for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) - { - if ( hIvasRend->inputsMasa[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - latency_ns = (int32_t) ( (float) IVAS_FB_DEC_DELAY_NS + 0.5f ); - max_latency_ns = max( max_latency_ns, latency_ns ); - } - } + max_latency_ns = getMaxGlobalDelayNs( hIvasRend ); - *nSamples = (int16_t) roundf( (float) max_latency_ns * *timeScale / 1000000000.f ); + *nSamples = latencyNsToSamples( hIvasRend->sampleRateOut, max_latency_ns ); return IVAS_ERR_OK; } - /*-------------------------------------------------------------------* * IVAS_REND_FeedInputAudio() * @@ -4014,9 +4276,10 @@ ivas_error IVAS_REND_GetDelay( *-------------------------------------------------------------------*/ ivas_error IVAS_REND_FeedInputAudio( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const IVAS_REND_ReadOnlyAudioBuffer inputAudio /* i : buffer with input audio */ + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_REND_ReadOnlyAudioBuffer inputAudio, /* i : buffer with input audio */ + const bool flushInputs /* i : flush input audio */ ) { ivas_error error; @@ -4072,9 +4335,16 @@ ivas_error IVAS_REND_FeedInputAudio( } inputBase->inputBuffer.config = inputAudio.config; - - mvr2r( inputAudio.data, inputBase->inputBuffer.data, inputAudio.config.numSamplesPerChannel * inputAudio.config.numChannels ); - + if ( ( error = alignInputDelay( + inputBase, + inputAudio, + hIvasRend->maxGlobalDelayNs, + hIvasRend->sampleRateOut, + cldfb2tdSampleFact, + flushInputs ) ) != IVAS_ERR_OK ) + { + return error; + } inputBase->numNewSamplesPerChannel = inputAudio.config.numSamplesPerChannel / cldfb2tdSampleFact; return IVAS_ERR_OK; @@ -4382,6 +4652,7 @@ ivas_error IVAS_REND_FeedRenderConfig( if ( pMasaInput->hMasaExtRend->hDiracDecBin[0] != NULL && pMasaInput->hMasaExtRend->hDiracDecBin[0]->hReverb != NULL ) { ivas_binaural_reverb_close( &pMasaInput->hMasaExtRend->hDiracDecBin[0]->hReverb ); + if ( ( error = ivas_binaural_reverb_init( &pMasaInput->hMasaExtRend->hDiracDecBin[0]->hReverb, hIvasRend->hHrtfs.hHrtfStatistics, pMasaInput->hMasaExtRend->hSpatParamRendCom->num_freq_bands, @@ -4399,6 +4670,7 @@ ivas_error IVAS_REND_FeedRenderConfig( if ( pMasaInput->hMasaExtRend->hReverb != NULL ) { ivas_binaural_reverb_close( &pMasaInput->hMasaExtRend->hReverb ); + if ( ( error = ivas_binaural_reverb_init( &pMasaInput->hMasaExtRend->hReverb, hIvasRend->hHrtfs.hHrtfStatistics, pMasaInput->hMasaExtRend->hSpatParamRendCom->num_freq_bands, @@ -4431,6 +4703,7 @@ ivas_error IVAS_REND_FeedRenderConfig( return error; } } + if ( pMcInput->crendWrapper != NULL && pMcInput->crendWrapper->hCrend[0] && pMcInput->crendWrapper->hCrend[0]->hReverb != NULL ) { if ( ( error = ivas_reverb_open( &pMcInput->crendWrapper->hCrend[0]->hReverb, hIvasRend->hHrtfs.hHrtfStatistics, hRenderConfig, *pMcInput->base.ctx.pOutSampleRate ) ) != IVAS_ERR_OK ) @@ -4481,6 +4754,7 @@ ivas_error IVAS_REND_FeedRenderConfig( { int16_t cldfb_in_flag; cldfb_in_flag = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); + if ( hIvasRend->splitRendWrapper != NULL ) { ISAR_PRE_REND_close( hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer ); @@ -4488,7 +4762,7 @@ ivas_error IVAS_REND_FeedRenderConfig( hIvasRend->splitRendWrapper = NULL; } - if ( ( error = ivas_pre_rend_init( hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, hIvasRend->num_subframes ) ) != IVAS_ERR_OK ) + if ( ( error = isar_pre_rend_init( hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in_flag, hIvasRend->num_subframes ) ) != IVAS_ERR_OK ) { return error; } @@ -5217,7 +5491,7 @@ static ivas_error renderIsmToBinaural( push_wmops( "renderIsmToBinaural" ); /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ - ism_md_subframe_update_ext = (int16_t) roundf( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + ism_md_subframe_update_ext = (int16_t) roundf( ismInput->ism_metadata_delay_ms / ( 1000.f / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); copyBufferTo2dArray( ismInput->base.inputBuffer, tmpTDRendBuffer ); if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, ismInput->hReverb, ism_md_subframe_update_ext, @@ -5421,7 +5695,7 @@ static ivas_error renderIsmToBinauralReverb( push_wmops( "renderIsmToBinauralRoom" ); /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ - ism_md_subframe_update_ext = (int16_t) roundf( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + ism_md_subframe_update_ext = (int16_t) roundf( ismInput->ism_metadata_delay_ms / ( 1000.f / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); @@ -5587,8 +5861,10 @@ static ivas_error renderIsmToSplitBinaural( const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; const SPLIT_REND_WRAPPER *pSplitRendWrapper; IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES]; - int16_t i; + int16_t i, ch, slot_idx, num_bands; float tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; + float tmpBinaural_CldfbRe[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float tmpBinaural_CldfbIm[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; int16_t output_frame = ismInput->base.inputBuffer.config.numSamplesPerChannel; COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; int16_t ism_md_subframe_update_ext; @@ -5599,7 +5875,7 @@ static ivas_error renderIsmToSplitBinaural( pMultiBinPoseData = &pSplitRendWrapper->multiBinPoseData; /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ - ism_md_subframe_update_ext = (int16_t) roundf( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); + ism_md_subframe_update_ext = (int16_t) roundf( ismInput->ism_metadata_delay_ms / ( 1000.f / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); pCombinedOrientationData = *ismInput->base.ctx.pCombinedOrientationData; @@ -5655,10 +5931,29 @@ static ivas_error renderIsmToSplitBinaural( return error; } - /* Copy rendered audio to tmp storage buffer. Copying directly to output would - * overwrite original audio, which is still needed for rendering next head pose. */ - mvr2r( tmpProcessing[0], tmpBinaural[2 * pos_idx], output_frame ); - mvr2r( tmpProcessing[1], tmpBinaural[2 * pos_idx + 1], output_frame ); + if ( outAudio.config.is_cldfb ) + { + /* Perform CLDFB analysis on rendered audio, since the output buffer is CLDFB domain */ + num_bands = (int16_t) ( ( BINAURAL_MAXBANDS * *ismInput->base.ctx.pOutSampleRate ) / 48000 ); + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + for ( slot_idx = 0; slot_idx < IVAS_CLDFB_NO_COL_MAX; slot_idx++ ) + { + cldfbAnalysis_ts( &tmpProcessing[ch][num_bands * slot_idx], + &tmpBinaural_CldfbRe[BINAURAL_CHANNELS * pos_idx + ch][slot_idx][0], + &tmpBinaural_CldfbIm[BINAURAL_CHANNELS * pos_idx + ch][slot_idx][0], + num_bands, + ismInput->base.ctx.pSplitRendWrapper->hCldfbHandles->cldfbAna[pos_idx + ch] ); + } + } + } + else + { + /* Copy rendered audio to tmp storage buffer. Copying directly to output would + * overwrite original audio, which is still needed for rendering next head pose. */ + mvr2r( tmpProcessing[0], tmpBinaural[BINAURAL_CHANNELS * pos_idx], output_frame ); + mvr2r( tmpProcessing[1], tmpBinaural[BINAURAL_CHANNELS * pos_idx + 1], output_frame ); + } /* Overwrite processing buffer with original input audio again */ copyBufferTo2dArray( ismInput->base.inputBuffer, tmpProcessing ); @@ -5670,7 +5965,14 @@ static ivas_error renderIsmToSplitBinaural( pCombinedOrientationData->Quaternions[i] = originalHeadRot[i]; } - accumulate2dArrayToBuffer( tmpBinaural, &outAudio ); + if ( outAudio.config.is_cldfb ) + { + accumulateCLDFBArrayToBuffer( tmpBinaural_CldfbRe, tmpBinaural_CldfbIm, &outAudio ); + } + else + { + accumulate2dArrayToBuffer( tmpBinaural, &outAudio ); + } pop_wmops(); /* Encoding to split rendering bitstream done at a higher level */ @@ -5705,11 +6007,13 @@ static ivas_error renderInputIsm( { ivas_error error; IVAS_REND_AudioBuffer inAudio; + int16_t cldfb2tdSampleFact; error = IVAS_ERR_OK; inAudio = ismInput->base.inputBuffer; - if ( ismInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) + cldfb2tdSampleFact = outAudio.config.is_cldfb ? 2 : 1; + if ( ismInput->base.numNewSamplesPerChannel * cldfb2tdSampleFact != outAudio.config.numSamplesPerChannel ) { return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); } @@ -5834,8 +6138,7 @@ static ivas_error renderLfeToBinaural( num_cpy_smpl_prev_frame = mcInput->binauralDelaySmp; num_cpy_smpl_cur_frame = frame_size - num_cpy_smpl_prev_frame; - /* Assuming LFE should be delayed by less that the duration of one frame */ - assert( mcInput->binauralDelaySmp < frame_size ); + assert( mcInput->binauralDelaySmp <= MAX_BIN_DELAY_SAMPLES ); /* Get delayed LFE signal from previous frame, apply gain and save in tmp buffer */ v_multc( mcInput->lfeDelayBuffer, gain, tmpLfeBuffer, num_cpy_smpl_prev_frame ); @@ -7413,15 +7716,13 @@ ivas_error IVAS_REND_SetIsmMetadataDelay( *-------------------------------------------------------------------*/ static ivas_error getSamplesInternal( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ - IVAS_REND_BitstreamBuffer *hBits /*i/o: buffer for input/output bitstream. Needed in split rendering mode*/ + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ ) { ivas_error error; int16_t numOutChannels; int16_t cldfb2tdSampleFact; - IVAS_REND_AudioBuffer outAudioOrig; /* Validate function arguments */ if ( hIvasRend == NULL || outAudio.data == NULL ) @@ -7481,34 +7782,21 @@ static ivas_error getSamplesInternal( } } - if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_GetNumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK ) { return error; } - if ( numOutChannels != outAudio.config.numChannels && hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + if ( numOutChannels != outAudio.config.numChannels && + hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && + hIvasRend->outputConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) { return IVAS_ERR_WRONG_NUM_CHANNELS; } - /* Clear original output buffer */ + /* Clear output buffer */ set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); - outAudioOrig = outAudio; - /* Use internal buffer if outputting split rendering bitstream */ - if ( ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) || - ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) - { - int16_t num_poses_orig; - num_poses_orig = hIvasRend->splitRendWrapper->multiBinPoseData.num_poses; - outAudio = hIvasRend->splitRendEncBuffer; - ISAR_PRE_REND_GetMultiBinPoseData( &hIvasRend->hRendererConfig->split_rend_config, &hIvasRend->splitRendWrapper->multiBinPoseData, hIvasRend->headRotData.sr_pose_pred_axis ); - assert( num_poses_orig == hIvasRend->splitRendWrapper->multiBinPoseData.num_poses && "number of poses should not change dynamically" ); - - /* Clear output buffer for split rendering bitstream */ - set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); - } - if ( ( error = renderActiveInputsIsm( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) { return error; @@ -7529,73 +7817,26 @@ static ivas_error getSamplesInternal( return error; } - if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) - { - ISAR_SPLIT_REND_BITS_DATA bits; - int16_t cldfb_in_flag; - float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - int16_t ch; - int16_t i, ro_md_flag; - float *tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS], tmpBinaural_buff[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; - - for ( ch = 0; ch < MAX_OUTPUT_CHANNELS; ch++ ) - { - tmpBinaural[ch] = tmpBinaural_buff[ch]; - } - - if ( outAudio.config.is_cldfb == 1 ) - { - cldfb_in_flag = 1; - copyBufferToCLDFBarray( outAudio, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural ); - } - else - { - cldfb_in_flag = 0; - copyBufferTo2dArray( outAudio, tmpBinaural_buff ); - } + return IVAS_ERR_OK; +} - /* Encode split rendering bitstream */ - convertBitsBufferToInternalBitsBuff( *hBits, &bits ); - ro_md_flag = 0; - for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) - { - if ( hIvasRend->inputsIsm[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) - { - ro_md_flag = 1; - break; - } - } - - if ( ( error = ISAR_PRE_REND_MultiBinToSplitBinaural( hIvasRend->splitRendWrapper, - hIvasRend->headRotData.headPositions[0], - hIvasRend->hRendererConfig->split_rend_config.splitRendBitRate, - hIvasRend->hRendererConfig->split_rend_config.codec, - hIvasRend->hRendererConfig->split_rend_config.isar_frame_size_ms, - hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, - &bits, - Cldfb_RealBuffer_Binaural, - Cldfb_ImagBuffer_Binaural, - ( const int16_t )( ( BINAURAL_MAXBANDS * hIvasRend->sampleRateOut ) / 48000 ), - tmpBinaural, - 1, - cldfb_in_flag, - ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0, - ro_md_flag ) ) != IVAS_ERR_OK ) - { - return error; - } - - convertInternalBitsBuffToBitsBuffer( hBits, bits ); +/*-------------------------------------------------------------------* + * IVAS_REND_GetSamples() + * + * + *-------------------------------------------------------------------*/ - /* reset to outAudioOrig in case of PCM output */ - outAudio = outAudioOrig; +ivas_error IVAS_REND_GetSamples( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ +) +{ + ivas_error error; - if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) - { - accumulate2dArrayToBuffer( tmpBinaural_buff, &outAudio ); - } + if ( ( error = getSamplesInternal( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + { + return error; } if ( outAudio.config.is_cldfb == 0 ) @@ -7615,22 +7856,6 @@ static ivas_error getSamplesInternal( } -/*-------------------------------------------------------------------* - * IVAS_REND_GetSamples() - * - * - *-------------------------------------------------------------------*/ - -ivas_error IVAS_REND_GetSamples( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ -) -{ - - return getSamplesInternal( hIvasRend, outAudio, NULL ); -} - - /*-------------------------------------------------------------------* * IVAS_REND_GetSplitBinauralBitstream() * @@ -7643,23 +7868,121 @@ ivas_error IVAS_REND_GetSplitBinauralBitstream( IVAS_REND_BitstreamBuffer *hBits /* o : buffer for output bitstream */ ) { + ivas_error error; + int16_t ch; int16_t cldfb_in_flag; + int16_t i, ro_md_flag; + int16_t num_poses_orig; + float *tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS], tmpBinaural_buff[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; + float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_RealBuffer_Binaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + IVAS_REND_AudioBufferConfig *pSplitEncBufConfig; + ISAR_SPLIT_REND_CONFIG_HANDLE pSplitRendConfig; + ISAR_SPLIT_REND_BITS_DATA bits; + + for ( ch = 0; ch < MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS; ch++ ) + { + tmpBinaural[ch] = tmpBinaural_buff[ch]; + } cldfb_in_flag = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); - hIvasRend->splitRendEncBuffer.config.is_cldfb = cldfb_in_flag; - if ( hIvasRend->hRendererConfig->split_rend_config.dof == 0 || hIvasRend->hRendererConfig->split_rend_config.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) + pSplitEncBufConfig = &hIvasRend->splitRendEncBuffer.config; + pSplitRendConfig = &hIvasRend->hRendererConfig->split_rend_config; + + /* 0 DoF / No pose correction retains frame size */ + pSplitEncBufConfig->is_cldfb = cldfb_in_flag; + if ( pSplitRendConfig->dof == 0 || pSplitRendConfig->poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) + { + pSplitEncBufConfig->numSamplesPerChannel = outAudio.config.numSamplesPerChannel; + } + /* Pose correction requires 20ms */ + else + { + pSplitEncBufConfig->numSamplesPerChannel = (int16_t) ( hIvasRend->sampleRateOut / FRAMES_PER_SEC ); + } + pSplitEncBufConfig->numSamplesPerChannel *= cldfb_in_flag ? 2 : 1; + + num_poses_orig = hIvasRend->splitRendWrapper->multiBinPoseData.num_poses; + ISAR_PRE_REND_GetMultiBinPoseData( pSplitRendConfig, + &hIvasRend->splitRendWrapper->multiBinPoseData, + hIvasRend->headRotData.sr_pose_pred_axis ); + assert( num_poses_orig == hIvasRend->splitRendWrapper->multiBinPoseData.num_poses && "number of poses should not change dynamically" ); + + /* hIvasRend->splitRendEncBuffer contains multi-pose data for BINAURAL_SPLIT_CODED output + outAudio used later for main pose BINAURAL_SPLIT_PCM output */ + if ( ( error = getSamplesInternal( hIvasRend, hIvasRend->splitRendEncBuffer ) ) != IVAS_ERR_OK ) { - hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = outAudio.config.numSamplesPerChannel; + return error; + } + + /* copy outputs */ + if ( hIvasRend->splitRendEncBuffer.config.is_cldfb == 1 ) + { + cldfb_in_flag = 1; + copyBufferToCLDFBarray( hIvasRend->splitRendEncBuffer, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural ); } else { - hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = (int16_t) ( hIvasRend->sampleRateOut / FRAMES_PER_SEC ); + cldfb_in_flag = 0; + copyBufferTo2dArray( hIvasRend->splitRendEncBuffer, tmpBinaural_buff ); + } + + /* Encode split rendering bitstream */ + convertBitsBufferToInternalBitsBuff( *hBits, &bits ); + + ro_md_flag = 0; + for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i ) + { + if ( hIvasRend->inputsIsm[i].base.inConfig != IVAS_AUDIO_CONFIG_INVALID ) + { + ro_md_flag = 1; + break; + } + } + + if ( ( error = ISAR_PRE_REND_MultiBinToSplitBinaural( hIvasRend->splitRendWrapper, + hIvasRend->headRotData.headPositions[0], + pSplitRendConfig->splitRendBitRate, + pSplitRendConfig->codec, + pSplitRendConfig->isar_frame_size_ms, + pSplitRendConfig->codec_frame_size_ms, + &bits, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + ( const int16_t )( ( BINAURAL_MAXBANDS * hIvasRend->sampleRateOut ) / 48000 ), + tmpBinaural, + 1, + cldfb_in_flag, + ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0, + ro_md_flag ) ) != IVAS_ERR_OK ) + { + return error; + } + + convertInternalBitsBuffToBitsBuffer( hBits, bits ); + + /* copy over first pose data to outAudio */ + if ( hIvasRend->outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + /* set outAudio to zero - getSamplesInternal only cleared splitRendEncBuffer */ + set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); + accumulate2dArrayToBuffer( tmpBinaural_buff, &outAudio ); + } + + if ( outAudio.config.is_cldfb == 0 ) + { +#ifndef DISABLE_LIMITER +#ifdef DEBUGGING + hIvasRend->numClipping += +#endif + limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); +#endif } - hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel *= cldfb_in_flag ? 2 : 1; - /* hIvasRend->splitRendEncBuffer used for BINAURAL_SPLIT_CODED output - outAudio used for BINAURAL_SPLIT_PCM output */ - return getSamplesInternal( hIvasRend, outAudio, hBits ); + /* update global cominbed orientation start index */ + ivas_combined_orientation_update_start_index( hIvasRend->hCombinedOrientationData, outAudio.config.numSamplesPerChannel ); + + return IVAS_ERR_OK; } diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index 194bca0165ca3f22a16af67ac4a7b9c795d8d91b..ac9516521d0486dcc9224b2f3f6af10e918b81fa 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -114,15 +114,15 @@ typedef enum _IVAS_REND_COMPLEXITY_LEVEL /* Functions to be called before rendering */ ivas_error IVAS_REND_Open( - IVAS_REND_HANDLE *phIvasRend, /* i/o: Pointer to renderer handle */ - const int32_t outputSampleRate, /* i : output sampling rate */ - const IVAS_AUDIO_CONFIG outConfig, /* i : output audio config */ - const bool asHrtfBinary, /* i : load hrtf binary file */ - const int16_t nonDiegeticPan, /* i : non-diegetic object flag */ - const float nonDiegeticPanGain, /* i : non-diegetic panning gain */ - const int16_t Opt_Headrotation, /* i : indicates whether head-rotation is used */ - const int16_t Opt_ExternalOrientation, /* i : indicates whether external orientations are used */ - const int16_t num_subframes /* i : number of subframes */ + IVAS_REND_HANDLE *phIvasRend, /* i/o: Pointer to renderer handle */ + const int32_t outputSampleRate, /* i : output sampling rate */ + const IVAS_AUDIO_CONFIG outConfig, /* i : output audio config */ + const bool asHrtfBinary, /* i : load hrtf binary file */ + const int16_t nonDiegeticPan, /* i : non-diegetic object flag */ + const float nonDiegeticPanGain, /* i : non-diegetic panning gain */ + const int16_t Opt_Headrotation, /* i : indicates whether head-rotation is used */ + const int16_t Opt_ExternalOrientation, /* i : indicates whether external orientations are used */ + const int16_t num_subframes /* i : number of subframes */ ); /* Note: this will reset custom LFE routings set for any MC input */ @@ -134,7 +134,7 @@ ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( /* Functions to be called before/during rendering */ -ivas_error IVAS_REND_NumOutChannels( +ivas_error IVAS_REND_GetNumOutChannels( IVAS_REND_CONST_HANDLE hIvasRend, /* i : Renderer handle */ int16_t *numOutChannels /* o : number of output channels */ ); @@ -212,16 +212,17 @@ ivas_error IVAS_REND_GetHrtfParamBinHandle( ); ivas_error IVAS_REND_GetHrtfStatisticsHandle( - IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ - IVAS_DEC_HRTF_STATISTICS_HANDLE **hHrtfStatistics /* o : HRTF statistics handle */ + IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */ + IVAS_DEC_HRTF_STATISTICS_HANDLE **hHrtfStatistics /* o : HRTF statistics handle */ ); /* Functions to be called during rendering */ ivas_error IVAS_REND_FeedInputAudio( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - const IVAS_REND_InputId inputId, /* i : ID of the input */ - const IVAS_REND_ReadOnlyAudioBuffer inputAudio /* i : buffer with input audio */ + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_REND_ReadOnlyAudioBuffer inputAudio, /* i : buffer with input audio */ + const bool flushInputs /* i : flush input audio */ ); ivas_error IVAS_REND_FeedInputObjectMetadata( @@ -382,7 +383,7 @@ ivas_error IVAS_REND_GetSamples( /* Functions to be called after rendering */ void IVAS_REND_Close( - IVAS_REND_HANDLE* phIvasRend /* i/o: Pointer to renderer handle */ + IVAS_REND_HANDLE *phIvasRend /* i/o: Pointer to renderer handle */ );