Commit 48681f8c authored by Jan Kiene's avatar Jan Kiene
Browse files

port 2171 from float repo to BASOP main 

Resolve "Delay alignment in external renderer" and "External renderer crashes when rendering OMASA"
parent 7c2dae81

Workspace_msvc/lib_rend.vcxproj

+2 −1
Original line number Diff line number Diff line
@@ -175,6 +175,7 @@ 
    <ClCompile Include="..\lib_rend\ivas_rom_rend_fx.c" />

    <ClCompile Include="..\lib_rend\ivas_shoebox_fx.c" />

    <ClCompile Include="..\lib_rend\ivas_td_decorr_fx.c" />

    <ClCompile Include="..\lib_rend\ivas_td_ring_buffer_fx.c" />

    <ClCompile Include="..\lib_rend\ivas_vbap_fx.c" />

    <ClCompile Include="..\lib_rend\lib_rend_fx.c" />

  </ItemGroup>

apps/renderer.c

+63 −51
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ 
*******************************************************************************************************/



#include "lib_rend.h"

#include "typedef.h"

#include <assert.h>

#include <math.h>

#include <string.h>
@@ -797,6 +798,7 @@ int main( 
    SplitFileReadWrite *hSplitRendFileReadWrite;

    int16_t delayNumSamples_temp;

    int32_t delayTimeScale_temp;

    bool flushRendererLastFrame = false;

    int16_t numSamplesRead;

    int16_t delayNumSamples = -1;

    int16_t delayNumSamples_orig = 0;
@@ -842,8 +844,11 @@ int main( 


    CmdlnArgs args = parseCmdlnArgs( argc, argv );



    if ( args.nonDiegeticPan && !( ( args.inConfig.numAudioObjects == 0 && args.inConfig.multiChannelBuses[0].audioConfig == IVAS_AUDIO_CONFIG_MONO ) ||

                                   ( args.inConfig.numAudioObjects > 0 && args.inConfig.audioObjects[0].audioConfig == IVAS_AUDIO_CONFIG_OBA && args.inConfig.numAudioObjects == 1 ) ) )

    if ( args.nonDiegeticPan &&

         !( ( args.inConfig.numAudioObjects == 0 &&

              args.inConfig.multiChannelBuses[0].audioConfig == IVAS_AUDIO_CONFIG_MONO ) ||

            ( args.inConfig.numAudioObjects > 0 &&

              args.inConfig.audioObjects[0].audioConfig == IVAS_AUDIO_CONFIG_OBA && args.inConfig.numAudioObjects == 1 ) ) )

    {

        fprintf( stderr, "\ninvalid configuration - non-diegetic panning requires mono or ISM1 input\n" );

        goto cleanup;
@@ -1173,7 +1178,10 @@ int main( 
#endif



        /* sanity check */

        if ( ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL ) && ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR ) && ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && !is_split_pre_rend_mode( &args ) )

        if ( ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL ) &&

             ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR ) &&

             ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) &&

             !is_split_pre_rend_mode( &args ) )

        {

            fprintf( stderr, "\nExternal Renderer Config is supported only when binaural output configurations is used as output OR when Split pre-rendering mode is enabled. Exiting. \n" );

            goto cleanup;
@@ -1490,7 +1498,7 @@ int main( 
    }



    Word16 numOutChannels;

    if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK )

    if ( ( error = IVAS_REND_GetNumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "\nError in IVAS_REND_NumOutChannels(): %s\n", ivas_error_to_string( error ) );

        goto cleanup;
@@ -1505,31 +1513,16 @@ int main( 
        }

    }



    if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED )

    {

        if ( ( error = IVAS_REND_GetSplitRendBitstreamHeader( hIvasRend, &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection, &bitsBuffer.config.codec_frame_size_ms, &bitsBuffer.config.isar_frame_size_ms ) ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nError in IVAS_REND_GetSplitRendBitstreamHeader(): %s!\n", ivas_error_to_string( error ) );

            goto cleanup;

        }



        if ( IVAS_REND_GetDelay_fx( hIvasRend, &delayNumSamples_temp, &delayTimeScale_temp ) != IVAS_ERR_OK )

    if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM )

    {

            fprintf( stderr, "\nUnable to get delay of renderer!\n" );

            goto cleanup;

        }

        char *outFile = args.outMetadataFilePath;



        if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, args.outputFilePath, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK )

        if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED )

        {

            fprintf( stderr, "\nCould not open split rend metadata file %s\n", args.outputFilePath );

            goto cleanup;

        }

            outFile = args.outputFilePath;

            audioWriter = NULL;

        }

    else

    {

        if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM )

        {



        if ( ( error = IVAS_REND_GetSplitRendBitstreamHeader( hIvasRend, &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection, &bitsBuffer.config.codec_frame_size_ms, &bitsBuffer.config.isar_frame_size_ms ) ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nError in IVAS_REND_GetSplitRendBitstreamHeader(): %s!\n", ivas_error_to_string( error ) );
@@ -1542,13 +1535,15 @@ int main( 
            goto cleanup;

        }



            if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, args.outMetadataFilePath, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK )

        if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, outFile, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK )

        {

                fprintf( stderr, "\nCould not open split rend metadata file %s\n", args.outMetadataFilePath );

            fprintf( stderr, "\nCould not open split rend metadata file %s\n", outFile );

            goto cleanup;

        }

    }



    if ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED )

    {

        if ( AudioFileWriter_open( &audioWriter, args.outputFilePath, args.sampleRate, numOutChannels ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nFailed to open file: %s\n", args.outputFilePath );
@@ -1693,19 +1688,26 @@ int main( 
        if ( numSamplesRead == 0 )

        {

            /* end of input data */

            break;

            flushRendererLastFrame = true;

        }



        /* Convert from int to float and from interleaved to packed */

        if ( !flushRendererLastFrame )

        {

            Word16 Q_out;

            *outBuffer.pq_fact = 16 - ( gd_bits );

            convertInputBuffer_fx( inpInt16Buffer, numSamplesRead, inBuffer.config.numSamplesPerChannel, num_in_channels, inInt32Buffer, *outBuffer.pq_fact, inBuffer.config.is_cldfb, cldfbAna, &Q_out );

            *outBuffer.pq_fact = Q_out;

        }

        else

        {

            memset( inBuffer.data_fx, 0, inBuffer.config.numChannels * inBuffer.config.numSamplesPerChannel * sizeof( Word32 ) );

        }



        int16_t num_subframes, sf_idx;

        num_subframes = (int16_t) args.render_framesize;



        if ( isCurrentFrameMultipleOf20ms )

        if ( isCurrentFrameMultipleOf20ms && !flushRendererLastFrame )

        {

            IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer );
@@ -1817,7 +1819,7 @@ int main( 
            }



            IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.multiChannelBuses[i].inputChannelIndex, numChannels );

            if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, mcIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

            if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, mcIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                goto cleanup;
@@ -1831,7 +1833,7 @@ int main( 
                if ( i == 0 )

                {

                    IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.audioObjects[i].inputChannelIndex, args.inConfig.numAudioObjects );

                    if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, ismIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

                    if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, ismIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

                    {

                        fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                        goto cleanup;
@@ -1847,7 +1849,7 @@ int main( 
            else

            {

                IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.audioObjects[i].inputChannelIndex, 1 );

                if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, ismIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

                if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, ismIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

                {

                    fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                    goto cleanup;
@@ -1870,7 +1872,7 @@ int main( 
            }



            IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.ambisonicsBuses[i].inputChannelIndex, numChannels );

            if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, sbaIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

            if ( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, sbaIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                goto cleanup;
@@ -1887,13 +1889,13 @@ int main( 


            IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.masaBuses[i].inputChannelIndex, numChannels );



            IF( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, masaIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

            IF( ( error = IVAS_REND_FeedInputAudio_fx( hIvasRend, masaIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                goto cleanup;

            }



            if ( isCurrentFrameMultipleOf20ms )

            if ( isCurrentFrameMultipleOf20ms && !flushRendererLastFrame )

            {

                if ( masaReaders[i] != NULL )

                {
@@ -1956,7 +1958,7 @@ int main( 
            zeroPad = delayNumSamples;

        }



        if ( is_split_pre_rend_mode( &args ) )

        if ( is_split_pre_rend_mode( &args ) && !flushRendererLastFrame )

        {

            if ( split_rend_write_bitstream_to_file( hSplitRendFileReadWrite, bitsBuffer.bits, &bitsBuffer.config.bitsRead,

                                                     &bitsBuffer.config.bitsWritten ) != IVAS_ERR_OK )
@@ -1966,7 +1968,7 @@ int main( 
            }

        }



        if ( audioWriter != NULL )

        if ( audioWriter != NULL && !flushRendererLastFrame )

        {

            if ( delayNumSamples * num_out_channels < outBufferSize )

            {
@@ -1987,7 +1989,7 @@ int main( 
        bitsBuffer.config.bitsWritten = 0;



        /* Write MASA metadata for MASA outputs */

        if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA1 || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA2 )

        if ( !flushRendererLastFrame && ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA1 || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA2 ) )

        {

            IVAS_REND_AudioConfigType inputType1;

            IVAS_REND_AudioConfigType inputType2;
@@ -2059,7 +2061,8 @@ int main( 
                    }

                }



                if ( ( args.inConfig.numAmbisonicsBuses > 0 || args.inConfig.numMultiChannelBuses > 0 || args.inConfig.numMasaBuses > 0 ) && args.inConfig.numAudioObjects > 0 )

                if ( ( args.inConfig.numAmbisonicsBuses > 0 || args.inConfig.numMultiChannelBuses > 0 || args.inConfig.numMasaBuses > 0 ) &&

                     args.inConfig.numAudioObjects > 0 )

                {

                    inputType2 = IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED;

                    if ( ( error = IVAS_REND_MergeMasaMetadata( hIvasRend, &hMetaOutput, inputType1, inputType2 ) ) != IVAS_ERR_OK )
@@ -2075,6 +2078,13 @@ int main( 
            }

        }



        /* no new input was actually read, only delay buffers were flushed

         * therefore this is not a real frame */

        if ( flushRendererLastFrame )

        {

            break;

        }



        frame++;

        if ( !args.quietModeEnabled )

        {
@@ -2087,12 +2097,13 @@ int main( 
#endif

    }



    /* add zeros at the end to have equal length of synthesized signals */

    /* add zeros at the end to have equal length of synthesized signals

     * the output buffer will contain either leftover input samples from delay aligned inputs

     * or zeros for padding */

    if ( audioWriter != NULL )

    {

        for ( zeroPadToWrite = zeroPad; zeroPadToWrite > frameSize_smpls; zeroPadToWrite -= frameSize_smpls )

        {

            memset( outInt16Buffer, 0, outBufferSize * sizeof( int16_t ) );

            if ( ( error = AudioFileWriter_write( audioWriter, outInt16Buffer, outBufferSize ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nOutput audio file writer error\n" );
@@ -2100,7 +2111,6 @@ int main( 
            }

        }



        memset( outInt16Buffer, 0, zeroPadToWrite * outBuffer.config.numChannels * sizeof( int16_t ) );

        if ( ( error = AudioFileWriter_write( audioWriter, outInt16Buffer, zeroPadToWrite * outBuffer.config.numChannels ) ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nOutput audio file writer error\n" );
@@ -2110,7 +2120,8 @@ int main( 
    }





    if ( args.inConfig.numAudioObjects != 0 && ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) )

    if ( args.inConfig.numAudioObjects != 0 &&

         ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) )

    {

        fprintf( stdout, "\n\nMetadata delayed %d subframes\n\n", (int16_t) round( args.syncMdDelay / ( 1000 / IVAS_NUM_FRAMES_PER_SEC / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ) ) );

    }
@@ -3509,7 +3520,8 @@ static void parseObjectPosition( 
    *positionDuration = (uint16_t) strtol( line, &endptr, 10 );

    readNextMetadataChunk( line, "\n" );



    read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f", &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] );

    read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f",

                                    &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] );



    if ( read_values < 2 )

    {

lib_isar/lib_isar_post_rend.c

+3 −3
Original line number Diff line number Diff line
@@ -1735,12 +1735,12 @@ ivas_error ISAR_POST_REND_getSamples( 
    }



#ifndef DISABLE_LIMITER

    Word32 limiter_thresold;

    Word32 limiter_threshold;

#ifdef DEBUGGING

    hIvasRend->numClipping +=

#endif

        limiter_thresold = L_shl( IVAS_LIMITER_THRESHOLD, *outAudio.pq_fact );

    limitRendererOutput_fx( hIvasRend->hLimiter, outAudio.data_fx, outAudio.config.numSamplesPerChannel, limiter_thresold, *outAudio.pq_fact );

        limiter_threshold = L_shl( IVAS_LIMITER_THRESHOLD, *outAudio.pq_fact );

    limitRendererOutput_fx( hIvasRend->hLimiter, outAudio.data_fx, outAudio.config.numSamplesPerChannel, limiter_threshold, *outAudio.pq_fact );

#endif



    return IVAS_ERR_OK;

lib_isar/lib_isar_pre_rend.c

+2 −3
Original line number Diff line number Diff line
@@ -96,9 +96,8 @@ ivas_error ISAR_PRE_REND_open( 
        isCldfbNeeded = 1;

    }



    hSplitRendWrapper->hCldfbHandles = NULL;



    IF( isCldfbNeeded )

    test();

    IF( isCldfbNeeded && hSplitRendWrapper->hCldfbHandles == NULL )

    {

        IF( ( hSplitRendWrapper->hCldfbHandles = (CLDFB_HANDLES_WRAPPER_HANDLE) malloc( sizeof( CLDFB_HANDLES_WRAPPER ) ) ) == NULL )

        {

lib_rend/ivas_prot_rend_fx.h

+35 −0
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ 
#ifndef IVAS_PROT_REND_H

#define IVAS_PROT_REND_H



#include "typedef.h"

#include <stdint.h>

#include "options.h"

#include "ivas_error.h"
@@ -1594,6 +1595,40 @@ void ivas_rend_closeCldfbRend( 
    CLDFB_REND_WRAPPER *pCldfbRend 

);



/*----------------------------------------------------------------------------------*

 * Time domain ring buffer prototypes

 *----------------------------------------------------------------------------------*/



ivas_error ivas_TD_RINGBUF_Open(

    TD_RINGBUF_HANDLE *ph,                                      /* i/o: Ring buffer handle                     */

    const UWord32 capacity_per_channel,                        /* i  : Number of samples stored per channel   */

    const UWord16 num_channels                                 /* i  : Number of channels                     */

);



void ivas_TD_RINGBUF_Close( 

    TD_RINGBUF_HANDLE *ph                                       /* i/o: Ring buffer handle                      */

);



void ivas_TD_RINGBUF_Push( 

    TD_RINGBUF_HANDLE h,                                        /* i/o: Ring buffer handle                       */

    const Word32 *data,                                          /* i  : Input data                               */

    const UWord32 num_samples_per_channel                      /* i  : Number of samples per channel to store   */

);



void ivas_TD_RINGBUF_PushZeros( 

    TD_RINGBUF_HANDLE h,                                        /* i/o: Ring buffer handle                      */

    const UWord32 num_samples_per_channel                      /* i  : Number of zeros per channel to store    */

);



void ivas_TD_RINGBUF_Pop( 

    TD_RINGBUF_HANDLE h,                                        /* i/o: Ring buffer handle                      */

    Word32 *data,                                                /* i  : Output data                             */

    const UWord32 num_samples_per_channel                      /* i  : Number of samples per channel to retrieve*/

);



uint32_t ivas_TD_RINGBUF_Size(

    const TD_RINGBUF_HANDLE h                                   /* i  : Ring buffer handle                      */

);



/* clang-format on */