Commit 4a04e7ed authored by Jan Kiene's avatar Jan Kiene
Browse files

port MR 2173 from float repo 

Resolve "Delay alignment in external renderer" and "External renderer crashes when rendering OMASA"
parent ab0b493c

Workspace_msvc/lib_rend.vcxproj

+1 −0
Original line number Diff line number Diff line
@@ -175,6 +175,7 @@ 
    <ClCompile Include="..\lib_rend\ivas_rom_rend.c" />

    <ClCompile Include="..\lib_rend\ivas_shoebox.c" />

    <ClCompile Include="..\lib_rend\ivas_td_decorr.c" />

    <ClCompile Include="..\lib_rend\ivas_td_ring_buffer.c" />

    <ClCompile Include="..\lib_rend\ivas_vbap.c" />

    <ClCompile Include="..\lib_rend\lib_rend.c" />

  </ItemGroup>

apps/renderer.c

+60 −49
Original line number Diff line number Diff line
@@ -724,6 +724,7 @@ int main( 
    SplitFileReadWrite *hSplitRendFileReadWrite;

    int16_t delayNumSamples_temp;

    int32_t delayTimeScale_temp;

    bool flushRendererLastFrame = false;

    int16_t numSamplesRead;

    int16_t delayNumSamples = -1;

    int16_t delayNumSamples_orig = 0;
@@ -770,8 +771,11 @@ int main( 


    CmdlnArgs args = parseCmdlnArgs( argc, argv );



    if ( args.nonDiegeticPan && !( ( args.inConfig.numAudioObjects == 0 && args.inConfig.multiChannelBuses[0].audioConfig == IVAS_AUDIO_CONFIG_MONO ) ||

                                   ( args.inConfig.numAudioObjects > 0 && args.inConfig.audioObjects[0].audioConfig == IVAS_AUDIO_CONFIG_OBA && args.inConfig.numAudioObjects == 1 ) ) )

    if ( args.nonDiegeticPan &&

         !( ( args.inConfig.numAudioObjects == 0 &&

              args.inConfig.multiChannelBuses[0].audioConfig == IVAS_AUDIO_CONFIG_MONO ) ||

            ( args.inConfig.numAudioObjects > 0 &&

              args.inConfig.audioObjects[0].audioConfig == IVAS_AUDIO_CONFIG_OBA && args.inConfig.numAudioObjects == 1 ) ) )

    {

        fprintf( stderr, "\ninvalid configuration - non-diegetic panning requires mono or ISM1 input\n" );

        goto cleanup;
@@ -1114,7 +1118,10 @@ int main( 
        IVAS_RENDER_CONFIG_DATA renderConfig;

#endif

        /* sanity check */

        if ( ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL ) && ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR ) && ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && !is_split_pre_rend_mode( &args ) )

        if ( ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL ) &&

             ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR ) &&

             ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) &&

             !is_split_pre_rend_mode( &args ) )

        {

            fprintf( stderr, "\nExternal Renderer Config is supported only when binaural output configurations is used as output OR when Split pre-rendering mode is enabled. Exiting. \n" );

            goto cleanup;
@@ -1409,7 +1416,7 @@ int main( 
    }



    int16_t numOutChannels;

    if ( ( error = IVAS_REND_NumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK )

    if ( ( error = IVAS_REND_GetNumOutChannels( hIvasRend, &numOutChannels ) ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "\nError in IVAS_REND_NumOutChannels(): %s\n", ivas_error_to_string( error ) );

        goto cleanup;
@@ -1424,31 +1431,16 @@ int main( 
        }

    }



    if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED )

    {

        if ( ( error = IVAS_REND_GetSplitRendBitstreamHeader( hIvasRend, &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection, &bitsBuffer.config.codec_frame_size_ms, &bitsBuffer.config.isar_frame_size_ms ) ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nError in IVAS_REND_GetSplitRendBitstreamHeader()!\n" );

            goto cleanup;

        }



        if ( IVAS_REND_GetDelay( hIvasRend, &delayNumSamples_temp, &delayTimeScale_temp ) != IVAS_ERR_OK )

    if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM )

    {

            fprintf( stderr, "\nUnable to get delay of renderer!\n" );

            goto cleanup;

        }

        char *outFile = args.outMetadataFilePath;



        if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, args.outputFilePath, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK )

        if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED )

        {

            fprintf( stderr, "\nCould not open split rend metadata file %s\n", args.outputFilePath );

            goto cleanup;

        }

            outFile = args.outputFilePath;

            audioWriter = NULL;

        }

    else

    {

        if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM )

        {



        if ( ( error = IVAS_REND_GetSplitRendBitstreamHeader( hIvasRend, &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection, &bitsBuffer.config.codec_frame_size_ms, &bitsBuffer.config.isar_frame_size_ms ) ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nError in IVAS_REND_GetSplitRendBitstreamHeader()!\n" );
@@ -1461,13 +1453,15 @@ int main( 
            goto cleanup;

        }



            if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, args.outMetadataFilePath, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK )

        if ( ( error = split_rend_writer_open( &hSplitRendFileReadWrite, outFile, delayNumSamples_temp, delayTimeScale_temp, bitsBuffer.config.codec, bitsBuffer.config.poseCorrection, bitsBuffer.config.codec_frame_size_ms, bitsBuffer.config.isar_frame_size_ms, args.sampleRate, bitsBuffer.config.lc3plus_highres ) ) != IVAS_ERR_OK )

        {

                fprintf( stderr, "\nCould not open split rend metadata file %s\n", args.outMetadataFilePath );

            fprintf( stderr, "\nCould not open split rend metadata file %s\n", outFile );

            goto cleanup;

        }

    }



    if ( args.outConfig.audioConfig != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED )

    {

        if ( AudioFileWriter_open( &audioWriter, args.outputFilePath, args.sampleRate, numOutChannels ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nFailed to open file: %s\n", args.outputFilePath );
@@ -1597,16 +1591,23 @@ int main( 
        if ( numSamplesRead == 0 )

        {

            /* end of input data */

            break;

            flushRendererLastFrame = true;

        }



        /* Convert from int to float and from interleaved to packed */

        if ( !flushRendererLastFrame )

        {

            convertInputBuffer( inpInt16Buffer, numSamplesRead, inBuffer.config.numSamplesPerChannel, num_in_channels, inFloatBuffer, inBuffer.config.is_cldfb, cldfbAna );

        }

        else

        {

            memset( inBuffer.data, 0, inBuffer.config.numChannels * inBuffer.config.numSamplesPerChannel * sizeof( float ) );

        }



        int16_t num_subframes, sf_idx;

        num_subframes = (int16_t) args.render_framesize;



        if ( isCurrentFrameMultipleOf20ms )

        if ( isCurrentFrameMultipleOf20ms && !flushRendererLastFrame )

        {

            IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer );
@@ -1718,7 +1719,7 @@ int main( 
            }

            IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.multiChannelBuses[i].inputChannelIndex, numChannels );



            if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, mcIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

            if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, mcIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                goto cleanup;
@@ -1733,7 +1734,7 @@ int main( 
                {

                    IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.audioObjects[i].inputChannelIndex, args.inConfig.numAudioObjects );



                    if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

                    if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

                    {

                        fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                        goto cleanup;
@@ -1750,7 +1751,7 @@ int main( 
            {

                IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.audioObjects[i].inputChannelIndex, 1 );



                if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

                if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, ismIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

                {

                    fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                    goto cleanup;
@@ -1773,7 +1774,7 @@ int main( 
            }

            IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.ambisonicsBuses[i].inputChannelIndex, numChannels );



            if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, sbaIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

            if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, sbaIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nIVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                goto cleanup;
@@ -1789,13 +1790,13 @@ int main( 
            }

            IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.masaBuses[i].inputChannelIndex, numChannels );



            if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, masaIds[i], tmpBuffer ) ) != IVAS_ERR_OK )

            if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, masaIds[i], tmpBuffer, flushRendererLastFrame ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "IVAS_REND_FeedInputAudio failed: %s\n", ivas_error_to_string( error ) );

                goto cleanup;

            }



            if ( isCurrentFrameMultipleOf20ms )

            if ( isCurrentFrameMultipleOf20ms && !flushRendererLastFrame )

            {

                if ( masaReaders[i] != NULL )

                {
@@ -1858,7 +1859,7 @@ int main( 
            zeroPad = delayNumSamples;

        }



        if ( is_split_pre_rend_mode( &args ) )

        if ( is_split_pre_rend_mode( &args ) && !flushRendererLastFrame )

        {

            if ( split_rend_write_bitstream_to_file( hSplitRendFileReadWrite, bitsBuffer.bits, &bitsBuffer.config.bitsRead,

                                                     &bitsBuffer.config.bitsWritten ) != IVAS_ERR_OK )
@@ -1868,7 +1869,7 @@ int main( 
            }

        }



        if ( audioWriter != NULL )

        if ( audioWriter != NULL && !flushRendererLastFrame )

        {

            if ( delayNumSamples * num_out_channels < outBufferSize )

            {
@@ -1889,7 +1890,7 @@ int main( 
        bitsBuffer.config.bitsWritten = 0;



        /* Write MASA metadata for MASA outputs */

        if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA1 || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA2 )

        if ( !flushRendererLastFrame && ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA1 || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_MASA2 ) )

        {

            IVAS_REND_AudioConfigType inputType1;

            IVAS_REND_AudioConfigType inputType2;
@@ -1961,7 +1962,8 @@ int main( 
                    }

                }



                if ( ( args.inConfig.numAmbisonicsBuses > 0 || args.inConfig.numMultiChannelBuses > 0 || args.inConfig.numMasaBuses > 0 ) && args.inConfig.numAudioObjects > 0 )

                if ( ( args.inConfig.numAmbisonicsBuses > 0 || args.inConfig.numMultiChannelBuses > 0 || args.inConfig.numMasaBuses > 0 ) &&

                     args.inConfig.numAudioObjects > 0 )

                {

                    inputType2 = IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED;

                    if ( ( error = IVAS_REND_MergeMasaMetadata( hIvasRend, &hMetaOutput, inputType1, inputType2 ) ) != IVAS_ERR_OK )
@@ -1977,6 +1979,13 @@ int main( 
            }

        }



        /* no new input was actually read, only delay buffers were flushed

         * therefore this is not a real frame */

        if ( flushRendererLastFrame )

        {

            break;

        }



        frame++;

        if ( !args.quietModeEnabled )

        {
@@ -1989,12 +1998,13 @@ int main( 
#endif

    }



    /* add zeros at the end to have equal length of synthesized signals */

    /* add zeros at the end to have equal length of synthesized signals

     * the output buffer will contain either leftover input samples from delay aligned inputs

     * or zeros for padding */

    if ( audioWriter != NULL )

    {

        for ( zeroPadToWrite = zeroPad; zeroPadToWrite > frameSize_smpls; zeroPadToWrite -= frameSize_smpls )

        {

            memset( outInt16Buffer, 0, outBufferSize * sizeof( int16_t ) );

            if ( ( error = AudioFileWriter_write( audioWriter, outInt16Buffer, outBufferSize ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nOutput audio file writer error\n" );
@@ -2002,7 +2012,6 @@ int main( 
            }

        }



        memset( outInt16Buffer, 0, zeroPadToWrite * outBuffer.config.numChannels * sizeof( int16_t ) );

        if ( ( error = AudioFileWriter_write( audioWriter, outInt16Buffer, zeroPadToWrite * outBuffer.config.numChannels ) ) != IVAS_ERR_OK )

        {

            fprintf( stderr, "\nOutput audio file writer error\n" );
@@ -2011,9 +2020,10 @@ int main( 
        zeroPadToWrite = 0;

    }



    if ( args.inConfig.numAudioObjects != 0 && ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) )

    if ( args.inConfig.numAudioObjects != 0 &&

         ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) )

    {

        fprintf( stdout, "\n\nMetadata delayed %d subframes\n\n", (int16_t) round( args.syncMdDelay / ( 1000 / IVAS_NUM_FRAMES_PER_SEC / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ) ) );

        fprintf( stdout, "\n\nMetadata delayed %d subframes\n\n", (int16_t) round( args.syncMdDelay / ( 1000.f / IVAS_NUM_FRAMES_PER_SEC / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ) ) );

    }



    if ( !args.quietModeEnabled && args.delayCompensationEnabled )
@@ -3433,7 +3443,8 @@ static void parseObjectPosition( 
    *positionDuration = (uint16_t) strtol( line, &endptr, 10 );

    readNextMetadataChunk( line, "\n" );



    read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f", &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] );

    read_values = (int16_t) sscanf( line, "%f,%f,%f,%f,%f,%f,%f,%f",

                                    &meta_prm[0], &meta_prm[1], &meta_prm[2], &meta_prm[3], &meta_prm[4], &meta_prm[5], &meta_prm[6], &meta_prm[7] );



    if ( read_values < 2 )

    {

lib_isar/lib_isar_pre_rend.c

+1 −3
Original line number Diff line number Diff line
@@ -115,9 +115,7 @@ ivas_error ISAR_PRE_REND_open( 
        isCldfbNeeded = 1;

    }



    hSplitBinRend->hCldfbHandles = NULL;



    if ( isCldfbNeeded )

    if ( isCldfbNeeded && hSplitBinRend->hCldfbHandles == NULL )

    {

        if ( ( hSplitBinRend->hCldfbHandles = (CLDFB_HANDLES_WRAPPER_HANDLE) malloc( sizeof( CLDFB_HANDLES_WRAPPER ) ) ) == NULL )

        {

lib_rend/ivas_prot_rend.h

+34 −0
Original line number Diff line number Diff line
@@ -1622,6 +1622,40 @@ void ivas_rend_closeCldfbRend( 
    CLDFB_REND_WRAPPER *pCldfbRend 

);



/*----------------------------------------------------------------------------------*

 * Time domain ring buffer prototypes

 *----------------------------------------------------------------------------------*/



ivas_error ivas_TD_RINGBUF_Open(

    TD_RINGBUF_HANDLE *ph,                                      /* i/o: Ring buffer handle                     */

    const uint32_t capacity_per_channel,                        /* i  : Number of samples stored per channel   */

    const uint16_t num_channels                                 /* i  : Number of channels                     */

);



void ivas_TD_RINGBUF_Close( 

    TD_RINGBUF_HANDLE *ph                                       /* i/o: Ring buffer handle                      */

);



void ivas_TD_RINGBUF_Push( 

    TD_RINGBUF_HANDLE h,                                        /* i/o: Ring buffer handle                       */

    const float *data,                                          /* i  : Input data                               */

    const uint32_t num_samples_per_channel                      /* i  : Number of samples per channel to store   */

);



void ivas_TD_RINGBUF_PushZeros( 

    TD_RINGBUF_HANDLE h,                                        /* i/o: Ring buffer handle                      */

    const uint32_t num_samples_per_channel                      /* i  : Number of zeros per channel to store    */

);



void ivas_TD_RINGBUF_Pop( 

    TD_RINGBUF_HANDLE h,                                        /* i/o: Ring buffer handle                      */

    float *data,                                                /* i  : Output data                             */

    const uint32_t num_samples_per_channel                      /* i  : Number of samples per channel to retrieve*/

);



uint32_t ivas_TD_RINGBUF_Size(

    const TD_RINGBUF_HANDLE h                                   /* i  : Ring buffer handle                      */

);



/* clang-format on */

lib_rend/ivas_stat_rend.h

+16 −0
Original line number Diff line number Diff line
@@ -1498,6 +1498,22 @@ typedef struct 
} CLDFB_REND_WRAPPER;





/*----------------------------------------------------------------------------------*

 * Time domain ring buffer structure

 *----------------------------------------------------------------------------------*/



typedef struct

{

    float *data; /* samples in interleaved layout */

    uint32_t capacity;

    uint16_t num_channels;

    uint32_t write_pos;

    uint32_t read_pos;

    int16_t is_full;



} TD_RINGBUF_DATA, *TD_RINGBUF_HANDLE;





/*----------------------------------------------------------------------------------*

 * MASA external renderer structure

 *----------------------------------------------------------------------------------*/