Commit c4d393b2 authored by Archit Tamarapu's avatar Archit Tamarapu
Browse files

add split rendering support for MC format

parent b8c5069e

lib_com/options.h

+1 −0
Original line number Diff line number Diff line
@@ -180,6 +180,7 @@ 
#define SPLIT_REND_STACK_OPT



#define SPLIT_REND_LC3PLUS                              /* FhG: split rendering using LC3plus codec */

#define SPLIT_REND_LC3PLUS_MC                           /* FhG: split rendering using LC3plus codec for Multichannel content */

#define FIX_SPLIT_REND_OPEN_ERROR_HANDLING              /* adds missing error handling around ivas_split_renderer_open calls  */

// #define SPLIT_REND_LC3PLUS_HBR_DBG                      /* Override bitrate per LC3 core */

#endif

lib_dec/ivas_dec.c

+43 −14
Original line number Diff line number Diff line
@@ -107,7 +107,14 @@ ivas_error ivas_dec( 
         ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) )

    {

#ifdef SPLIT_REND_LC3PLUS

#ifdef SPLIT_REND_LC3PLUS_MC

        assert( ( st_ivas->ivas_format == SBA_FORMAT ||

                  ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_DISC ) ||

                  ( st_ivas->ivas_format == MC_FORMAT && st_ivas->mc_mode == MC_MODE_MCT ) ) &&

                ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA, discrete ISM, or MCT-MC formats and 48 kHz sampling rate only" );

#else

        assert( ( st_ivas->ivas_format == SBA_FORMAT || ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_DISC ) ) && ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA or discrete ISM format and 48k sampling rate only" );

#endif

#else

        assert( st_ivas->ivas_format == SBA_FORMAT && ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA format and 48k sampling rate only" );

#endif
@@ -511,6 +518,14 @@ ivas_error ivas_dec( 
            /* Rendering */

            if ( st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV || st_ivas->renderer_type == RENDERER_BINAURAL_MIXER_CONV_ROOM )

            {

#ifdef SPLIT_REND_LC3PLUS_MC

                if ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CLDFB )

                {

                    ivas_rend_crendProcessSplitBin( st_ivas, output, output_frame );

                }

                else

                {

#endif

                    if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig,

                                                           st_ivas->hCombinedOrientationData, &st_ivas->hIntSetup, st_ivas->hEFAPdata, p_output, output_Fs

#ifdef SPLIT_REND_WITH_HEAD_ROT
@@ -524,6 +539,9 @@ ivas_error ivas_dec( 
                    }



                    ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output );

#ifdef SPLIT_REND_LC3PLUS_MC

                }

#endif

            }

            else if ( st_ivas->renderer_type == RENDERER_MC )

            {
@@ -535,12 +553,23 @@ ivas_error ivas_dec( 
            }

            else if ( st_ivas->renderer_type == RENDERER_BINAURAL_OBJECTS_TD )

            {

#ifdef SPLIT_REND_LC3PLUS_MC

                if ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CLDFB )

                {

                    ObjRenderIvasFrame_splitBinaural( st_ivas, output, output_frame ); // TODO tmu check

                }

                else

                {

#endif

                    if ( ( ivas_td_binaural_renderer( st_ivas, p_output, output_frame ) ) != IVAS_ERR_OK )

                    {

                        return error;

                    }



                    ivas_binaural_add_LFE( st_ivas, output_frame, p_output, p_output );

#ifdef SPLIT_REND_LC3PLUS_MC

                }

#endif

            }

        }

        else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX )

lib_dec/ivas_init_dec.c

+26 −1
Original line number Diff line number Diff line
@@ -1333,7 +1333,7 @@ ivas_error ivas_init_decoder( 
                &st_ivas->hRenderConfig->split_rend_config,

                &st_ivas->splitBinRend.splitrend.multiBinPoseData,

                st_ivas->hHeadTrackData->sr_pose_pred_axis );

            /* Split rendering with ISMs only supported with LC3plus */

            /* Split rendering with MC/ISMs only supported with LC3plus */

#ifndef SPLIT_REND_CLDFB_ISM

            assert( st_ivas->hRenderConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS );

#endif
@@ -1392,12 +1392,37 @@ ivas_error ivas_init_decoder( 
                return error;

            }

        }

#ifdef SPLIT_REND_LC3PLUS_MC



        if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CLDFB )

        {

            ivas_renderSplitGetMultiBinPoseData(

                &st_ivas->hRenderConfig->split_rend_config,

                &st_ivas->splitBinRend.splitrend.multiBinPoseData,

                st_ivas->hHeadTrackData->sr_pose_pred_axis );



            /* Split rendering with MC only supported with LC3plus */

            assert( st_ivas->hRenderConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS );



            error = ivas_split_renderer_open( &st_ivas->splitBinRend.splitrend,

                                              &st_ivas->hRenderConfig->split_rend_config,

                                              hDecoderConfig->output_Fs, 0, 0 );

            if ( error != IVAS_ERR_OK )

            {

                return error;

            }

        }

#endif



        if ( ( error = ivas_rend_openCrend( &( st_ivas->hCrendWrapper ), st_ivas->intern_config, st_ivas->hDecoderConfig->output_config,

                                            st_ivas->hRenderConfig, st_ivas->hSetOfHRTF, st_ivas->hDecoderConfig->output_Fs

#ifdef SPLIT_REND_WITH_HEAD_ROT

                                            ,

#ifdef SPLIT_REND_LC3PLUS_MC

                                            ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CLDFB ) ? st_ivas->splitBinRend.splitrend.multiBinPoseData.num_poses : 1

#else

                                            1

#endif

#endif

                                            ) ) != IVAS_ERR_OK )

        {

lib_dec/ivas_output_config.c

+4 −0
Original line number Diff line number Diff line
@@ -203,7 +203,11 @@ void ivas_renderer_select( 
            else

            {

                *internal_config = transport_config;

#ifdef SPLIT_REND_LC3PLUS_MC

                if ( output_config == AUDIO_CONFIG_BINAURAL || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CLDFB || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM )

#else

                if ( output_config == AUDIO_CONFIG_BINAURAL )

#endif

                {

#ifdef DEBUGGING

                    if ( ( ( ( st_ivas->transport_config == AUDIO_CONFIG_5_1 || st_ivas->transport_config == AUDIO_CONFIG_7_1 ) && ( st_ivas->hDecoderConfig->Opt_Headrotation || st_ivas->hDecoderConfig->Opt_ExternalOrientation ) ) || ( st_ivas->hDecoderConfig->force_rend == FORCE_TD_RENDERER ) ) && ( st_ivas->mc_mode == MC_MODE_MCT || st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) && !( st_ivas->hDecoderConfig->force_rend == FORCE_CLDFB_RENDERER ) )

lib_rend/ivas_crend.c

+116 −0
Original line number Diff line number Diff line
@@ -1741,3 +1741,119 @@ ivas_error ivas_rend_crendProcessSubframe( 


    return IVAS_ERR_OK;

}

#ifdef SPLIT_REND_LC3PLUS_MC





/*-----------------------------------------------------------------------------------------*

 * Function ivas_rend_crend_ProcessSplitBin()

 *

 * Process call for IVAS Crend renderer

 *-----------------------------------------------------------------------------------------*/



ivas_error ivas_rend_crendProcessSplitBin(

    Decoder_Struct *st_ivas,    /* i/o: IVAS decoder structure            */

    float output[][L_FRAME48k], /* i/o: SCE channels / Binaural synthesis */

    const int16_t output_frame  /* i  : output frame length               */

)

{

    int16_t i;

    int16_t pos_idx;

    float gain_lfe;

    float tmpLfeBuffer[L_FRAME48k];

    float tmpInputBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k];

    float tmpSplitBinBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k];

    IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES];

    MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData;

    ivas_error error;



    pMultiBinPoseData = &st_ivas->splitBinRend.splitrend.multiBinPoseData;



    /* Save current head positions */

    for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i )

    {

        originalHeadRot[i] = st_ivas->hHeadTrackData->Quaternions[i];

    }



    /* copy input */

    for ( i = 0; i < st_ivas->nchan_transport; ++i )

    {

        mvr2r( output[i], tmpInputBuffer[i], output_frame );

    }



    /* TODO tmu : partial duplication of ivas_binaural_add_LFE() for efficiency */

    /* copy LFE */

    mvr2r( output[st_ivas->hIntSetup.index_lfe[0]], tmpLfeBuffer, output_frame );

    gain_lfe = ( ( st_ivas->hCrendWrapper != NULL ) && ( st_ivas->hCrendWrapper->hHrtfCrend != NULL ) ) ? st_ivas->hCrendWrapper->hHrtfCrend->gain_lfe : GAIN_LFE;

    v_multc( tmpLfeBuffer, gain_lfe, tmpLfeBuffer, output_frame );



    for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; ++pos_idx )

    {

        /* Update head positions */

        if ( pos_idx != 0 )

        {

            for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i )

            {

                if ( originalHeadRot[i].w == -3.0f )

                {

                    st_ivas->hHeadTrackData->Quaternions[i].w = -3.0f;

                    st_ivas->hHeadTrackData->Quaternions[i].x = originalHeadRot[i].x + pMultiBinPoseData->relative_head_poses[pos_idx][0];

                    st_ivas->hHeadTrackData->Quaternions[i].y = originalHeadRot[i].y + pMultiBinPoseData->relative_head_poses[pos_idx][1];

                    st_ivas->hHeadTrackData->Quaternions[i].z = originalHeadRot[i].z + pMultiBinPoseData->relative_head_poses[pos_idx][2];

                }

                else

                {

                    st_ivas->hHeadTrackData->Quaternions[i].w = -3.0f;

                    Quat2EulerDegree( originalHeadRot[i], /* TODO tmu : fix bug with ordering*/

                                      &st_ivas->hHeadTrackData->Quaternions[i].z,

                                      &st_ivas->hHeadTrackData->Quaternions[i].y,

                                      &st_ivas->hHeadTrackData->Quaternions[i].x );

                    st_ivas->hHeadTrackData->Quaternions[i].x += pMultiBinPoseData->relative_head_poses[pos_idx][0];

                    st_ivas->hHeadTrackData->Quaternions[i].y += pMultiBinPoseData->relative_head_poses[pos_idx][1];

                    st_ivas->hHeadTrackData->Quaternions[i].z += pMultiBinPoseData->relative_head_poses[pos_idx][2];

                }

            }

        }



        /* render inplace to first two channels of tmpInputBuffer */

        if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper,

                                               st_ivas->intern_config,

                                               st_ivas->hOutSetup.output_config,

                                               st_ivas->hDecoderConfig,

                                               st_ivas->hHeadTrackData,

                                               &st_ivas->hIntSetup,

                                               st_ivas->hEFAPdata,

                                               tmpInputBuffer,

                                               st_ivas->hDecoderConfig->output_Fs,

                                               pos_idx ) ) != IVAS_ERR_OK )

        {

            return error;

        }



        for ( i = 0; i < BINAURAL_CHANNELS; ++i )

        {

            /* accumulate LFE to output */

            v_add( tmpInputBuffer[i], tmpLfeBuffer, tmpInputBuffer[i], output_frame );



            /* move to split bin output buffer */

            mvr2r( tmpInputBuffer[i], tmpSplitBinBuffer[pos_idx * BINAURAL_CHANNELS + i], output_frame );



            /* overwrite rendered channels with input again for next iteration */

            mvr2r( output[i], tmpInputBuffer[i], output_frame );

        }

    }



    /* copy split binaural rendered signals to final output */

    for ( i = 0; i < BINAURAL_CHANNELS * pMultiBinPoseData->num_poses; ++i )

    {

        mvr2r( tmpSplitBinBuffer[i], output[i], output_frame );

    }



    /* Restore original head rotation */

    for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i )

    {

        st_ivas->hHeadTrackData->Quaternions[i] = originalHeadRot[i];

    }



    return IVAS_ERR_OK;

}

#endif