Commit a38f9e50 authored by Tapani Pihlajakuja's avatar Tapani Pihlajakuja
Browse files

Merge branch 'nokia/contribution-28-adaptive-transport-param-bin' into 'main' 

[non-BE] Merge contribution 28 - Adaptive transport audio signals for head tracked parametric binauralization

See merge request !531
parents 192200eb ae569d64

lib_com/options.h

+2 −0
Original line number Diff line number Diff line
@@ -154,6 +154,8 @@ 
#define SBA2MONO                                        /* FhG: Issue 365: Adapt processing of SBA mono output to be in line with stereo output (less delay, lower complexity) */



#define NOKIA_PARAMBIN_REQULARIZATION                   /* Nokia: Contribution - Configured reqularization factor for parametric binauralizer. */

#define NOKIA_ADAPTIVE_BINAURAL_PROTOS                  /* Nokia: Contribution 28: Adaptive binaural prototypes */

#define NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT              /* Nokia: enable adaptive binaural prototype complexity optimizations */



/* ################## End DEVELOPMENT switches ######################### */

/* clang-format on */

lib_rend/ivas_dirac_dec_binaural_functions.c

+218 −0
Original line number Diff line number Diff line
@@ -40,6 +40,10 @@ 
#include "ivas_cnst.h"

#include "ivas_rom_binauralRenderer.h"

#include "ivas_rom_rend.h"

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

#include "ivas_rom_com.h"

#endif



#ifdef DEBUGGING

#include "debug.h"

#endif
@@ -54,6 +58,20 @@ 
#define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN        ( 2.0f )

#define IVAS_TDET_DUCK_MULT_FAC_PARA_BIN_LOW_BR ( 3.0f )



#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT

/* powf(0.95f, 4.0f) for sub-frame smoothing instead of CLDFB slot */

#define ADAPT_HTPROTO_IIR_FAC 0.81450625f

#else

#define ADAPT_HTPROTO_IIR_FAC 0.95f

#endif



#define ADAPT_HTPROTO_ILD_LIM_DB0 1.0f

#define ADAPT_HTPROTO_ILD_LIM_DB1 4.0f

#define ADAPT_HTPROTO_ROT_LIM_0   0.4f

#define ADAPT_HTPROTO_ROT_LIM_1   0.8f

#endif



/*-------------------------------------------------------------------------

 * Local function prototypes

 *------------------------------------------------------------------------*/
@@ -68,6 +86,10 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( Decoder_Struc 


static void ivas_dirac_dec_binaural_process_output( Decoder_Struct *st_ivas, float output_f[][L_FRAME48k], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t max_band_decorr, const uint8_t numInputChannels, const uint8_t firstSlot, const uint8_t slotEnd );



#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

static void adaptTransportSignalsHeadtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] );

#endif



static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( HEAD_TRACK_DATA_HANDLE hHeadTrackData, float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const uint8_t firstSlot, const uint8_t slotEnd, const uint8_t nBins, float Rmat[3][3] );



static void formulate2x2MixingMatrix( float Ein1, float Ein2, float CinRe, float CinIm, float Eout1, float Eout2, float CoutRe, float CoutIm, float Q[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Mre[BINAURAL_CHANNELS][BINAURAL_CHANNELS], float Mim[BINAURAL_CHANNELS][BINAURAL_CHANNELS], const float regularizationFactor );
@@ -554,6 +576,10 @@ static void ivas_dirac_dec_binaural_internal( 


        if ( nchan_transport == 2 )

        {

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

            adaptTransportSignalsHeadtracked( st_ivas->hHeadTrackData, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, firstSlot, slotEnd, nBins, Rmat );

#endif



            ivas_dirac_dec_binaural_check_and_switch_transports_headtracked( st_ivas->hHeadTrackData, Cldfb_ImagBuffer_in, Cldfb_RealBuffer_in, firstSlot, slotEnd, nBins, Rmat );

        }

    }
@@ -1332,6 +1358,198 @@ static void ivas_dirac_dec_binaural_process_output( 
}





#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

static void adaptTransportSignalsHeadtracked(

    HEAD_TRACK_DATA_HANDLE hHeadTrackData,

    float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX],

    float inRe[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX],

    const uint8_t firstSlot,

    const uint8_t slotEnd,

    const uint8_t nBins,

    float Rmat[3][3] )

{

    int16_t slot, ch, bin, louderCh;

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT

    float ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val, ene_proc, ene_target;

    uint8_t n_slots_per_sf, sf_idx, n_sf;

    int16_t max_band;

#else

    float re[2], im[2], ILD, mono_factor_ILD, mono_factor_rotation, mono_factor, y_val;

    float proc_re[2], proc_im[2], sum_re, sum_im, ene_proc, ene_target, mf;

#endif



    /* Determine head-orientation-based mono factor.

       Rmat[1][1] entry informs how close the ears are aligned according to transport signals. */

    y_val = 1.0f - fabsf( Rmat[1][1] );

    mono_factor_rotation = ( y_val - ADAPT_HTPROTO_ROT_LIM_0 ) / ( ADAPT_HTPROTO_ROT_LIM_1 - ADAPT_HTPROTO_ROT_LIM_0 );

    mono_factor_rotation = fmaxf( 0.0f, fminf( 1.0f, mono_factor_rotation ) );



    /* Adapt transport signals in frequency bands */

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT

    /* optimization grouping CLDFB bins into MASA bands (they are readily available in ROM and suitable for the task) AND group CLDFB slots into sub-frames */

    n_slots_per_sf = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES;

    n_sf = ( slotEnd - firstSlot ) / n_slots_per_sf;



    max_band = 0;

    while ( max_band < MASA_FREQUENCY_BANDS && MASA_band_grouping_24[max_band] < nBins )

    {

        max_band++;

    }



    for ( sf_idx = 0; sf_idx < n_sf; sf_idx++ )

    {

        float eqVal;

        uint8_t start_slot, stop_slot;

        int16_t band_idx, bin_lo, bin_hi;



        start_slot = firstSlot + sf_idx * n_slots_per_sf;

        stop_slot = start_slot + n_slots_per_sf;



        for ( band_idx = 0; band_idx < max_band; band_idx++ )

        {

            float ch_nrg[2]; /* storage for input signal channel energies */

            bin_lo = MASA_band_grouping_24[band_idx];

            bin_hi = min( MASA_band_grouping_24[band_idx + 1], (int16_t) nBins );

            for ( ch = 0; ch < 2; ch++ )

            {

                ch_nrg[ch] = 0.0f;

                for ( slot = start_slot; slot < stop_slot; slot++ )

                {

                    for ( bin = bin_lo; bin < bin_hi; bin++ )

                    {

                        ch_nrg[ch] += ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ) + ( inIm[ch][slot][bin] * inIm[ch][slot][bin] );

                    }

                }

                hHeadTrackData->chEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC;

                hHeadTrackData->chEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ch_nrg[ch];

            }



            /* Determine ILD */

            ILD = fabsf( 10.0f * log10f( fmaxf( 1e-12f, hHeadTrackData->chEneIIR[0][band_idx] ) / fmaxf( 1e-12f, hHeadTrackData->chEneIIR[1][band_idx] ) ) );

            if ( hHeadTrackData->chEneIIR[1][band_idx] > hHeadTrackData->chEneIIR[0][band_idx] )

            {

                louderCh = 1;

            }

            else

            {

                louderCh = 0;

            }



            /* Determine ILD-based mono factor */

            mono_factor_ILD = ( ILD - ADAPT_HTPROTO_ILD_LIM_DB0 ) / ( ADAPT_HTPROTO_ILD_LIM_DB1 - ADAPT_HTPROTO_ILD_LIM_DB0 );

            mono_factor_ILD = fmaxf( 0.0f, fminf( 1.0f, mono_factor_ILD ) );



            /* Combine mono factors */

            mono_factor = mono_factor_ILD * mono_factor_rotation;



            /* Mix original audio and sum signal according to determined mono factor */

            for ( ch = 0; ch < 2; ch++ )

            {

                if ( ch != louderCh )

                {

                    float band_nrg = 0.0f;



                    for ( slot = start_slot; slot < stop_slot; slot++ )

                    {

                        for ( bin = bin_lo; bin < bin_hi; bin++ )

                        {

                            /* mono sum signal with the computed weight + rest from the original channel */

                            inRe[ch][slot][bin] = mono_factor * ( inRe[0][slot][bin] + inRe[1][slot][bin] ) + ( 1.0f - mono_factor ) * inRe[ch][slot][bin];

                            inIm[ch][slot][bin] = mono_factor * ( inIm[0][slot][bin] + inIm[1][slot][bin] ) + ( 1.0f - mono_factor ) * inIm[ch][slot][bin];

                            band_nrg += ( inRe[ch][slot][bin] * inRe[ch][slot][bin] ) + ( inIm[ch][slot][bin] * inIm[ch][slot][bin] );

                        }

                    }

                    hHeadTrackData->procChEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC;

                    hHeadTrackData->procChEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * band_nrg;

                }

                else

                {

                    /* processed signal is input. use the original channel, so no need to compute new signals or signal energy */

                    hHeadTrackData->procChEneIIR[ch][band_idx] *= ADAPT_HTPROTO_IIR_FAC;

                    hHeadTrackData->procChEneIIR[ch][band_idx] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ch_nrg[ch];

                }

            }



            /* Equalize */

            ene_target = hHeadTrackData->chEneIIR[0][band_idx] + hHeadTrackData->chEneIIR[1][band_idx];

            ene_proc = hHeadTrackData->procChEneIIR[0][band_idx] + hHeadTrackData->procChEneIIR[1][band_idx];

            eqVal = fminf( 4.0f, sqrtf( ene_target / fmaxf( 1e-12f, ene_proc ) ) );



            for ( slot = start_slot; slot < stop_slot; slot++ )

            {

                for ( ch = 0; ch < 2; ch++ )

                {

                    for ( bin = bin_lo; bin < bin_hi; bin++ )

                    {

                        inRe[ch][slot][bin] *= eqVal;

                        inIm[ch][slot][bin] *= eqVal;

                    }

                }

            }

        }

    }

#else

    /* original contribution */

    for ( slot = firstSlot; slot < slotEnd; slot++ )

    {

        float eqVal[60];



        for ( bin = 0; bin < nBins; bin++ )

        {

            /* Determine channel energies */

            for ( ch = 0; ch < 2; ch++ )

            {

                re[ch] = inRe[ch][slot][bin];

                im[ch] = inIm[ch][slot][bin];



                hHeadTrackData->chEneIIR[ch][bin] *= ADAPT_HTPROTO_IIR_FAC;

                hHeadTrackData->chEneIIR[ch][bin] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ( ( re[ch] * re[ch] ) + ( im[ch] * im[ch] ) );

            }



            /* Determine ILD */

            ILD = fabsf( 10.0f * log10f( fmaxf( 1e-12f, hHeadTrackData->chEneIIR[0][bin] ) / fmaxf( 1e-12f, hHeadTrackData->chEneIIR[1][bin] ) ) );

            louderCh = ( hHeadTrackData->chEneIIR[1][bin] > hHeadTrackData->chEneIIR[0][bin] );



            /* Determine ILD-based mono factor */

            mono_factor_ILD = ( ILD - ADAPT_HTPROTO_ILD_LIM_DB0 ) / ( ADAPT_HTPROTO_ILD_LIM_DB1 - ADAPT_HTPROTO_ILD_LIM_DB0 );

            mono_factor_ILD = fmaxf( 0.0f, fminf( 1.0f, mono_factor_ILD ) );



            /* Combine mono factors */

            mono_factor = mono_factor_ILD * mono_factor_rotation;



            /* Mix original audio and sum signal according to determined mono factor */

            sum_re = re[0] + re[1];

            sum_im = im[0] + im[1];

            for ( ch = 0; ch < 2; ch++ )

            {

                mf = ( ch == louderCh ) ? 0.0f : mono_factor;



                proc_re[ch] = mf * sum_re + ( 1.0f - mf ) * re[ch];

                proc_im[ch] = mf * sum_im + ( 1.0f - mf ) * im[ch];



                hHeadTrackData->procChEneIIR[ch][bin] *= ADAPT_HTPROTO_IIR_FAC;

                hHeadTrackData->procChEneIIR[ch][bin] += ( 1.0f - ADAPT_HTPROTO_IIR_FAC ) * ( ( proc_re[ch] * proc_re[ch] ) + ( proc_im[ch] * proc_im[ch] ) );

            }



            /* Equalize */

            ene_target = hHeadTrackData->chEneIIR[0][bin] + hHeadTrackData->chEneIIR[1][bin];

            ene_proc = hHeadTrackData->procChEneIIR[0][bin] + hHeadTrackData->procChEneIIR[1][bin];

            eqVal[bin] = fminf( 4.0f, sqrtf( ene_target / fmaxf( 1e-12f, ene_proc ) ) );



            for ( ch = 0; ch < 2; ch++ )

            {

                inRe[ch][slot][bin] = proc_re[ch] * eqVal[bin];

                inIm[ch][slot][bin] = proc_im[ch] * eqVal[bin];

            }

        }

    }

#endif



    return;

}

#endif





static void ivas_dirac_dec_binaural_check_and_switch_transports_headtracked(

    HEAD_TRACK_DATA_HANDLE hHeadTrackData,

    float inIm[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX],

lib_rend/ivas_rotation.c

+14 −0
Original line number Diff line number Diff line
@@ -85,6 +85,20 @@ ivas_error ivas_headTrack_open( 
        ( *hHeadTrackData )->Rmat_prev[i][i] = 1.0f;

    }



#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT

    set_zero( ( *hHeadTrackData )->chEneIIR[0], MASA_FREQUENCY_BANDS );

    set_zero( ( *hHeadTrackData )->chEneIIR[1], MASA_FREQUENCY_BANDS );

    set_zero( ( *hHeadTrackData )->procChEneIIR[0], MASA_FREQUENCY_BANDS );

    set_zero( ( *hHeadTrackData )->procChEneIIR[1], MASA_FREQUENCY_BANDS );

#else

    set_zero( ( *hHeadTrackData )->chEneIIR[0], CLDFB_NO_CHANNELS_MAX );

    set_zero( ( *hHeadTrackData )->chEneIIR[1], CLDFB_NO_CHANNELS_MAX );

    set_zero( ( *hHeadTrackData )->procChEneIIR[0], CLDFB_NO_CHANNELS_MAX );

    set_zero( ( *hHeadTrackData )->procChEneIIR[1], CLDFB_NO_CHANNELS_MAX );

#endif

#endif



    return IVAS_ERR_OK;

}

lib_rend/ivas_stat_rend.h

+10 −0
Original line number Diff line number Diff line
@@ -268,6 +268,16 @@ typedef struct ivas_binaural_head_track_struct 
    uint8_t lrSwitchedCurrent;

    float lrSwitchInterpVal;



#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS

#ifdef NOKIA_ADAPTIVE_BINAURAL_PROTOS_OPT

    float chEneIIR[2][MASA_FREQUENCY_BANDS]; /* independent of the format. MASA bands are suitable for the task and readily available in ROM. */

    float procChEneIIR[2][MASA_FREQUENCY_BANDS];

#else

    float chEneIIR[2][CLDFB_NO_CHANNELS_MAX];

    float procChEneIIR[2][CLDFB_NO_CHANNELS_MAX];

#endif

#endif



    int16_t shd_rot_max_order;

    ivas_orient_trk_state_t *OrientationTracker;