Commit f0ee4c37 authored by bayers's avatar bayers
Browse files

adapt binaural reverb to variable subframe sizes, completely reintegrate JBM... 

adapt binaural reverb to variable subframe sizes, completely reintegrate JBM path into the parametric binaural rendering and remove duplicated functionality, fix typoes in ivas_mode.json, activate EXT interfache for JBM (not tested yet, but runs)
parent ca4fecd2

apps/decoder.c

+49 −0
Original line number Diff line number Diff line
@@ -2294,6 +2294,55 @@ static ivas_error decodeVoIP( 
            {

                delayNumSamples -= nOutSamples;

            }



#ifdef JBM_TSM_ON_TCS

            /* Write ISM metadata to external file(s) */

            if ( decodedGoodFrame && arg.outputFormat == IVAS_DEC_OUTPUT_EXT )

            {

                int16_t i;



                if ( bsFormat == IVAS_DEC_BS_OBJ )

                {

                    if ( ( error = IVAS_DEC_GetNumObjects( hIvasDec, &numObj ) ) != IVAS_ERR_OK )

                    {

                        fprintf( stderr, "\nError in IVAS_DEC_GetNumObjects: %s\n", IVAS_DEC_GetErrorMessage( error ) );

                        goto cleanup;

                    }



                    for ( i = 0; i < numObj; ++i )

                    {

                        IVAS_ISM_METADATA IsmMetadata;



                        if ( ( error = IVAS_DEC_GetObjectMetadata( hIvasDec, &IsmMetadata, 0, i ) ) != IVAS_ERR_OK )

                        {

                            fprintf( stderr, "\nError in IVAS_DEC_GetObjectMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );

                            goto cleanup;

                        }



                        if ( ( IsmFileWriter_writeFrame( IsmMetadata, ismWriters[i] ) ) != IVAS_ERR_OK )

                        {

                            fprintf( stderr, "\nError writing ISM metadata to file %s\n", IsmFileWriter_getFilePath( ismWriters[i] ) );

                            goto cleanup;

                        }

                    }

                }

                else if ( bsFormat == IVAS_DEC_BS_MASA )

                {

                    MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta;

                    if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta ) ) != IVAS_ERR_OK )

                    {

                        fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );

                        goto cleanup;

                    }



                    if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK )

                    {

                        fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) );

                        goto cleanup;

                    }

                }

            }

#endif

        }



        if ( !arg.quietModeEnabled )

lib_com/ivas_prot.h

+2 −0
Original line number Diff line number Diff line
@@ -5030,6 +5030,7 @@ void ivas_masa_prerender( 
);



#ifdef JBM_TSM_ON_TCS

#ifndef FIX_355_REFACTOR_PARAMBIN_TO_5MS

void ivas_spar_param_to_masa_param_mapping_sf(

    Decoder_Struct *st_ivas, /* i/o: IVAS decoder struct               */

#ifdef MASA_PARAMBIN_SF_MAP
@@ -5049,6 +5050,7 @@ void ivas_spar_param_to_masa_param_mapping_sf( 
    float inIm[][JBM_CLDFB_SLOTS_IN_SUBFRAME][CLDFB_NO_CHANNELS_MAX]  /* i  : Input audio in CLDFB domain, imag */

);

#endif

#endif



#ifdef FIX_355_REFACTOR_PARAMBIN_TO_5MS

void ivas_spar_param_to_masa_param_mapping(

lib_dec/ivas_binRenderer_internal.c

+4 −0
Original line number Diff line number Diff line
@@ -1206,7 +1206,11 @@ void ivas_binRenderer( 
        }



#ifdef FIX_355_REFACTOR_PARAMBIN_TO_5MS

#ifdef JBM_TSM_ON_TCS

        ivas_binaural_reverb_processSubframe( hBinRenderer->hReverb, BINAURAL_CHANNELS, numTimeSlots, inRe, inIm, reverbRe, reverbIm );

#else

        ivas_binaural_reverb_processSubframe( hBinRenderer->hReverb, BINAURAL_CHANNELS, inRe, inIm, reverbRe, reverbIm );

#endif

#else

        ivas_binaural_reverb_processFrame( hBinRenderer->hReverb, BINAURAL_CHANNELS, inRe, inIm, reverbRe, reverbIm, 0u );

#endif

lib_dec/ivas_masa_dec.c

+95 −34
Original line number Diff line number Diff line
@@ -1278,12 +1278,13 @@ ivas_error ivas_masa_dec_reconfigure( 
}





#ifdef JBM_TSM_ON_TCS

#ifndef FIX_355_REFACTOR_PARAMBIN_TO_5MS

/*-------------------------------------------------------------------*

 * ivas_spar_param_to_masa_param_mapping()

 *

 * Determine MASA metadata from the SPAR metadata

 *-------------------------------------------------------------------*/

#ifdef JBM_TSM_ON_TCS

void ivas_spar_param_to_masa_param_mapping_sf(

    Decoder_Struct *st_ivas, /* i/o: IVAS decoder struct               */

#ifdef MASA_PARAMBIN_SF_MAP
@@ -1437,9 +1438,9 @@ void ivas_spar_param_to_masa_param_mapping_sf( 


    if ( hDiffuseDist != NULL )

    {

        set_zero( hDiffuseDist->diffuseRatioX[0], CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioY[0], CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioZ[0], CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioX, CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioY, CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioZ, CLDFB_NO_CHANNELS_MAX );

    }



    for ( bin = 0; bin < nBins; bin++ )
@@ -1520,15 +1521,15 @@ void ivas_spar_param_to_masa_param_mapping_sf( 


            if ( diffuseGainSum == 0.0f )

            {

                hDiffuseDist->diffuseRatioX[0][bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioY[0][bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioZ[0][bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioX[bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioY[bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioZ[bin] = 1.0f / 3.0f;

            }

            else

            {

                hDiffuseDist->diffuseRatioX[0][bin] = diffuseGainX / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioY[0][bin] = diffuseGainY / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioZ[0][bin] = diffuseGainZ / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioX[bin] = diffuseGainX / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioY[bin] = diffuseGainY / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioZ[bin] = diffuseGainZ / ( diffuseGainSum + EPSILON );

            }

        }

    }
@@ -1547,6 +1548,7 @@ void ivas_spar_param_to_masa_param_mapping_sf( 
    return;

}

#endif

#endif



#ifdef FIX_355_REFACTOR_PARAMBIN_TO_5MS

void ivas_spar_param_to_masa_param_mapping(
@@ -1561,8 +1563,8 @@ void ivas_spar_param_to_masa_param_mapping( 
    int16_t dirac_write_idx;

    DIRAC_DEC_HANDLE hDirAC;

    DIFFUSE_DISTRIBUTION_HANDLE hDiffuseDist;

    float mixer_mat_sf_bands_real[MAX_PARAM_SPATIAL_SUBFRAMES][SPAR_DIRAC_SPLIT_START_BAND][FOA_CHANNELS][FOA_CHANNELS];

    float mixer_mat_sf_bins_real[MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX][FOA_CHANNELS][FOA_CHANNELS];

    float mixer_mat_sf_bands_real[SPAR_DIRAC_SPLIT_START_BAND][FOA_CHANNELS][FOA_CHANNELS];

    float mixer_mat_sf_bins_real[CLDFB_NO_CHANNELS_MAX][FOA_CHANNELS][FOA_CHANNELS];

    int16_t *band_grouping;

    int16_t band_start, band_end;

    float transportSignalEnergies[2][CLDFB_NO_CHANNELS_MAX];
@@ -1572,6 +1574,12 @@ void ivas_spar_param_to_masa_param_mapping( 
    float foaCovarianceMtx[FOA_CHANNELS][FOA_CHANNELS];

    float Iy, Iz, Ix, E, azi, ele, I, ratio;

    float diffuseGainX, diffuseGainY, diffuseGainZ, diffuseGainSum;

#ifdef JBM_TSM_ON_TCS

    int16_t slot_idx, slot_idx_start, sf;

    SPAR_DEC_HANDLE hSpar;

    float slot_fac;

#endif





    /* Set values */

    hDirAC = st_ivas->hDirAC;
@@ -1579,7 +1587,12 @@ void ivas_spar_param_to_masa_param_mapping( 
    hDiffuseDist = st_ivas->hDirAC->hDiffuseDist;

    nchan_transport = st_ivas->nchan_transport;

    band_grouping = hDirAC->band_grouping;

#ifdef JBM_TSM_ON_TCS

    hSpar = st_ivas->hSpar;

    dirac_write_idx = hDirAC->render_to_md_map[subframe];

#else

    dirac_write_idx = hDirAC->dirac_read_idx; /* Mixing matrices, from which MASA meta is determined, already have the delay compensation */

#endif



    /* Init arrays */

    for ( i = 0; i < FOA_CHANNELS; i++ )
@@ -1588,34 +1601,61 @@ void ivas_spar_param_to_masa_param_mapping( 
    }



    /* Delay the SPAR mixing matrices to have them synced with the audio */

#ifdef JBM_TSM_ON_TCS

    slot_idx_start = hSpar->slots_rendered;

    slot_fac = 1.0f / (float) hSpar->subframe_nbslots[subframe];



    for ( slot_idx = 0; slot_idx < hSpar->subframe_nbslots[subframe]; slot_idx++ )

    {

        sf = hSpar->render_to_md_map[slot_idx + slot_idx_start] / JBM_CLDFB_SLOTS_IN_SUBFRAME;

#endif

        if ( subframe < SPAR_META_DELAY_SUBFRAMES )

        {

#ifdef JBM_TSM_ON_TCS

            mixer_mat_index = sf + MAX_PARAM_SPATIAL_SUBFRAMES - SPAR_META_DELAY_SUBFRAMES + 1;

#else

        mixer_mat_index = subframe + MAX_PARAM_SPATIAL_SUBFRAMES - SPAR_META_DELAY_SUBFRAMES + 1;

#endif

            for ( band = 0; band < SPAR_DIRAC_SPLIT_START_BAND; band++ )

            {

                for ( i = 0; i < FOA_CHANNELS; i++ )

                {

                    for ( j = 0; j < FOA_CHANNELS; j++ )

                    {

                    mixer_mat_sf_bands_real[subframe][band][i][j] = st_ivas->hSpar->hMdDec->mixer_mat_prev[mixer_mat_index][i][j][band];

#ifdef JBM_TSM_ON_TCS

                        mixer_mat_sf_bands_real[band][i][j] = slot_fac * st_ivas->hSpar->hMdDec->mixer_mat_prev[mixer_mat_index][i][j][band];

#else

                    mixer_mat_sf_bands_real[band][i][j] = st_ivas->hSpar->hMdDec->mixer_mat_prev[mixer_mat_index][i][j][band];

#endif

                    }

                }

            }

        }

        else

        {

#ifdef JBM_TSM_ON_TCS

            mixer_mat_index = sf - SPAR_META_DELAY_SUBFRAMES;

#else

        mixer_mat_index = subframe - SPAR_META_DELAY_SUBFRAMES;

#endif

            for ( band = 0; band < SPAR_DIRAC_SPLIT_START_BAND; band++ )

            {

                for ( i = 0; i < FOA_CHANNELS; i++ )

                {

                    for ( j = 0; j < FOA_CHANNELS; j++ )

                    {

                    mixer_mat_sf_bands_real[subframe][band][i][j] = st_ivas->hSpar->hMdDec->mixer_mat[i][j][band + mixer_mat_index * IVAS_MAX_NUM_BANDS];

#ifdef JBM_TSM_ON_TCS

                        mixer_mat_sf_bands_real[band][i][j] = slot_fac * st_ivas->hSpar->hMdDec->mixer_mat[i][j][band + mixer_mat_index * IVAS_MAX_NUM_BANDS];

#else

                    mixer_mat_sf_bands_real[band][i][j] = st_ivas->hSpar->hMdDec->mixer_mat[i][j][band + mixer_mat_index * IVAS_MAX_NUM_BANDS];

#endif

                    }

                }

            }

        }

#ifdef JBM_TSM_ON_TCS

    }

#endif



    /* Map the mixing matrices from the frequency bands to frequency bins */

    bin = 0;
@@ -1629,7 +1669,7 @@ void ivas_spar_param_to_masa_param_mapping( 
            {

                for ( j = 0; j < FOA_CHANNELS; j++ )

                {

                    mixer_mat_sf_bins_real[subframe][bin][i][j] = mixer_mat_sf_bands_real[subframe][band][i][j];

                    mixer_mat_sf_bins_real[bin][i][j] = mixer_mat_sf_bands_real[band][i][j];

                }

            }

        }
@@ -1666,9 +1706,15 @@ void ivas_spar_param_to_masa_param_mapping( 


    if ( hDiffuseDist != NULL )

    {

#ifdef JBM_TSM_ON_TCS

        set_zero( hDiffuseDist->diffuseRatioX, CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioY, CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioZ, CLDFB_NO_CHANNELS_MAX );

#else

        set_zero( hDiffuseDist->diffuseRatioX[subframe], CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioY[subframe], CLDFB_NO_CHANNELS_MAX );

        set_zero( hDiffuseDist->diffuseRatioZ[subframe], CLDFB_NO_CHANNELS_MAX );

#endif

    }



    for ( bin = 0; bin < nBins; bin++ )
@@ -1695,7 +1741,7 @@ void ivas_spar_param_to_masa_param_mapping( 
            inCovarianceMtx[1][0] = inCovarianceMtx[0][1];

        }



        compute_foa_cov_matrix( foaCovarianceMtx, inCovarianceMtx, mixer_mat_sf_bins_real[subframe][bin] );

        compute_foa_cov_matrix( foaCovarianceMtx, inCovarianceMtx, mixer_mat_sf_bins_real[bin] );



        /* Estimate MASA metadata */

        Iy = foaCovarianceMtx[0][1];                                                                                      /* Intensity in Y direction */
@@ -1721,15 +1767,15 @@ void ivas_spar_param_to_masa_param_mapping( 
        {

            if ( nchan_transport == 1 )

            {

                diffuseGainY = fabsf( mixer_mat_sf_bins_real[subframe][bin][1][1] );

                diffuseGainX = fabsf( mixer_mat_sf_bins_real[subframe][bin][3][2] );

                diffuseGainZ = fabsf( mixer_mat_sf_bins_real[subframe][bin][2][3] );

                diffuseGainY = fabsf( mixer_mat_sf_bins_real[bin][1][1] );

                diffuseGainX = fabsf( mixer_mat_sf_bins_real[bin][3][2] );

                diffuseGainZ = fabsf( mixer_mat_sf_bins_real[bin][2][3] );

            }

            else if ( nchan_transport == 2 )

            {

                diffuseGainY = fabsf( mixer_mat_sf_bins_real[subframe][bin][1][1] * transportSignalEnergies[1][bin] );

                diffuseGainX = fabsf( mixer_mat_sf_bins_real[subframe][bin][3][2] * transportSignalEnergies[0][bin] ) + fabsf( mixer_mat_sf_bins_real[subframe][bin][3][1] * transportSignalEnergies[1][bin] );

                diffuseGainZ = fabsf( mixer_mat_sf_bins_real[subframe][bin][2][3] * transportSignalEnergies[0][bin] ) + fabsf( mixer_mat_sf_bins_real[subframe][bin][2][1] * transportSignalEnergies[1][bin] );

                diffuseGainY = fabsf( mixer_mat_sf_bins_real[bin][1][1] * transportSignalEnergies[1][bin] );

                diffuseGainX = fabsf( mixer_mat_sf_bins_real[bin][3][2] * transportSignalEnergies[0][bin] ) + fabsf( mixer_mat_sf_bins_real[bin][3][1] * transportSignalEnergies[1][bin] );

                diffuseGainZ = fabsf( mixer_mat_sf_bins_real[bin][2][3] * transportSignalEnergies[0][bin] ) + fabsf( mixer_mat_sf_bins_real[bin][2][1] * transportSignalEnergies[1][bin] );

            }

            else

            {
@@ -1740,6 +1786,20 @@ void ivas_spar_param_to_masa_param_mapping( 


            diffuseGainSum = diffuseGainY + diffuseGainX + diffuseGainZ;



#ifdef JBM_TSM_ON_TCS

            if ( diffuseGainSum == 0.0f )

            {

                hDiffuseDist->diffuseRatioX[bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioY[bin] = 1.0f / 3.0f;

                hDiffuseDist->diffuseRatioZ[bin] = 1.0f / 3.0f;

            }

            else

            {

                hDiffuseDist->diffuseRatioX[bin] = diffuseGainX / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioY[bin] = diffuseGainY / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioZ[bin] = diffuseGainZ / ( diffuseGainSum + EPSILON );

            }

#else

            if ( diffuseGainSum == 0.0f )

            {

                hDiffuseDist->diffuseRatioX[subframe][bin] = 1.0f / 3.0f;
@@ -1752,6 +1812,7 @@ void ivas_spar_param_to_masa_param_mapping( 
                hDiffuseDist->diffuseRatioY[subframe][bin] = diffuseGainY / ( diffuseGainSum + EPSILON );

                hDiffuseDist->diffuseRatioZ[subframe][bin] = diffuseGainZ / ( diffuseGainSum + EPSILON );

            }

#endif

        }

    }

lib_dec/ivas_stat_dec.h

+6 −0
Original line number Diff line number Diff line
@@ -619,9 +619,15 @@ typedef struct 
/* Diffuse sound directional distribution data structure */

typedef struct ivas_diffuse_distribution_data_structure

{

#ifdef JBM_TSM_ON_TCS

    float diffuseRatioX[CLDFB_NO_CHANNELS_MAX];

    float diffuseRatioY[CLDFB_NO_CHANNELS_MAX];

    float diffuseRatioZ[CLDFB_NO_CHANNELS_MAX];

#else

    float diffuseRatioX[MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];

    float diffuseRatioY[MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];

    float diffuseRatioZ[MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX];

#endif



} DIFFUSE_DISTRIBUTION_DATA, *DIFFUSE_DISTRIBUTION_HANDLE;