From 09a1f44f81dee688c757a857c454fd32c7b1b2e1 Mon Sep 17 00:00:00 2001 From: Lauros Pajunen Date: Tue, 28 Mar 2023 17:12:43 +0300 Subject: [PATCH 1/4] Detect potential MASA metadata framing asynchrony and reduce the quality degradation by modifying the metadata --- lib_com/ivas_cnst.h | 9 +- lib_com/ivas_prot.h | 56 ++--- lib_com/options.h | 1 + lib_dec/lib_dec.h | 2 +- lib_enc/ivas_enc.c | 9 +- lib_enc/ivas_masa_enc.c | 446 +++++++++++++++++++++++++++++++++++++++- lib_enc/ivas_stat_enc.h | 14 ++ 7 files changed, 500 insertions(+), 37 deletions(-) diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 3f48b0b43e..79f5c9032c 100644 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -357,7 +357,7 @@ typedef enum #define ISM_Q_STEP_LOW (ISM_Q_STEP * 2) #define ISM_Q_STEP_BORDER_LOW (ISM_Q_STEP_BORDER * 2) #else -#define PARAM_ISM_DTX_AZI_BITS 5 +#define PARAM_ISM_DTX_AZI_BITS 5 #define PARAM_ISM_DTX_ELE_BITS 4 #endif @@ -1214,6 +1214,13 @@ typedef enum MASA_STEREO_DOWNMIX } MASA_TRANSPORT_SIGNAL_TYPE; +#ifdef FIX_382_MASA_META_FRAMING_ASYNC +typedef enum +{ + MASA_FRAME_1SF, + MASA_FRAME_4SF +} MASA_FRAME_MODE; +#endif /*----------------------------------------------------------------------------------* * Multichannel format diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index e8597bbf3c..a7e95b4ebc 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -793,7 +793,7 @@ ivas_error ivas_set_ism_metadata( const float pitch /* i : pitch */ #else const float elevation /* i : elevation value */ -#endif +#endif ); ivas_error ivas_ism_metadata_enc_create( @@ -4025,40 +4025,40 @@ void ivas_spar_bitrate_dist( const int16_t bwidth /* i : audio bandwidth */ ); -void ivas_mdct( - const float *pIn, - float *pOut, - const int16_t length +void ivas_mdct( + const float *pIn, + float *pOut, + const int16_t length ); -void ivas_dct_windowing( - const int16_t fade_len, - const int16_t full_len, - const int16_t dct_len, - const int16_t zero_pad_len, - const float *pWindow_coeffs, - const int16_t frame_len, - float *pOut_buf, - float *pBuffer_prev, - float *pTemp_lfe +void ivas_dct_windowing( + const int16_t fade_len, + const int16_t full_len, + const int16_t dct_len, + const int16_t zero_pad_len, + const float *pWindow_coeffs, + const int16_t frame_len, + float *pOut_buf, + float *pBuffer_prev, + float *pTemp_lfe ); -void ivas_tda( - const float *pIn, - float *pOut, - const int16_t length +void ivas_tda( + const float *pIn, + float *pOut, + const int16_t length ); -void ivas_imdct( - const float *pIn, - float *pOut, - const int16_t length +void ivas_imdct( + const float *pIn, + float *pOut, + const int16_t length ); -void ivas_itda( - const float *re, - float *pOut, - const int16_t length +void ivas_itda( + const float *re, + float *pOut, + const int16_t length ); void ivas_spar_get_cldfb_gains( @@ -4282,7 +4282,7 @@ void ivas_transient_det_close( ivas_trans_det_state_t **hTranDet /* i/o: Transient detector handle */ ); -void ivas_transient_det_process( +void ivas_transient_det_process( ivas_trans_det_state_t *hTranDet, /* i/o: SPAR TD handle */ float *pIn_pcm, /* i : input audio channels */ const int16_t frame_len, /* i : frame length in samples */ diff --git a/lib_com/options.h b/lib_com/options.h index c054a9b5a7..45ae98fcc3 100755 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -163,6 +163,7 @@ #define DISCRETE_ISM_DTX_CNG /* FhG/VA: contribution 15 - DTX/CNG for (discrete) ISM */ #define NCHAN_ISM_PARAMETER /* VA: make 'nchan_ism' parameter part of st_ivas/hEncoderConfig */ +#define FIX_382_MASA_META_FRAMING_ASYNC /* Nokia: Issue 382: detect potential MASA metadata framing offset */ /* ################## End DEVELOPMENT switches ######################### */ /* clang-format on */ diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index a717e0002e..03f71e44ec 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -176,7 +176,7 @@ ivas_error IVAS_DEC_GetMasaMetadata( /*! r: error code */ ivas_error IVAS_DEC_FeedHeadTrackData( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ -#ifdef TD5 +#ifdef TD5 IVAS_QUATERNION *orientation, /* i : head-tracking data */ IVAS_POSITION *Pos /* i : listener position */ #else diff --git a/lib_enc/ivas_enc.c b/lib_enc/ivas_enc.c index 1e4ba82207..535a1196ab 100644 --- a/lib_enc/ivas_enc.c +++ b/lib_enc/ivas_enc.c @@ -227,13 +227,16 @@ ivas_error ivas_enc( } else { +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + ivas_masa_estimate_energy( st_ivas->hMasa, data_f, input_frame, st_ivas->nchan_transport ); /* energy-estimation uses TF-resolution: 4x24 */ +#endif if ( ( error = ivas_masa_enc_config( st_ivas ) ) != IVAS_ERR_OK ) { return error; } - - ivas_masa_estimate_energy( st_ivas->hMasa, data_f, input_frame, st_ivas->nchan_transport ); - +#ifndef FIX_382_MASA_META_FRAMING_ASYNC + ivas_masa_estimate_energy( st_ivas->hMasa, data_f, input_frame, st_ivas->nchan_transport ); /* energy-estimation uses TF-resolution: 4x24 */ +#endif if ( ( error = ivas_masa_encode( st_ivas->hMasa, st_ivas->hQMetaData, hMetaData, &nb_bits_metadata[0], st_ivas->nchan_transport, ivas_format, ivas_total_brate, hEncoderConfig->Opt_DTX_ON, st_ivas->nchan_transport == 2 ? st_ivas->hCPE[0]->element_mode : -1 ) ) != IVAS_ERR_OK ) { diff --git a/lib_enc/ivas_masa_enc.c b/lib_enc/ivas_masa_enc.c index 337753925c..ec8e3fe799 100644 --- a/lib_enc/ivas_masa_enc.c +++ b/lib_enc/ivas_masa_enc.c @@ -31,8 +31,8 @@ *******************************************************************************************************/ #include -#include "options.h" #include +#include "options.h" #include "ivas_cnst.h" #include "ivas_prot.h" #include "ivas_rom_com.h" @@ -40,6 +40,10 @@ #include "wmc_auto.h" #include "prot.h" +#ifdef FIX_382_MASA_META_FRAMING_ASYNC +#include +#endif + /*-----------------------------------------------------------------------* * Local function prototypes *-----------------------------------------------------------------------*/ @@ -60,6 +64,17 @@ static int16_t encode_lfe_to_total_energy_ratio( MASA_ENCODER_HANDLE hMasa, BSTR static void reduce_metadata_further( MASA_ENCODER_HANDLE hMasa, IVAS_QMETADATA_HANDLE hqmetadata, const IVAS_FORMAT ivas_format ); +#ifdef FIX_382_MASA_META_FRAMING_ASYNC +static void average_masa_metadata( MASA_ENCODER_HANDLE hMasa ); + +static void copy_masa_metadata_subframe( MASA_METADATA_HANDLE hMetaFrom, const uint8_t sfFrom, MASA_METADATA_HANDLE hMetaTo, const uint8_t sfTo ); + +static void copy_masa_metadata( MASA_METADATA_HANDLE hMetaFrom, MASA_METADATA_HANDLE hMetaTo ); + +static uint8_t are_masa_subframes_similar( MASA_METADATA_HANDLE frame1, const uint8_t sf1_idx, MASA_METADATA_HANDLE frame2, const uint8_t sf2_idx ); + +static void detect_framing_async( MASA_ENCODER_HANDLE hMasa ); +#endif /*-----------------------------------------------------------------------* * Local constants @@ -136,6 +151,13 @@ ivas_error ivas_masa_enc_open( set_zero( hMasa->data.lfeToTotalEnergyRatio, MAX_PARAM_SPATIAL_SUBFRAMES ); hMasa->data.prevq_lfeToTotalEnergyRatio = 0.0f; hMasa->data.prevq_lfeIndex = 0; + +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + hMasa->data.sync_state.prev_sim_stop = 0; + hMasa->data.sync_state.prev_offset = 0; + hMasa->data.sync_state.frame_mode = MASA_FRAME_4SF; +#endif + st_ivas->hMasa = hMasa; return error; @@ -506,6 +528,15 @@ ivas_error ivas_masa_enc_config( if ( ivas_format == MASA_FORMAT ) { +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + detect_framing_async( hMasa ); /* detect the offset, set 1sf/4sf mode based on this. potentially also shift the metadata using a history buffer */ + if ( hMasa->data.sync_state.frame_mode == MASA_FRAME_1SF && hMasa->data.sync_state.prev_offset != 0 ) + { + /* average over sub-frames */ + average_masa_metadata( hMasa ); + } +#endif + /* Inspect metadata for parameter changes that affect coding. */ detect_metadata_composition( hMasa, &joinedSubframes, &coherencePresent, &isActualTwoDir ); hMasa->config.joinedSubframes = joinedSubframes; @@ -514,7 +545,7 @@ ivas_error ivas_masa_enc_config( } else if ( ivas_format == MC_FORMAT && st_ivas->mc_mode == MC_MODE_MCMASA ) { - /* For mcMASA, these are set only once as this function is called only once. */ + /* For McMASA, these are set only once as this function is called only once. */ hMasa->config.joinedSubframes = 0; hMasa->config.numberOfDirections = 1; } @@ -709,9 +740,10 @@ static void combine_freqbands_and_subframes( if ( numCodingBands < MASA_FREQUENCY_BANDS ) { + /* reduce metadata *frequency* resolution. time resolution is not touched */ for ( i = 0; i < numDirections; i++ ) { - for ( j = 0; j < numSf; j++ ) + for ( j = 0; j < numSf; j++ ) /* NB: for numSf==1, operates only on first sub-frame */ { for ( k = 0; k < MASA_FREQUENCY_BANDS; k++ ) { @@ -785,7 +817,7 @@ static void combine_freqbands_and_subframes( } } } - else if ( mergeRatiosOverSubframes ) + else if ( mergeRatiosOverSubframes ) /* keep frequency resolution */ { for ( j = 0; j < numSf; j++ ) { @@ -1092,11 +1124,18 @@ static void detect_metadata_composition( ) { MASA_METADATA_FRAME *hMeta; +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + int8_t sf, band, dir, numDir; +#else int16_t sf, band, dir, numDir; +#endif int16_t nSubFrames; uint8_t dirValid[2] = { FALSE }; uint8_t cohPresent = FALSE; uint8_t sfDiffer = FALSE; +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + uint8_t sfSimilar; +#endif hMeta = &( hMasa->masaMetadata ); numDir = hMeta->descriptive_meta.numberOfDirections + 1; @@ -1161,6 +1200,16 @@ static void detect_metadata_composition( } /* Check if data over subframes is identical. Check is done by comparing to first subframe. */ +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + sfSimilar = TRUE; + sf = 1; + while ( ( sfSimilar == TRUE ) && ( sf < MAX_PARAM_SPATIAL_SUBFRAMES ) ) + { + sfSimilar = are_masa_subframes_similar( hMeta, 0, hMeta, sf ); + sf++; + } + sfDiffer = sfSimilar == TRUE ? FALSE : TRUE; +#else dir = 0; while ( sfDiffer == FALSE && dir < numDir ) { @@ -1207,6 +1256,7 @@ static void detect_metadata_composition( } dir++; } +#endif /* Further checks can be done with just one subframe if they are identical */ nSubFrames = sfDiffer == TRUE ? MAX_PARAM_SPATIAL_SUBFRAMES : 1; @@ -1750,3 +1800,391 @@ void ivas_masa_enc_reconfigure( return; } + +#ifdef FIX_382_MASA_META_FRAMING_ASYNC +/*-------------------------------------------------------------------* + * average_masa_metadata() + * + * Average MASA metadata frame subframe contents: applies aggregation over time + *-------------------------------------------------------------------*/ +static void average_masa_metadata( + MASA_ENCODER_HANDLE hMasa /* i/o: MASA encoder handle */ +) +{ + int16_t i, j, k; + float azi_rad, ele_rad; + uint8_t numDirections; + MASA_METADATA_HANDLE hMeta; + + hMeta = &( hMasa->masaMetadata ); + + /* use the nominal values without data-adaptivity */ + numDirections = hMeta->descriptive_meta.numberOfDirections + 1; + + /* azi/ele/nrg into vectors for each sub-frame and band */ + for ( i = 0; i < numDirections; i++ ) + { + for ( k = 0; k < MASA_FREQUENCY_BANDS; k++ ) + { + float x_sum, y_sum, z_sum, energy_sum, vec_len, spread_coh_sum, surr_coh_sum; + + x_sum = 0.0f; + y_sum = 0.0f; + z_sum = 0.0f; + energy_sum = 0.0f; + spread_coh_sum = 0.0f; + surr_coh_sum = 0.0f; + for ( j = 0; j < MAX_PARAM_SPATIAL_SUBFRAMES; j++ ) + { + azi_rad = hMeta->directional_meta[i].azimuth[j][k] / 180.0f * EVS_PI; + ele_rad = hMeta->directional_meta[i].elevation[j][k] / 180.0f * EVS_PI; + vec_len = hMeta->directional_meta[i].energy_ratio[j][k] * hMasa->data.energy[j][k]; + + /* energy-weighted sum over subframes */ + x_sum += cosf( azi_rad ) * cosf( ele_rad ) * vec_len; + y_sum += sinf( azi_rad ) * cosf( ele_rad ) * vec_len; + z_sum += sinf( ele_rad ) * vec_len; + + energy_sum += hMasa->data.energy[j][k]; + + spread_coh_sum += hMeta->directional_meta[i].spread_coherence[j][k] * hMasa->data.energy[j][k]; + if ( i == 0 ) + { + /* this is in common metadata and not in each direction */ + surr_coh_sum += hMeta->common_meta.surround_coherence[j][k] * hMasa->data.energy[j][k]; + } + } + + /* the data from the combined sub-frames is written into the first sub-frame band */ + j = 0; + hMeta->directional_meta[i].azimuth[j][k] = atan2f( y_sum, x_sum ) / EVS_PI * 180.0f; + hMeta->directional_meta[i].elevation[j][k] = atan2f( z_sum, sqrtf( x_sum * x_sum + y_sum * y_sum ) ) / EVS_PI * 180.0f; + + vec_len = sqrtf( x_sum * x_sum + y_sum * y_sum + z_sum * z_sum ); + hMeta->directional_meta[i].energy_ratio[j][k] = vec_len / ( energy_sum + EPSILON ); + + hMeta->directional_meta[i].spread_coherence[j][k] = spread_coh_sum / ( energy_sum + EPSILON ); + if ( i == 0 ) + { + hMeta->common_meta.surround_coherence[j][k] = surr_coh_sum / ( energy_sum + EPSILON ); + } + + /* copy the same value to all subframes */ + for ( j = 1; j < MAX_PARAM_SPATIAL_SUBFRAMES; j++ ) + { + hMeta->directional_meta[i].azimuth[j][k] = hMeta->directional_meta[i].azimuth[0][k]; + hMeta->directional_meta[i].elevation[j][k] = hMeta->directional_meta[i].elevation[0][k]; + hMeta->directional_meta[i].energy_ratio[j][k] = hMeta->directional_meta[i].energy_ratio[0][k]; + hMeta->directional_meta[i].spread_coherence[j][k] = hMeta->directional_meta[i].spread_coherence[0][k]; + if ( i == 0 ) + { + hMeta->common_meta.surround_coherence[j][k] = hMeta->common_meta.surround_coherence[0][k]; + } + } + } + } + + for ( k = 0; k < MASA_FREQUENCY_BANDS; k++ ) + { + for ( j = 0; j < MAX_PARAM_SPATIAL_SUBFRAMES; j++ ) + { + if ( numDirections == 2 ) + { + hMeta->common_meta.diffuse_to_total_ratio[j][k] = max( 0.0f, 1.0f - hMeta->directional_meta[1].energy_ratio[j][k] - hMeta->directional_meta[0].energy_ratio[j][k] ); + } + else + { + hMeta->common_meta.diffuse_to_total_ratio[j][k] = max( 0.0f, 1.0f - hMeta->directional_meta[0].energy_ratio[j][k] ); + } + hMeta->common_meta.remainder_to_total_ratio[j][k] = 0.0f; + } + } +} + +/*-------------------------------------------------------------------* + * copy_masa_metadata_subframe() + * + * Copy MASA metadata frame subframe contents + *-------------------------------------------------------------------*/ + +static void copy_masa_metadata_subframe( + MASA_METADATA_HANDLE hMetaFrom, /* i : MASA frame metdata to be copied */ + const uint8_t sfFrom, /* i : subframe index of the copy source */ + MASA_METADATA_HANDLE hMetaTo, /* o : MASA frame metadata copy destination */ + const uint8_t sfTo /* i : subframe index of the copy target */ +) +{ + uint8_t dir; + /* directional metadata */ + for ( dir = 0; dir < MASA_MAXIMUM_DIRECTIONS; dir++ ) + { + mvr2r( hMetaFrom->directional_meta[dir].azimuth[sfFrom], hMetaTo->directional_meta[dir].azimuth[sfTo], MASA_FREQUENCY_BANDS ); + mvr2r( hMetaFrom->directional_meta[dir].elevation[sfFrom], hMetaTo->directional_meta[dir].elevation[sfTo], MASA_FREQUENCY_BANDS ); + mvr2r( hMetaFrom->directional_meta[dir].energy_ratio[sfFrom], hMetaTo->directional_meta[dir].energy_ratio[sfTo], MASA_FREQUENCY_BANDS ); + mvr2r( hMetaFrom->directional_meta[dir].spread_coherence[sfFrom], hMetaTo->directional_meta[dir].spread_coherence[sfTo], MASA_FREQUENCY_BANDS ); + } + + /* common metadata */ + mvr2r( hMetaFrom->common_meta.diffuse_to_total_ratio[sfFrom], hMetaTo->common_meta.diffuse_to_total_ratio[sfTo], MASA_FREQUENCY_BANDS ); + mvr2r( hMetaFrom->common_meta.surround_coherence[sfFrom], hMetaTo->common_meta.surround_coherence[sfTo], MASA_FREQUENCY_BANDS ); + mvr2r( hMetaFrom->common_meta.remainder_to_total_ratio[sfFrom], hMetaTo->common_meta.remainder_to_total_ratio[sfTo], MASA_FREQUENCY_BANDS ); +} + +/*-------------------------------------------------------------------* + * copy_masa_metadata() + * + * Copy MASA metada frame contents + *-------------------------------------------------------------------*/ + +static void copy_masa_metadata( + MASA_METADATA_HANDLE hMetaFrom, /* i : MASA frame metadata to be copied */ + MASA_METADATA_HANDLE hMetaTo /* o : MASA frame metadata copy destination */ +) +{ + uint8_t sf; + + /* descriptive metadata */ + memcpy( hMetaTo->descriptive_meta.formatDescriptor, hMetaFrom->descriptive_meta.formatDescriptor, sizeof( hMetaFrom->descriptive_meta.formatDescriptor ) ); + + hMetaTo->descriptive_meta.numberOfDirections = hMetaFrom->descriptive_meta.numberOfDirections; + hMetaTo->descriptive_meta.numberOfChannels = hMetaFrom->descriptive_meta.numberOfChannels; + hMetaTo->descriptive_meta.sourceFormat = hMetaFrom->descriptive_meta.sourceFormat; + hMetaTo->descriptive_meta.transportDefinition = hMetaFrom->descriptive_meta.transportDefinition; + hMetaTo->descriptive_meta.channelAngle = hMetaFrom->descriptive_meta.channelAngle; + hMetaTo->descriptive_meta.channelDistance = hMetaFrom->descriptive_meta.channelDistance; + hMetaTo->descriptive_meta.channelLayout = hMetaFrom->descriptive_meta.channelLayout; + + /* directional and common metadata */ + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + copy_masa_metadata_subframe( hMetaFrom, sf, hMetaTo, sf ); + } +} + +/*-------------------------------------------------------------------* + * are_masa_subframes_similar() + * + * Compare the similarity of MASA metadata in two sub-frames + *-------------------------------------------------------------------*/ + +static uint8_t are_masa_subframes_similar( + MASA_METADATA_HANDLE frame1, /* i : MASA metadata frame 1 */ + const uint8_t sf1_idx, /* i : index of the subframe of frame1 to inspect */ + MASA_METADATA_HANDLE frame2, /* i : MASA metadata frame 2 */ + const uint8_t sf2_idx /* o : index of the subframe of frame2 to inspect */ +) +{ + uint8_t num_dir; + uint8_t dir; + uint8_t band_idx; + uint8_t sf_differ; + + num_dir = frame1->descriptive_meta.numberOfDirections; + dir = 0; + band_idx = 0; + sf_differ = FALSE; + + if ( num_dir != frame2->descriptive_meta.numberOfDirections ) + { + sf_differ = TRUE; + } + else + { + /* check per-direction metadata */ + dir = 0; + band_idx = 0; + + while ( ( sf_differ == FALSE ) && ( dir <= num_dir ) ) + { + band_idx = 0; + while ( ( sf_differ == FALSE ) && ( band_idx < MASA_FREQUENCY_BANDS ) ) + { + float azi_dif; + azi_dif = fabsf( frame1->directional_meta[dir].azimuth[sf1_idx][band_idx] - frame2->directional_meta[dir].azimuth[sf2_idx][band_idx] ); + azi_dif = azi_dif > 180.0f ? 360.0f - azi_dif : azi_dif; + + if ( azi_dif > MASA_ANGLE_TOLERANCE ) + { + sf_differ = TRUE; + break; + } + + if ( fabsf( frame1->directional_meta[dir].elevation[sf1_idx][band_idx] - frame2->directional_meta[dir].elevation[sf2_idx][band_idx] ) > MASA_ANGLE_TOLERANCE ) + { + sf_differ = TRUE; + break; + } + + if ( fabsf( frame1->directional_meta[dir].energy_ratio[sf1_idx][band_idx] - frame2->directional_meta[dir].energy_ratio[sf2_idx][band_idx] ) > MASA_RATIO_TOLERANCE ) + { + sf_differ = TRUE; + break; + } + + if ( fabsf( frame1->directional_meta[dir].spread_coherence[sf1_idx][band_idx] - frame2->directional_meta[dir].spread_coherence[sf2_idx][band_idx] ) > MASA_COHERENCE_TOLERANCE ) + { + sf_differ = TRUE; + break; + } + + band_idx++; + } + dir++; + } + + /* check the common metadata */ + while ( ( sf_differ == FALSE ) && ( band_idx < MASA_FREQUENCY_BANDS ) ) + { + if ( fabsf( frame1->common_meta.surround_coherence[sf1_idx][band_idx] - frame2->common_meta.surround_coherence[sf2_idx][band_idx] ) > MASA_COHERENCE_TOLERANCE ) + { + sf_differ = TRUE; + break; + } + + band_idx++; + } + } + + /* TODO: a nicer negation */ + if ( sf_differ ) + { + return FALSE; + } + else + { + return TRUE; + } +} + +/*-------------------------------------------------------------------* + * detect_framing_async() + * + * Compare the similarity of MASA metadata in two sub-frames + * Analysis result is stored in hMasa->data.sync_state, and + * potentially hMasa->masaMetadata is modified + *-------------------------------------------------------------------*/ + +static void detect_framing_async( + MASA_ENCODER_HANDLE hMasa /* i/o: MASA encoder structure */ +) +{ + MASA_METADATA_HANDLE current_meta; + MASA_METADATA_HANDLE previous_meta; + MASA_SYNC_HANDLE sync_state; + MASA_FRAME_MODE frame_mode; + uint8_t n_sim_start, n_sim_stop, sf_idx; + uint8_t found_offset; + + current_meta = &( hMasa->masaMetadata ); /* metadata from current frame */ + sync_state = &( hMasa->data.sync_state ); /* synchronization structure */ + previous_meta = &( sync_state->previous_metadata ); + + /* check current frame, how many are similar from the start and from the end */ + n_sim_start = 1; + for ( sf_idx = n_sim_start; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) + { + if ( are_masa_subframes_similar( current_meta, 0, current_meta, sf_idx ) == TRUE ) + { + n_sim_start = sf_idx + 1; + } + else + { + break; + } + } + + /* number of similar sub-frames starting from the end of the frame */ + if ( n_sim_start == MAX_PARAM_SPATIAL_SUBFRAMES ) /* shortcut */ + { + n_sim_stop = n_sim_start; + } + else + { + n_sim_stop = 1; + for ( sf_idx = 2; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) + { + /* we need to check only the two middle sub-frames, as all being the same would have taken the shortcut above */ + if ( are_masa_subframes_similar( current_meta, MAX_PARAM_SPATIAL_SUBFRAMES - 1, current_meta, MAX_PARAM_SPATIAL_SUBFRAMES - sf_idx ) == TRUE ) + { + n_sim_stop = sf_idx; + } + else + { + break; + } + } + } + + frame_mode = MASA_FRAME_4SF; /* default mode: 4sf */ + if ( sync_state->prev_offset > MAX_PARAM_SPATIAL_SUBFRAMES - 2 ) + { + /* earlier offset was large => reset the offset */ + found_offset = 0; + } + else + { + /* keep previous offset unless something else is found. alternatively, we could reset always */ + found_offset = sync_state->prev_offset; + } + + if ( ( n_sim_start == MAX_PARAM_SPATIAL_SUBFRAMES ) && ( n_sim_stop == MAX_PARAM_SPATIAL_SUBFRAMES ) ) + { + /* full frame consists of similar sub-frames */ + frame_mode = MASA_FRAME_1SF; + if ( ( sync_state->prev_sim_stop != 0 ) && ( are_masa_subframes_similar( current_meta, 0, previous_meta, MAX_PARAM_SPATIAL_SUBFRAMES - 1 ) == TRUE ) ) + { + /* > 4 sub-frames of similar data */ + if ( sync_state->prev_sim_stop < 3 ) + { + /* can nicely align the framing with the earlier data and a small offset */ + found_offset = sync_state->prev_sim_stop; + } + else + { + /* too many similar sub-frames to determine the offset accurately => keep earlier value */ + found_offset = sync_state->prev_offset; + } + } + else + { + /* earlier window was different => reset the offset */ + found_offset = 0; + } + } + else if ( n_sim_stop == 3 ) + { + /* first sub-frame different that the rest 3 + => make a risky guess that the future sf would be the same too and we're in an offset case */ + frame_mode = MASA_FRAME_1SF; + found_offset = 3; + } + else if ( ( sync_state->prev_sim_stop > 0 ) && ( are_masa_subframes_similar( current_meta, 0, previous_meta, MAX_PARAM_SPATIAL_SUBFRAMES - 1 ) == TRUE ) ) + { + /* seeing data similar to past */ + if ( ( n_sim_start > 1 ) && ( n_sim_start + sync_state->prev_sim_stop >= MAX_PARAM_SPATIAL_SUBFRAMES ) ) + { + /* with the past, would have at least one long frame similar subframes */ + frame_mode = MASA_FRAME_1SF; + + if ( sync_state->prev_offset == 0 ) + { + found_offset = min( 2, sync_state->prev_sim_stop ); + } + else + { + found_offset = sync_state->prev_offset; + } + } + } + + /* keep the original contents of the frame, but then perform interpolation later */ + /* just copy current frame to storage */ + copy_masa_metadata( current_meta, previous_meta ); + + sync_state->prev_sim_stop = n_sim_stop; + sync_state->prev_offset = found_offset; + sync_state->frame_mode = frame_mode; + +} +#endif + diff --git a/lib_enc/ivas_stat_enc.h b/lib_enc/ivas_stat_enc.h index a56f6423d3..c6dc6b35ca 100644 --- a/lib_enc/ivas_stat_enc.h +++ b/lib_enc/ivas_stat_enc.h @@ -740,6 +740,17 @@ typedef struct ivas_param_mc_enc_data_structure * MASA encoder structures *----------------------------------------------------------------------------------*/ +#ifdef FIX_382_MASA_META_FRAMING_ASYNC +/* structure storing MASA framing sync detection and compensation data */ +typedef struct ivas_masa_sync_struct +{ + MASA_METADATA_FRAME previous_metadata; + uint8_t prev_sim_stop; + uint8_t prev_offset; + MASA_FRAME_MODE frame_mode; +} MASA_SYNC_STATE, *MASA_SYNC_HANDLE; +#endif + typedef struct ivas_masa_encoder_data_struct { float energy[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; @@ -759,6 +770,9 @@ typedef struct ivas_masa_encoder_data_struct float onset_detector_1; float onset_detector_2; +#ifdef FIX_382_MASA_META_FRAMING_ASYNC + MASA_SYNC_STATE sync_state; +#endif } MASA_ENCODER_DATA; typedef struct ivas_masa_encoder_struct -- GitLab From 3e1b75f32fbb12501afd072933b7eba1cd49823b Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Wed, 29 Mar 2023 08:00:32 +0200 Subject: [PATCH 2/4] clang formatting fix --- lib_enc/ivas_masa_enc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib_enc/ivas_masa_enc.c b/lib_enc/ivas_masa_enc.c index ec8e3fe799..b094e5a761 100644 --- a/lib_enc/ivas_masa_enc.c +++ b/lib_enc/ivas_masa_enc.c @@ -2184,7 +2184,5 @@ static void detect_framing_async( sync_state->prev_sim_stop = n_sim_stop; sync_state->prev_offset = found_offset; sync_state->frame_mode = frame_mode; - } #endif - -- GitLab From 0cdb3f71c6ebebcf1a0dc408f796e687da55f1bb Mon Sep 17 00:00:00 2001 From: vaclav Date: Wed, 29 Mar 2023 14:05:58 +0200 Subject: [PATCH 3/4] add three "VE:" comments; formal improvements --- lib_enc/ivas_masa_enc.c | 79 ++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/lib_enc/ivas_masa_enc.c b/lib_enc/ivas_masa_enc.c index b094e5a761..83381b7df5 100644 --- a/lib_enc/ivas_masa_enc.c +++ b/lib_enc/ivas_masa_enc.c @@ -41,7 +41,7 @@ #include "prot.h" #ifdef FIX_382_MASA_META_FRAMING_ASYNC -#include +#include // VE: please remove it #endif /*-----------------------------------------------------------------------* @@ -54,9 +54,9 @@ static void combine_directions( MASA_ENCODER_HANDLE hMasa ); static void find_n_largest( const float *input, int16_t *largestIndices, const int16_t numElements, const int16_t numLargest ); -static void move_metadata_to_qmetadata( MASA_ENCODER_HANDLE hMasa, IVAS_QMETADATA_HANDLE hQMeta ); +static void move_metadata_to_qmetadata( const MASA_ENCODER_HANDLE hMasa, IVAS_QMETADATA_HANDLE hQMeta ); -static void detect_metadata_composition( MASA_ENCODER_HANDLE hMasa, uint8_t *joinedSubframes, uint8_t *coherencePresent, uint8_t *isTwoDir ); +static void detect_metadata_composition( const MASA_ENCODER_HANDLE hMasa, uint8_t *joinedSubframes, uint8_t *coherencePresent, uint8_t *isTwoDir ); static void compensate_energy_ratios( MASA_ENCODER_HANDLE hMasa ); @@ -65,13 +65,13 @@ static int16_t encode_lfe_to_total_energy_ratio( MASA_ENCODER_HANDLE hMasa, BSTR static void reduce_metadata_further( MASA_ENCODER_HANDLE hMasa, IVAS_QMETADATA_HANDLE hqmetadata, const IVAS_FORMAT ivas_format ); #ifdef FIX_382_MASA_META_FRAMING_ASYNC -static void average_masa_metadata( MASA_ENCODER_HANDLE hMasa ); +static void average_masa_metadata( MASA_METADATA_FRAME *masaMetadata, float energy[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS] ); -static void copy_masa_metadata_subframe( MASA_METADATA_HANDLE hMetaFrom, const uint8_t sfFrom, MASA_METADATA_HANDLE hMetaTo, const uint8_t sfTo ); +static void copy_masa_metadata_subframe( const MASA_METADATA_HANDLE hMetaFrom, const uint8_t sfFrom, MASA_METADATA_HANDLE hMetaTo, const uint8_t sfTo ); -static void copy_masa_metadata( MASA_METADATA_HANDLE hMetaFrom, MASA_METADATA_HANDLE hMetaTo ); +static void copy_masa_metadata( const MASA_METADATA_HANDLE hMetaFrom, MASA_METADATA_HANDLE hMetaTo ); -static uint8_t are_masa_subframes_similar( MASA_METADATA_HANDLE frame1, const uint8_t sf1_idx, MASA_METADATA_HANDLE frame2, const uint8_t sf2_idx ); +static uint8_t are_masa_subframes_similar( const MASA_METADATA_HANDLE frame1, const uint8_t sf1_idx, const MASA_METADATA_HANDLE frame2, const uint8_t sf2_idx ); static void detect_framing_async( MASA_ENCODER_HANDLE hMasa ); #endif @@ -533,7 +533,7 @@ ivas_error ivas_masa_enc_config( if ( hMasa->data.sync_state.frame_mode == MASA_FRAME_1SF && hMasa->data.sync_state.prev_offset != 0 ) { /* average over sub-frames */ - average_masa_metadata( hMasa ); + average_masa_metadata( &( hMasa->masaMetadata ), hMasa->data.energy ); } #endif @@ -1045,7 +1045,7 @@ static void find_n_largest( static void move_metadata_to_qmetadata( - MASA_ENCODER_HANDLE hMasa, + const MASA_ENCODER_HANDLE hMasa, IVAS_QMETADATA_HANDLE hQMeta ) { int16_t dir, sf, band; @@ -1117,10 +1117,10 @@ static void move_metadata_to_qmetadata( /* This function studies parametric MASA metadata to provide information for codec configuration */ static void detect_metadata_composition( - MASA_ENCODER_HANDLE hMasa, /* i : MASA encoder data */ - uint8_t *joinedSubframes, /* o : Result of subframe composition */ - uint8_t *coherencePresent, /* o : Result of coherence presence */ - uint8_t *isTwoDir /* o : Result of two direction check */ + const MASA_ENCODER_HANDLE hMasa, /* i : MASA encoder data */ + uint8_t *joinedSubframes, /* o : Result of subframe composition */ + uint8_t *coherencePresent, /* o : Result of coherence presence */ + uint8_t *isTwoDir /* o : Result of two direction check */ ) { MASA_METADATA_FRAME *hMeta; @@ -1801,22 +1801,21 @@ void ivas_masa_enc_reconfigure( return; } + #ifdef FIX_382_MASA_META_FRAMING_ASYNC /*-------------------------------------------------------------------* * average_masa_metadata() * * Average MASA metadata frame subframe contents: applies aggregation over time *-------------------------------------------------------------------*/ + static void average_masa_metadata( - MASA_ENCODER_HANDLE hMasa /* i/o: MASA encoder handle */ -) + MASA_METADATA_FRAME *hMeta, + float energy[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS] ) { int16_t i, j, k; float azi_rad, ele_rad; uint8_t numDirections; - MASA_METADATA_HANDLE hMeta; - - hMeta = &( hMasa->masaMetadata ); /* use the nominal values without data-adaptivity */ numDirections = hMeta->descriptive_meta.numberOfDirections + 1; @@ -1838,20 +1837,20 @@ static void average_masa_metadata( { azi_rad = hMeta->directional_meta[i].azimuth[j][k] / 180.0f * EVS_PI; ele_rad = hMeta->directional_meta[i].elevation[j][k] / 180.0f * EVS_PI; - vec_len = hMeta->directional_meta[i].energy_ratio[j][k] * hMasa->data.energy[j][k]; + vec_len = hMeta->directional_meta[i].energy_ratio[j][k] * energy[j][k]; /* energy-weighted sum over subframes */ x_sum += cosf( azi_rad ) * cosf( ele_rad ) * vec_len; y_sum += sinf( azi_rad ) * cosf( ele_rad ) * vec_len; z_sum += sinf( ele_rad ) * vec_len; - energy_sum += hMasa->data.energy[j][k]; + energy_sum += energy[j][k]; - spread_coh_sum += hMeta->directional_meta[i].spread_coherence[j][k] * hMasa->data.energy[j][k]; + spread_coh_sum += hMeta->directional_meta[i].spread_coherence[j][k] * energy[j][k]; if ( i == 0 ) { /* this is in common metadata and not in each direction */ - surr_coh_sum += hMeta->common_meta.surround_coherence[j][k] * hMasa->data.energy[j][k]; + surr_coh_sum += hMeta->common_meta.surround_coherence[j][k] * energy[j][k]; } } @@ -1899,8 +1898,11 @@ static void average_masa_metadata( hMeta->common_meta.remainder_to_total_ratio[j][k] = 0.0f; } } + + return; } + /*-------------------------------------------------------------------* * copy_masa_metadata_subframe() * @@ -1908,13 +1910,14 @@ static void average_masa_metadata( *-------------------------------------------------------------------*/ static void copy_masa_metadata_subframe( - MASA_METADATA_HANDLE hMetaFrom, /* i : MASA frame metdata to be copied */ - const uint8_t sfFrom, /* i : subframe index of the copy source */ - MASA_METADATA_HANDLE hMetaTo, /* o : MASA frame metadata copy destination */ - const uint8_t sfTo /* i : subframe index of the copy target */ + const MASA_METADATA_HANDLE hMetaFrom, /* i : MASA frame metdata to be copied */ + const uint8_t sfFrom, /* i : subframe index of the copy source */ + MASA_METADATA_HANDLE hMetaTo, /* o : MASA frame metadata copy destination */ + const uint8_t sfTo /* i : subframe index of the copy target */ ) { uint8_t dir; + /* directional metadata */ for ( dir = 0; dir < MASA_MAXIMUM_DIRECTIONS; dir++ ) { @@ -1928,8 +1931,11 @@ static void copy_masa_metadata_subframe( mvr2r( hMetaFrom->common_meta.diffuse_to_total_ratio[sfFrom], hMetaTo->common_meta.diffuse_to_total_ratio[sfTo], MASA_FREQUENCY_BANDS ); mvr2r( hMetaFrom->common_meta.surround_coherence[sfFrom], hMetaTo->common_meta.surround_coherence[sfTo], MASA_FREQUENCY_BANDS ); mvr2r( hMetaFrom->common_meta.remainder_to_total_ratio[sfFrom], hMetaTo->common_meta.remainder_to_total_ratio[sfTo], MASA_FREQUENCY_BANDS ); + + return; } + /*-------------------------------------------------------------------* * copy_masa_metadata() * @@ -1937,13 +1943,14 @@ static void copy_masa_metadata_subframe( *-------------------------------------------------------------------*/ static void copy_masa_metadata( - MASA_METADATA_HANDLE hMetaFrom, /* i : MASA frame metadata to be copied */ - MASA_METADATA_HANDLE hMetaTo /* o : MASA frame metadata copy destination */ + const MASA_METADATA_HANDLE hMetaFrom, /* i : MASA frame metadata to be copied */ + MASA_METADATA_HANDLE hMetaTo /* o : MASA frame metadata copy destination */ ) { uint8_t sf; /* descriptive metadata */ + // VE: please do not use memcpy() and replace by mvr2r() etc. memcpy( hMetaTo->descriptive_meta.formatDescriptor, hMetaFrom->descriptive_meta.formatDescriptor, sizeof( hMetaFrom->descriptive_meta.formatDescriptor ) ); hMetaTo->descriptive_meta.numberOfDirections = hMetaFrom->descriptive_meta.numberOfDirections; @@ -1959,8 +1966,11 @@ static void copy_masa_metadata( { copy_masa_metadata_subframe( hMetaFrom, sf, hMetaTo, sf ); } + + return; } + /*-------------------------------------------------------------------* * are_masa_subframes_similar() * @@ -1968,10 +1978,10 @@ static void copy_masa_metadata( *-------------------------------------------------------------------*/ static uint8_t are_masa_subframes_similar( - MASA_METADATA_HANDLE frame1, /* i : MASA metadata frame 1 */ - const uint8_t sf1_idx, /* i : index of the subframe of frame1 to inspect */ - MASA_METADATA_HANDLE frame2, /* i : MASA metadata frame 2 */ - const uint8_t sf2_idx /* o : index of the subframe of frame2 to inspect */ + const MASA_METADATA_HANDLE frame1, /* i : MASA metadata frame 1 */ + const uint8_t sf1_idx, /* i : index of the subframe of frame1 to inspect */ + const MASA_METADATA_HANDLE frame2, /* i : MASA metadata frame 2 */ + const uint8_t sf2_idx /* o : index of the subframe of frame2 to inspect */ ) { uint8_t num_dir; @@ -2045,7 +2055,7 @@ static uint8_t are_masa_subframes_similar( } } - /* TODO: a nicer negation */ + /* TODO: a nicer negation */ // VE: ?? if ( sf_differ ) { return FALSE; @@ -2056,6 +2066,7 @@ static uint8_t are_masa_subframes_similar( } } + /*-------------------------------------------------------------------* * detect_framing_async() * @@ -2184,5 +2195,7 @@ static void detect_framing_async( sync_state->prev_sim_stop = n_sim_stop; sync_state->prev_offset = found_offset; sync_state->frame_mode = frame_mode; + + return; } #endif -- GitLab From f8eebc9709f4df3108d916d14865f839d719d806 Mon Sep 17 00:00:00 2001 From: Jouni Paulus Date: Wed, 29 Mar 2023 14:34:02 +0200 Subject: [PATCH 4/4] remove memcpy() --- lib_enc/ivas_masa_enc.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib_enc/ivas_masa_enc.c b/lib_enc/ivas_masa_enc.c index 83381b7df5..dccd38100a 100644 --- a/lib_enc/ivas_masa_enc.c +++ b/lib_enc/ivas_masa_enc.c @@ -40,9 +40,6 @@ #include "wmc_auto.h" #include "prot.h" -#ifdef FIX_382_MASA_META_FRAMING_ASYNC -#include // VE: please remove it -#endif /*-----------------------------------------------------------------------* * Local function prototypes @@ -1947,11 +1944,13 @@ static void copy_masa_metadata( MASA_METADATA_HANDLE hMetaTo /* o : MASA frame metadata copy destination */ ) { - uint8_t sf; + uint8_t sf, byte_idx; /* descriptive metadata */ - // VE: please do not use memcpy() and replace by mvr2r() etc. - memcpy( hMetaTo->descriptive_meta.formatDescriptor, hMetaFrom->descriptive_meta.formatDescriptor, sizeof( hMetaFrom->descriptive_meta.formatDescriptor ) ); + for ( byte_idx = 0; byte_idx < 8; byte_idx++ ) + { + hMetaTo->descriptive_meta.formatDescriptor[byte_idx] = hMetaFrom->descriptive_meta.formatDescriptor[byte_idx]; + } hMetaTo->descriptive_meta.numberOfDirections = hMetaFrom->descriptive_meta.numberOfDirections; hMetaTo->descriptive_meta.numberOfChannels = hMetaFrom->descriptive_meta.numberOfChannels; -- GitLab