From 4343f3de35815cdb3ac86d8321cac0c91b8ee2e6 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Thu, 24 Jul 2025 16:17:03 +0300 Subject: [PATCH 1/2] Port MR 1365 OMASA EXT from IVAS PC to BASOP float reference --- apps/decoder.c | 52 ++- apps/renderer.c | 5 +- lib_com/ivas_cnst.h | 6 + lib_com/ivas_prot.h | 24 ++ lib_com/options.h | 1 + lib_dec/ivas_init_dec.c | 58 +++- lib_dec/ivas_jbm_dec.c | 65 +++- lib_dec/ivas_masa_dec.c | 76 ++++- lib_dec/ivas_omasa_dec.c | 611 ++++++++++++++++++++++++++++++++++- lib_dec/ivas_output_config.c | 16 + lib_dec/ivas_sba_dec.c | 4 + lib_dec/ivas_stat_dec.h | 17 + lib_dec/lib_dec.c | 56 +++- lib_enc/ivas_masa_enc.c | 35 +- lib_rend/ivas_output_init.c | 6 + lib_util/masa_file_writer.c | 119 ++++++- lib_util/masa_file_writer.h | 7 +- 17 files changed, 1129 insertions(+), 29 deletions(-) diff --git a/apps/decoder.c b/apps/decoder.c index 1df9c9353..d817efe64 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -1975,7 +1975,12 @@ static ivas_error initOnFirstGoodFrame( for ( int16_t j = 0; j < numInitialBadFrames; ++j ) { +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + float delayMs = (float) ( pFullDelayNumSamples[0] ) / (float) ( *delayTimeScale ); + if ( ( error = MasaFileWriter_writeFrame( *ppMasaWriter, hMasaExtOutMeta, &delayMs ) ) != IVAS_ERR_OK ) +#else if ( ( error = MasaFileWriter_writeFrame( *ppMasaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( *ppMasaWriter ) ); return error; @@ -2442,13 +2447,27 @@ static ivas_error decodeG192( if ( bsFormat == IVAS_DEC_BS_MASA || bsFormat == IVAS_DEC_BS_MASA_ISM ) { IVAS_MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + int16_t fullDelayNumSamples[3]; + float delayMs; + + /* delayNumSamples is zeroed, and delayNumSamples_orig is updated only on first good frame, so need to re-fetch delay info */ + if ( ( error = IVAS_DEC_GetDelay( hIvasDec, fullDelayNumSamples, &delayTimeScale ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nUnable to get delay of decoder: %s\n", ivas_error_to_string( error ) ); + } +#endif if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + delayMs = (float) ( fullDelayNumSamples[0] ) / (float) ( delayTimeScale ); + if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta, &delayMs ) ) != IVAS_ERR_OK ) +#else if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); goto cleanup; @@ -2575,13 +2594,26 @@ static ivas_error decodeG192( if ( bsFormat == IVAS_DEC_BS_MASA || bsFormat == IVAS_DEC_BS_MASA_ISM ) { IVAS_MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + int16_t fullDelayNumSamples[3]; + float delayMs; + + if ( ( error = IVAS_DEC_GetDelay( hIvasDec, fullDelayNumSamples, &delayTimeScale ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nUnable to get delay of decoder: %s\n", ivas_error_to_string( error ) ); + } +#endif if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + delayMs = (float) ( fullDelayNumSamples[0] ) / (float) ( delayTimeScale ); + if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta, &delayMs ) ) != IVAS_ERR_OK ) +#else if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); goto cleanup; @@ -3229,13 +3261,27 @@ static ivas_error decodeVoIP( if ( bsFormat == IVAS_DEC_BS_MASA || bsFormat == IVAS_DEC_BS_MASA_ISM ) { IVAS_MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + int16_t fullDelayNumSamples[3]; + float delayMs; + + /* delayNumSamples_orig is fetched only for the first good frame, but here the delay can change between frames, so need to re-fetch */ + if ( ( error = IVAS_DEC_GetDelay( hIvasDec, fullDelayNumSamples, &delayTimeScale ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nUnable to get delay of decoder: %s\n", ivas_error_to_string( error ) ); + } +#endif if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 1 ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + delayMs = (float) ( fullDelayNumSamples[0] ) / (float) ( delayTimeScale ); + if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta, &delayMs ) ) != IVAS_ERR_OK ) +#else if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); goto cleanup; diff --git a/apps/renderer.c b/apps/renderer.c index 26fc9a7f0..ac79538a1 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1844,8 +1844,11 @@ int main( } } } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMetaOutput, NULL ) ) != IVAS_ERR_OK ) /* NULL -> use default metadata delay settings */ +#else if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMetaOutput ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); } diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 07d70b083..602a042e5 100644 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -118,7 +118,13 @@ typedef enum RENDERER_NON_DIEGETIC_DOWNMIX, RENDERER_OSBA_STEREO, RENDERER_OSBA_AMBI, +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + RENDERER_OSBA_LS, + RENDERER_OMASA_OBJECT_EXT, + RENDERER_OMASA_MIX_EXT +#else RENDERER_OSBA_LS +#endif } RENDERER_TYPE; diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 8ec991e1b..27f15cb1e 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -5733,6 +5733,30 @@ void ivas_omasa_rearrange_channels( const int16_t output_frame /* i : output frame length per channel */ ); +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT +ivas_error ivas_omasa_combine_separate_ism_with_masa_open( + Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +); + +void ivas_omasa_combine_separate_ism_with_masa( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + float *output[], /* o : output synthesis signal */ + const int16_t nchan_ism, /* i : number of ISMs */ + const int16_t output_frame /* i : output frame length per channel */ +); + +ivas_error ivas_omasa_render_objects_from_mix_open( + Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +); + +void ivas_omasa_render_objects_from_mix( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + float *output[], /* o : output synthesis signal */ + const int16_t nchan_ism, /* i : number of ISMs */ + const int16_t output_frame /* i : output frame length per channel */ +); + +#endif void ivas_omasa_dirac_rend_jbm( Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ const uint16_t nSamplesAsked, /* i : number of samples requested */ diff --git a/lib_com/options.h b/lib_com/options.h index 8e261a70e..1e105c7fd 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -186,6 +186,7 @@ #define NONBE_FIX_991_PARAMBIN_BINARY_HRTF /* Nokia: issue #991: fix using of binary file HRTF in ParamBin (to actiate when USE_NEW_HRTF_BINARY_FILE_FORMAT and FIX_777_COMBI_RENDER_CONFIG_FILE are on )*/ +#define NONBE_FIX_984_OMASA_EXT_OUTPUT /* Nok: issue 1497 - porting OMASA EXT MR */ /* #################### End BASOP porting switches ############################ */ /* clang-format on */ diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 7ee397f5a..88b8d3d6f 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -343,7 +343,22 @@ ivas_error ivas_dec_setup( { st_ivas->nchan_transport = 1; } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + /* this should be non-zero if original input format was MASA_ISM_FORMAT */ + st_ivas->nchan_ism = st_ivas->bit_stream[( ivas_total_brate / FRAMES_PER_SEC ) - 3] + 2 * st_ivas->bit_stream[( ivas_total_brate / FRAMES_PER_SEC ) - 2]; + if ( st_ivas->nchan_ism > 0 ) + { + /* the input_ivas_format should be MASA_ISM_FORMAT, but we cannot initialize it now */ + if ( st_ivas->nchan_transport == 2 && st_ivas->nchan_ism == 3 ) + { + st_ivas->nchan_ism = 4; + } + /* for MASA_ISM_FORMAT at input the number of MASA transport channels is always 2 */ + st_ivas->nchan_transport = 2; + element_mode_flag = 1; + } +#endif if ( st_ivas->ini_frame > 0 ) { /* reconfigure in case a change of operation mode is detected */ @@ -1062,7 +1077,11 @@ ivas_error ivas_init_decoder( hDecoderConfig->nchan_out = audioCfg2channels( IVAS_AUDIO_CONFIG_HOA3 ); hDecoderConfig->nchan_out += st_ivas->nchan_ism; } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT || st_ivas->ivas_format == MASA_FORMAT ) +#else else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) +#endif { hDecoderConfig->nchan_out = st_ivas->nchan_transport + st_ivas->nchan_ism; } @@ -2004,6 +2023,31 @@ ivas_error ivas_init_decoder( return error; } } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) + { + /* Allocate 'hIsmRendererData' handle and memory for delay buffer within 'hMasaIsmData' */ + if ( ( error = ivas_omasa_render_objects_from_mix_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( ( error = ivas_spat_hSpatParamRendCom_config( &st_ivas->hSpatParamRendCom, DIRAC_OPEN, 0, + st_ivas->ivas_format, st_ivas->mc_mode, output_Fs, 0, 0 ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + if ( st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT ) + { + /* Allocate 'hIsmRendererData' handle */ + if ( ( error = ivas_omasa_combine_separate_ism_with_masa_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif } if ( ( st_ivas->ivas_format == ISM_FORMAT || st_ivas->ivas_format == SBA_ISM_FORMAT ) && @@ -2845,6 +2889,16 @@ void ivas_init_dec_get_num_cldfb_instances( *numCldfbSyntheses = 0; } break; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + case RENDERER_OMASA_OBJECT_EXT: + *numCldfbAnalyses = st_ivas->nchan_transport; + *numCldfbSyntheses = st_ivas->hDecoderConfig->nchan_out; + break; + case RENDERER_OMASA_MIX_EXT: + *numCldfbAnalyses = st_ivas->nchan_transport + 1; + *numCldfbSyntheses = 0; + break; +#endif default: assert( 0 && "Renderer not handled for CLDFB reservation." ); } @@ -2977,7 +3031,7 @@ static ivas_error doSanityChecks_IVAS( return IVAS_ERROR( IVAS_ERR_ACOUSTIC_ENVIRONMENT_NOT_SUPPORTED, "Wrong set-up: Acoustic environment is not supported in this output configuration." ); } } - +#ifndef NONBE_FIX_984_OMASA_EXT_OUTPUT if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) { if ( st_ivas->ism_mode != ISM_MASA_MODE_DISC && output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) @@ -2985,7 +3039,7 @@ static ivas_error doSanityChecks_IVAS( return IVAS_ERROR( IVAS_ERR_INVALID_OUTPUT_FORMAT, "Incorrect output configuration specified for combined MASA and ISM format" ); } } - +#endif #ifdef DEBUGGING if ( ( st_ivas->hDecoderConfig->force_rend == FORCE_TD_RENDERER ) && ( ( st_ivas->ivas_format != MC_FORMAT && st_ivas->ivas_format != ISM_FORMAT ) || ( output_config != IVAS_AUDIO_CONFIG_BINAURAL && output_config != IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) || ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->ism_mode == ISM_MODE_PARAM ) || ( st_ivas->ivas_format == MC_FORMAT && st_ivas->mc_mode != MC_MODE_MCT ) ) ) { diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 094a861d1..3371cc929 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -311,6 +311,18 @@ ivas_error ivas_jbm_dec_tc( if ( st_ivas->ivas_format == MASA_FORMAT ) { ivas_masa_prerender( st_ivas, p_output, output_frame, nchan_remapped ); +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + /* external output */ + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL && st_ivas->hMasa->config.input_ivas_format == MASA_ISM_FORMAT ) + { + for ( n = 0; n < st_ivas->nchan_ism; n++ ) + { + set_zero( p_output[st_ivas->nchan_transport + n], output_frame ); + } + + ivas_omasa_rearrange_channels( p_output, st_ivas->nchan_ism, output_frame ); + } +#endif } else if ( st_ivas->ivas_format == SBA_FORMAT && ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) ) { @@ -387,6 +399,22 @@ ivas_error ivas_jbm_dec_tc( } else if ( output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) { +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + ivas_omasa_rearrange_channels( p_output, nchan_transport_ism, output_frame ); + } + else if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ ) + { + /* Convert separate object to MASA, combine with the original MASA, and output combined MASA + empty objects. */ + ivas_omasa_combine_separate_ism_with_masa( st_ivas, p_output, st_ivas->nchan_ism, output_frame ); + } + else if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + /* Extract objects from MASA, output MASA + all objects (i.e., extracted and separated objects) */ + ivas_omasa_render_objects_from_mix( st_ivas, p_output, st_ivas->nchan_ism, output_frame ); + } +#else /* sanity check in case of bitrate switching */ if ( st_ivas->ism_mode != ISM_MASA_MODE_DISC ) { @@ -394,7 +422,7 @@ ivas_error ivas_jbm_dec_tc( } ivas_omasa_rearrange_channels( p_output, nchan_transport_ism, output_frame ); - +#endif ivas_jbm_dec_copy_masa_meta_to_buffer( st_ivas ); } } @@ -870,17 +898,29 @@ void ivas_jbm_dec_feed_tc_to_renderer( } else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) { - if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT || st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) { - n_render_timeslots *= ( st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size ); + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + ivas_jbm_masa_sf_to_slot_map( st_ivas, n_render_timeslots ); } + else + { +#endif + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + n_render_timeslots *= ( st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size ); + } - ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); - if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) - { - ivas_ism_dec_digest_tc( st_ivas ); + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + ivas_ism_dec_digest_tc( st_ivas ); + } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT } +#endif if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_DIRAC ) { @@ -1148,6 +1188,13 @@ ivas_error ivas_jbm_dec_render( { ivas_omasa_dirac_rend_jbm( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output ); } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT || st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT ) + { + ivas_jbm_dec_tc_buffer_playout( st_ivas, nSamplesAskedLocal, nSamplesRendered, p_output ); + ivas_omasa_rearrange_channels( p_output, st_ivas->nchan_ism, *nSamplesRendered ); + } +#endif } else if ( st_ivas->ivas_format == SBA_ISM_FORMAT ) { @@ -2591,6 +2638,10 @@ TC_BUFFER_MODE ivas_jbm_dec_get_tc_buffer_mode( case RENDERER_PARAM_ISM: case RENDERER_BINAURAL_MIXER_CONV: case RENDERER_BINAURAL_MIXER_CONV_ROOM: +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + case RENDERER_OMASA_OBJECT_EXT: + case RENDERER_OMASA_MIX_EXT: +#endif case RENDERER_OSBA_AMBI: case RENDERER_OSBA_LS: buffer_mode = TC_BUFFER_MODE_RENDERER; diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index c93161d2b..722edf53e 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -249,6 +249,27 @@ ivas_error ivas_masa_decode( } } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + + /* read 2 bits: + '00' - MASA format at the encoder + '01' - MASA_ISM_FORMAT at the encoder, with 1 object + '10' - MASA_ISM_FORMAT at the encoder, with 2 objects + '11' - MASA_ISM_FORMAT at the encoder, with 3 or 4 objects + reading if 3 or 4 object is performed later + */ + byteBuffer = st->bit_stream[( st->next_bit_pos )--]; + byteBuffer = byteBuffer + 2 * st->bit_stream[( st->next_bit_pos )--]; + + if ( byteBuffer == 0 && ivas_format == MASA_FORMAT ) + { + hMasa->config.input_ivas_format = MASA_FORMAT; + } + else + { + hMasa->config.input_ivas_format = MASA_ISM_FORMAT; + } +#else /* read the MASA_ISM_FORMAT bit */ byteBuffer = st->bit_stream[( st->next_bit_pos )--]; if ( byteBuffer == 1 ) @@ -261,6 +282,7 @@ ivas_error ivas_masa_decode( } /* reserved bit */ byteBuffer = st->bit_stream[( st->next_bit_pos )--]; +#endif *nb_bits_read += MASA_HEADER_BITS; /* read number of directions */ @@ -342,6 +364,13 @@ ivas_error ivas_masa_decode( } } } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) + { + *nb_bits_read += ivas_decode_masaism_metadata( hQMetaData, st_ivas->hMasa, st_ivas->hMasaIsmData, st_ivas->nchan_ism, st->bit_stream, &st->next_bit_pos, + st_ivas->hMasaIsmData->idx_separated_ism, ism_imp, st_ivas->hSpatParamRendCom->dirac_bs_md_write_idx, st_ivas->hSpatParamRendCom->dirac_md_buffer_length ); + } +#endif else { *nb_bits_read += ivas_decode_masaism_metadata( hQMetaData, st_ivas->hMasa, st_ivas->hMasaIsmData, st_ivas->nchan_ism, st->bit_stream, &st->next_bit_pos, @@ -461,7 +490,12 @@ ivas_error ivas_masa_decode( dirac_bs_md_write_idx = st_ivas->hSpatParamRendCom->dirac_bs_md_write_idx; /* Store the write-index for this frame */ ivas_qmetadata_to_dirac( hQMetaData, st_ivas->hDirAC, hMasa, st_ivas->hSpatParamRendCom, ivas_total_brate, ivas_format, 0, 0 ); } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) + { + st_ivas->hSpatParamRendCom->dirac_bs_md_write_idx = ( st_ivas->hSpatParamRendCom->dirac_bs_md_write_idx + MAX_PARAM_SPATIAL_SUBFRAMES ) % st_ivas->hSpatParamRendCom->dirac_md_buffer_length; + } +#endif if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) { if ( hQMetaData->q_direction == NULL ) @@ -580,6 +614,14 @@ ivas_error ivas_masa_dec_open( } ism_total_brate = 0; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + /* ISM metadata */ + if ( st_ivas->ivas_format == MASA_FORMAT && st_ivas->hIsmMetaData[0] != NULL && st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) + { + /* these are not needed -> clean. EXT metafile writer in OMASA needs only the number of ISMs and writes default null-data */ + ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); + } +#endif if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->nSCE > 0 && ( st_ivas->ism_mode == ISM_MASA_MODE_DISC || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ ) ) { for ( i = 0; i < st_ivas->nSCE; i++ ) @@ -650,6 +692,13 @@ ivas_error ivas_masa_dec_open( nchan_transport = 1; nchan_to_allocate = 1; } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT && ( st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT || st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) ) + { + nchan_transport = st_ivas->nchan_transport + st_ivas->nchan_ism; + nchan_to_allocate = st_ivas->nchan_transport + st_ivas->nchan_ism; + } +#endif else if ( st_ivas->nchan_transport == 1 && ( st_ivas->renderer_type == RENDERER_DIRAC ) ) { /* addtl channel for CNG */ @@ -1404,7 +1453,17 @@ ivas_error ivas_masa_dec_reconfigure( if ( st_ivas->ivas_format == MASA_FORMAT ) { + +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_EXTERNAL ) + { + st_ivas->nchan_ism = 0; /* Initialization if it has not been already read from the end of the bitstream at the same time + with reading of the format: nchan_ism is needed in MASA format because for the EXT output in + MASA-only (pre-rendering mode of OMASA) the number of ISMs to output correct number of empty objects is needed */ + } +#else st_ivas->nchan_ism = 0; +#endif st_ivas->ism_mode = ISM_MODE_NONE; } @@ -1456,7 +1515,13 @@ ivas_error ivas_masa_dec_reconfigure( /* addtl channel for CNG */ tc_nchan_to_allocate++; } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT && ( st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT || st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) ) + { + tc_nchan_transport = st_ivas->nchan_transport + st_ivas->nchan_ism; + tc_nchan_to_allocate = st_ivas->nchan_transport + st_ivas->nchan_ism; + } +#endif if ( tc_nchan_transport != st_ivas->hTcBuffer->nchan_transport_jbm || tc_nchan_to_allocate != st_ivas->hTcBuffer->nchan_transport_internal || buffer_mode_new != st_ivas->hTcBuffer->tc_buffer_mode ) { @@ -2428,6 +2493,13 @@ static int16_t ivas_decode_masaism_metadata( { hMasaIsmData->energy_ratio_ism[dir][meta_write_index][b] = energy_ratio_ism[i][band][dir]; } + +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( hMasaIsmData->hExtData != NULL ) + { + hMasaIsmData->hExtData->masa_render_masa_to_total[meta_write_index][b] = hMasaIsmData->masa_to_total_energy_ratio[i][band]; + } +#endif } } } diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index 766635ac4..38bd17bc1 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -32,6 +32,9 @@ #include "options.h" #include +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT +#include +#endif #include "ivas_cnst.h" #include "ivas_prot.h" #include "prot.h" @@ -47,6 +50,9 @@ * Local constants *------------------------------------------------------------------------*/ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT +#define EXT_RENDER_IIR_FAC 0.95f +#endif /*-------------------------------------------------------------------* * ivas_omasa_data_open() @@ -101,7 +107,40 @@ ivas_error ivas_omasa_data_open( } set_s( hMasaIsmData->azimuth_separated_ism, 0, MAX_PARAM_SPATIAL_SUBFRAMES + DELAY_MASA_PARAM_DEC_SFR ); set_s( hMasaIsmData->elevation_separated_ism, 0, MAX_PARAM_SPATIAL_SUBFRAMES + DELAY_MASA_PARAM_DEC_SFR ); +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + hMasaIsmData->hExtData = NULL; + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) + { + MASA_ISM_EXT_DATA_HANDLE hExtData; + + if ( ( hExtData = (MASA_ISM_EXT_DATA_HANDLE) malloc( sizeof( MASA_ISM_EXT_DATA ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for MASA ISM data\n" ) ); + } + + hExtData->prev_idx_separated_ism = 0; + for ( ch = 0; ch < MAX_NUM_OBJECTS; ch++ ) + { + set_zero( hExtData->prev_panning_gains[ch], 2 ); + } + + for ( ch = 0; ch < MAX_NUM_OBJECTS; ch++ ) + { + set_zero( hExtData->ism_render_proto_energy[ch], CLDFB_NO_CHANNELS_MAX ); + set_zero( hExtData->ism_render_target_energy[ch], CLDFB_NO_CHANNELS_MAX ); + } + set_zero( hExtData->masa_render_proto_energy, CLDFB_NO_CHANNELS_MAX ); + set_zero( hExtData->masa_render_target_energy, CLDFB_NO_CHANNELS_MAX ); + + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES + DELAY_MASA_PARAM_DEC_SFR; sf++ ) + { + set_zero( hExtData->masa_render_masa_to_total[sf], CLDFB_NO_CHANNELS_MAX ); + } + + hMasaIsmData->hExtData = hExtData; + } +#endif st_ivas->hMasaIsmData = hMasaIsmData; return IVAS_ERR_OK; @@ -134,7 +173,13 @@ void ivas_omasa_data_close( free( ( *hMasaIsmData )->delayBuffer ); ( *hMasaIsmData )->delayBuffer = NULL; } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( ( *hMasaIsmData )->hExtData != NULL ) + { + free( ( *hMasaIsmData )->hExtData ); + ( *hMasaIsmData )->hExtData = NULL; + } +#endif free( *hMasaIsmData ); *hMasaIsmData = NULL; @@ -248,7 +293,48 @@ ivas_error ivas_omasa_dec_config( { /* ISM MD reconfig. */ n_MD = 0; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) + { + /* the full number of hIsmMetaData are needed for EXT output */ + n_MD = st_ivas->nchan_ism; + ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + + ivas_ism_metadata_close( st_ivas->hIsmMetaData, n_MD ); + } + else + { + if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + n_MD = 1; + if ( st_ivas->hIsmMetaData[0] == NULL ) + { + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, 1, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + } + } + else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + n_MD = st_ivas->nchan_ism; + + ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); + + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + ivas_ism_metadata_close( st_ivas->hIsmMetaData, n_MD ); + } +#else if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) { n_MD = 1; @@ -274,7 +360,7 @@ ivas_error ivas_omasa_dec_config( } ivas_ism_metadata_close( st_ivas->hIsmMetaData, n_MD ); - +#endif st_ivas->hCPE[0]->element_brate = ivas_total_brate - ism_total_brate; /*-----------------------------------------------------------------* @@ -303,8 +389,14 @@ ivas_error ivas_omasa_dec_config( } /* objects renderer reconfig. */ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->hMasaIsmData != NULL || st_ivas->hIsmRendererData != NULL ) + { + /* this calls also ivas_ism_renderer_close() closing st_ivas->hIsmRendererData used by the EXT renderers. also cleans st_ivas->hMasaIsmData */ +#else if ( st_ivas->hMasaIsmData != NULL ) { +#endif ivas_omasa_separate_object_renderer_close( st_ivas ); } @@ -363,7 +455,33 @@ ivas_error ivas_omasa_dec_config( ivas_omasa_separate_object_renderer_close( st_ivas ); } } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT ) + { + /* Allocate 'hIsmRendererData' handle */ + if ( ( error = ivas_omasa_combine_separate_ism_with_masa_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + if ( st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) + { + DIRAC_CONFIG_FLAG common_rend_config_flag = st_ivas->hSpatParamRendCom == NULL ? DIRAC_OPEN : DIRAC_RECONFIGURE; + /* Allocate 'hIsmRendererData' handle and memory for delay buffer within 'hMasaIsmData' */ + if ( ( error = ivas_omasa_render_objects_from_mix_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( ( error = ivas_spat_hSpatParamRendCom_config( &st_ivas->hSpatParamRendCom, common_rend_config_flag, 0, + st_ivas->ivas_format, st_ivas->mc_mode, st_ivas->hDecoderConfig->output_Fs, 0, 0 ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif /*-----------------------------------------------------------------* * TD Decorrelator *-----------------------------------------------------------------*/ @@ -557,6 +675,24 @@ ivas_error ivas_omasa_ism_metadata_dec( } } } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + else if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ && st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) + { + azimuth_ism = (int16_t) ( st_ivas->hIsmMetaData[0]->azimuth + 0.5f ); + elevation_ism = (int16_t) ( st_ivas->hIsmMetaData[0]->elevation + 0.5f ); + + for ( block = 0; block < 2; block++ ) + { + st_ivas->hMasaIsmData->azimuth_separated_ism[block] = st_ivas->hMasaIsmData->azimuth_separated_ism[block + 2]; + st_ivas->hMasaIsmData->elevation_separated_ism[block] = st_ivas->hMasaIsmData->elevation_separated_ism[block + 2]; + } + for ( block = 2; block < MAX_PARAM_SPATIAL_SUBFRAMES; block++ ) + { + st_ivas->hMasaIsmData->azimuth_separated_ism[block] = azimuth_ism; + st_ivas->hMasaIsmData->elevation_separated_ism[block] = elevation_ism; + } + } +#endif } return IVAS_ERR_OK; @@ -724,3 +860,474 @@ void ivas_omasa_rearrange_channels( return; } + +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + +/*-------------------------------------------------------------------------* + * ivas_omasa_combine_separate_ism_with_masa_open() + * + * Open structures, reserve memory, and init values. + *-------------------------------------------------------------------------*/ + +ivas_error ivas_omasa_combine_separate_ism_with_masa_open( + Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +) +{ + int16_t i; + int16_t interpolator_length; + + if ( ( st_ivas->hIsmRendererData = (ISM_RENDERER_HANDLE) malloc( sizeof( ISM_RENDERER_DATA ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Cannot allocate memory for MASA ISM renderer \n" ) ); + } + + for ( i = 0; i < MAX_NUM_OBJECTS; i++ ) + { + set_zero( st_ivas->hIsmRendererData->prev_gains[i], MAX_OUTPUT_CHANNELS ); + } + + interpolator_length = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ); + st_ivas->hIsmRendererData->interpolator = (float *) malloc( sizeof( float ) * interpolator_length ); + + for ( i = 0; i < interpolator_length; i++ ) + { + st_ivas->hIsmRendererData->interpolator[i] = (float) i / ( (float) interpolator_length ); + } + st_ivas->hIsmRendererData->interpolator_length = interpolator_length; + + return IVAS_ERR_OK; +} + + +/*--------------------------------------------------------------------------* + * ivas_omasa_combine_separate_ism_with_masa() + * + * in case of external rendering, combine separated ISM signal with MASA stream + *--------------------------------------------------------------------------*/ + +void ivas_omasa_combine_separate_ism_with_masa( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + float *output[], /* i/o: output synthesis signal */ + const int16_t nchan_ism, /* i : number of ISMs */ + const int16_t output_frame /* i : output frame length per channel */ +) +{ + int16_t n, sf, band, bin, k; + MASA_DECODER_EXT_OUT_META_HANDLE masaMetaHandle; + MASA_DECODER_EXT_OUT_META_HANDLE ismMetaHandle; + MASA_DECODER_EXT_OUT_META ismMeta; + float eneMasa[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; + float eneIsm[MAX_PARAM_SPATIAL_SUBFRAMES][MASA_FREQUENCY_BANDS]; + float inRe[3][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float inIm[3][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float azimuth, elevation; + uint16_t directionIndex; + float old_panning_gains[2]; + float new_panning_gains[2]; + float panned_signal[L_FRAME48k]; + int16_t nchan_in; + int16_t nBins; + int16_t slot; + int16_t mrange[2], brange[2]; + int16_t processing_len, offset; + float g1, g2; + + masaMetaHandle = st_ivas->hMasa->data.extOutMeta; + ismMetaHandle = &ismMeta; + + /* Compute CLDFB analysis */ + nchan_in = st_ivas->nchan_transport + 1; + nBins = output_frame / CLDFB_NO_COL_MAX; + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( n = 0; n < nchan_in; n++ ) + { + cldfbAnalysis_ts( + &( output[n][nBins * slot] ), + inRe[n][slot], + inIm[n][slot], + nBins, st_ivas->cldfbAnaDec[n] ); + } + } + + /* Determine energies */ + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + mrange[0] = st_ivas->hMasa->config.block_grouping[sf]; + mrange[1] = st_ivas->hMasa->config.block_grouping[sf + 1]; + + set_zero( eneMasa[sf], MASA_FREQUENCY_BANDS ); + set_zero( eneIsm[sf], MASA_FREQUENCY_BANDS ); + + for ( slot = mrange[0]; slot < mrange[1]; slot++ ) + { + for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) + { + brange[0] = st_ivas->hMasa->config.band_grouping[band]; + brange[1] = st_ivas->hMasa->config.band_grouping[band + 1]; + + if ( brange[1] > nBins ) + { + brange[1] = nBins; + } + + for ( n = 0; n < MASA_MAX_TRANSPORT_CHANNELS; n++ ) + { + for ( bin = brange[0]; bin < brange[1]; bin++ ) + { + eneMasa[sf][band] += inRe[n][slot][bin] * inRe[n][slot][bin] + inIm[n][slot][bin] * inIm[n][slot][bin]; + } + } + for ( bin = brange[0]; bin < brange[1]; bin++ ) + { + eneIsm[sf][band] += inRe[MASA_MAX_TRANSPORT_CHANNELS][slot][bin] * inRe[MASA_MAX_TRANSPORT_CHANNELS][slot][bin] + inIm[MASA_MAX_TRANSPORT_CHANNELS][slot][bin] * inIm[MASA_MAX_TRANSPORT_CHANNELS][slot][bin]; + } + } + } + } + + /* Determine MASA metadata for the object */ + ismMetaHandle->descriptiveMeta.numberOfDirections = 0u; + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[sf]; + elevation = st_ivas->hMasaIsmData->elevation_separated_ism[sf]; + + directionIndex = index_theta_phi_16( &elevation, &azimuth, st_ivas->hMasa->data.sph_grid16 ); + + for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) + { + ismMetaHandle->directionIndex[0][sf][band] = directionIndex; + ismMetaHandle->directToTotalRatio[0][sf][band] = UINT8_MAX; + ismMetaHandle->spreadCoherence[0][sf][band] = 0; + ismMetaHandle->surroundCoherence[sf][band] = 0; + ismMetaHandle->diffuseToTotalRatio[sf][band] = 0; + } + } + + /* Merge MASA metadatas */ + ivas_prerend_merge_masa_metadata( masaMetaHandle, masaMetaHandle, IVAS_REND_AUDIO_CONFIG_TYPE_MASA, eneMasa, ismMetaHandle, IVAS_REND_AUDIO_CONFIG_TYPE_OBJECT_BASED, eneIsm ); + + /* Mix the separated object audio signal to the MASA audio signals */ + ivas_get_stereo_panning_gains( st_ivas->hMasaIsmData->azimuth_separated_ism[0], st_ivas->hMasaIsmData->elevation_separated_ism[0], old_panning_gains ); + ivas_get_stereo_panning_gains( st_ivas->hMasaIsmData->azimuth_separated_ism[2], st_ivas->hMasaIsmData->elevation_separated_ism[2], new_panning_gains ); + + processing_len = output_frame / 2; + for ( n = 0; n < MASA_MAX_TRANSPORT_CHANNELS; n++ ) + { + v_multc( output[MASA_MAX_TRANSPORT_CHANNELS], old_panning_gains[n], panned_signal, processing_len ); + v_add( output[n], panned_signal, output[n], processing_len ); + } + offset = processing_len; + + processing_len = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; + for ( n = 0; n < MASA_MAX_TRANSPORT_CHANNELS; n++ ) + { + for ( k = 0; k < processing_len; k++ ) + { + g1 = st_ivas->hIsmRendererData->interpolator[k]; + g2 = 1.0f - g1; + output[n][k + offset] += ( g1 * new_panning_gains[n] + g2 * old_panning_gains[n] ) * output[MASA_MAX_TRANSPORT_CHANNELS][k + offset]; + } + } + offset += processing_len; + + for ( n = 0; n < MASA_MAX_TRANSPORT_CHANNELS; n++ ) + { + v_multc( &output[MASA_MAX_TRANSPORT_CHANNELS][offset], new_panning_gains[n], panned_signal, processing_len ); + v_add( &output[n][offset], panned_signal, &output[n][offset], processing_len ); + } + + /* Zero output object channels */ + for ( n = 0; n < nchan_ism; n++ ) + { + set_zero( output[MASA_MAX_TRANSPORT_CHANNELS + n], output_frame ); + } + + return; +} + +/*-------------------------------------------------------------------------* + * ivas_omasa_render_objects_from_mix_open() + * + * Open structures, reserve memory, and init values. + *-------------------------------------------------------------------------*/ + +ivas_error ivas_omasa_render_objects_from_mix_open( + Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +) +{ + int16_t i; + + st_ivas->hMasaIsmData->delayBuffer_nchan = 1; + + st_ivas->hMasaIsmData->delayBuffer_size = (int16_t) ( ( st_ivas->hDecoderConfig->output_Fs / 50 ) / MAX_PARAM_SPATIAL_SUBFRAMES ); + + if ( ( st_ivas->hMasaIsmData->delayBuffer = (float **) malloc( st_ivas->hMasaIsmData->delayBuffer_nchan * sizeof( float * ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Cannot allocate memory for MASA ISM delay buffer \n" ) ); + } + + for ( i = 0; i < st_ivas->hMasaIsmData->delayBuffer_nchan; i++ ) + { + if ( ( st_ivas->hMasaIsmData->delayBuffer[i] = (float *) malloc( st_ivas->hMasaIsmData->delayBuffer_size * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Cannot allocate memory for MASA ISM delay buffer \n" ) ); + } + set_zero( st_ivas->hMasaIsmData->delayBuffer[i], st_ivas->hMasaIsmData->delayBuffer_size ); + } + + return IVAS_ERR_OK; +} + +/*--------------------------------------------------------------------------* + * ivas_omasa_render_objects_from_mix() + * + * In case of external rendering, render objects from the transport signal + * mix containing MASA audio and object audio. + *--------------------------------------------------------------------------*/ + +void ivas_omasa_render_objects_from_mix( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + float *output[], /* o : output synthesis signal */ + const int16_t nchan_ism, /* i : number of ISMs */ + const int16_t output_frame /* i : output frame length per channel */ +) +{ + int16_t n, m, i; + MASA_ISM_EXT_DATA_HANDLE hExtData; + float separated_object[L_FRAME48k]; + float rendered_objects[MAX_NUM_OBJECTS][L_FRAME48k]; + int16_t coding_delay; + float new_panning_gains[2]; + float panning_gains[2]; + float azimuth, elevation; + float inRe[3][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float inIm[3][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float outRe[MAX_NUM_OBJECTS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float outIm[MAX_NUM_OBJECTS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float ism_proto_energy[MAX_NUM_OBJECTS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float transport_energy[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float ism_target_energy[MAX_NUM_OBJECTS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float masa_target_energy[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float ism_processing_gains[MAX_NUM_OBJECTS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + float masa_processing_gains[CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; + int16_t slot; + int16_t sf; + int16_t bin; + int16_t nchan_transport; + int16_t nBins; + float interpVal; + float *outSlotRePr, *outSlotImPr; + int16_t md_idx; + float iir_factor_prev, iir_factor_curr; + + /* Create slot to metadata map */ + for ( sf = 0, slot = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + for ( i = 0; i < CLDFB_SLOTS_PER_SUBFRAME; i++, slot++ ) + { + st_ivas->hSpatParamRendCom->render_to_md_map[slot] = st_ivas->hSpatParamRendCom->dirac_read_idx; + } + st_ivas->hSpatParamRendCom->dirac_read_idx = ( st_ivas->hSpatParamRendCom->dirac_read_idx + 1 ) % st_ivas->hSpatParamRendCom->dirac_md_buffer_length; + } + + /* Move separated object signal and object channels */ + mvr2r( output[CPE_CHANNELS], separated_object, output_frame ); + for ( n = 0; n < nchan_ism; n++ ) + { + set_zero( output[CPE_CHANNELS + n], output_frame ); + } + + /* Delay the separated object signal by the CLDFB delay */ + delay_signal( separated_object, output_frame, st_ivas->hMasaIsmData->delayBuffer[0], st_ivas->hMasaIsmData->delayBuffer_size ); + + /* Set object metadata to the ism struct */ + for ( n = 0; n < nchan_ism; n++ ) + { + st_ivas->hIsmMetaData[n]->azimuth = st_ivas->hMasaIsmData->azimuth_ism[n][st_ivas->hSpatParamRendCom->dirac_read_idx]; + st_ivas->hIsmMetaData[n]->elevation = st_ivas->hMasaIsmData->elevation_ism[n][st_ivas->hSpatParamRendCom->dirac_read_idx]; + } + + /* Move the separated object signal to the correct output channel */ + hExtData = st_ivas->hMasaIsmData->hExtData; + coding_delay = output_frame / 20 * 17; /* 17 ms of coding and CLDFB delay */ + mvr2r( separated_object, output[CPE_CHANNELS + hExtData->prev_idx_separated_ism], coding_delay ); + mvr2r( &separated_object[coding_delay], &output[CPE_CHANNELS + st_ivas->hMasaIsmData->idx_separated_ism][coding_delay], output_frame - coding_delay ); + + /* Compute CLDFB analysis */ + nchan_transport = st_ivas->nchan_transport; + nBins = output_frame / CLDFB_NO_COL_MAX; + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( n = 0; n < nchan_transport; n++ ) + { + cldfbAnalysis_ts( + &( output[n][nBins * slot] ), + inRe[n][slot], + inIm[n][slot], + nBins, st_ivas->cldfbAnaDec[n] ); + } + } + + /* Create prototype signals */ + for ( n = 0; n < nchan_ism; n++ ) + { + azimuth = (float) st_ivas->hIsmMetaData[n]->azimuth; + elevation = (float) st_ivas->hIsmMetaData[n]->elevation; + ivas_get_stereo_panning_gains( azimuth, elevation, new_panning_gains ); + interpVal = 0.0f; + + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + interpVal += 1.0f / (float) CLDFB_NO_COL_MAX; + for ( m = 0; m < 2; m++ ) + { + panning_gains[m] = ( 1.0f - interpVal ) * hExtData->prev_panning_gains[n][m] + interpVal * new_panning_gains[m]; + } + v_multc( inRe[0][slot], panning_gains[0], outRe[n][slot], nBins ); + v_multc( inIm[0][slot], panning_gains[0], outIm[n][slot], nBins ); + v_multc_acc( inRe[1][slot], panning_gains[1], outRe[n][slot], nBins ); + v_multc_acc( inIm[1][slot], panning_gains[1], outIm[n][slot], nBins ); + } + + for ( m = 0; m < 2; m++ ) + { + hExtData->prev_panning_gains[n][m] = new_panning_gains[m]; + } + } + + /* Determine prototype energy */ + for ( n = 0; n < nchan_ism; n++ ) + { + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + ism_proto_energy[n][slot][bin] = ( outRe[n][slot][bin] * outRe[n][slot][bin] ) + ( outIm[n][slot][bin] * outIm[n][slot][bin] ); + } + } + } + + /* Determine transport energy */ + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + set_zero( transport_energy[slot], nBins ); + } + for ( n = 0; n < CPE_CHANNELS; n++ ) + { + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + transport_energy[slot][bin] += ( inRe[n][slot][bin] * inRe[n][slot][bin] ) + ( inIm[n][slot][bin] * inIm[n][slot][bin] ); + } + } + } + + /* Determine target energy */ + for ( n = 0; n < nchan_ism; n++ ) + { + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + md_idx = st_ivas->hSpatParamRendCom->render_to_md_map[slot]; + for ( bin = 0; bin < nBins; bin++ ) + { + ism_target_energy[n][slot][bin] = transport_energy[slot][bin] * st_ivas->hMasaIsmData->energy_ratio_ism[n][md_idx][bin]; + } + } + } + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + md_idx = st_ivas->hSpatParamRendCom->render_to_md_map[slot]; + for ( bin = 0; bin < nBins; bin++ ) + { + masa_target_energy[slot][bin] = transport_energy[slot][bin] * hExtData->masa_render_masa_to_total[md_idx][bin]; + } + } + + /* Determine temporally smoothed energies and determine gains using them */ + iir_factor_curr = ( 1.0f - EXT_RENDER_IIR_FAC ); + iir_factor_prev = EXT_RENDER_IIR_FAC; + for ( n = 0; n < nchan_ism; n++ ) + { + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + hExtData->ism_render_proto_energy[n][bin] *= iir_factor_prev; + hExtData->ism_render_proto_energy[n][bin] += iir_factor_curr * ism_proto_energy[n][slot][bin]; + hExtData->ism_render_target_energy[n][bin] *= iir_factor_prev; + hExtData->ism_render_target_energy[n][bin] += iir_factor_curr * ism_target_energy[n][slot][bin]; + + ism_processing_gains[n][slot][bin] = fminf( 4.0f, sqrtf( hExtData->ism_render_target_energy[n][bin] / fmaxf( 1e-12f, hExtData->ism_render_proto_energy[n][bin] ) ) ); + } + } + } + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + hExtData->masa_render_proto_energy[bin] *= iir_factor_prev; + hExtData->masa_render_proto_energy[bin] += iir_factor_curr * transport_energy[slot][bin]; + hExtData->masa_render_target_energy[bin] *= iir_factor_prev; + hExtData->masa_render_target_energy[bin] += iir_factor_curr * masa_target_energy[slot][bin]; + + masa_processing_gains[slot][bin] = fminf( 4.0f, sqrtf( hExtData->masa_render_target_energy[bin] / fmaxf( 1e-12f, hExtData->masa_render_proto_energy[bin] ) ) ); + } + } + + /* Determine output signals */ + for ( n = 0; n < nchan_ism; n++ ) + { + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + outRe[n][slot][bin] *= ism_processing_gains[n][slot][bin]; + outIm[n][slot][bin] *= ism_processing_gains[n][slot][bin]; + } + } + } + for ( n = 0; n < MASA_MAX_TRANSPORT_CHANNELS; n++ ) + { + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( bin = 0; bin < nBins; bin++ ) + { + inRe[n][slot][bin] *= masa_processing_gains[slot][bin]; + inIm[n][slot][bin] *= masa_processing_gains[slot][bin]; + } + } + } + + /* Compute CLDFB synthesis */ + for ( slot = 0; slot < CLDFB_NO_COL_MAX; slot++ ) + { + for ( n = 0; n < nchan_transport; n++ ) + { + outSlotRePr = &( inRe[n][slot][0] ); + outSlotImPr = &( inIm[n][slot][0] ); + + cldfbSynthesis( &outSlotRePr, &outSlotImPr, &( output[n][nBins * slot] ), nBins, st_ivas->cldfbSynDec[n] ); + } + + for ( n = 0; n < nchan_ism; n++ ) + { + outSlotRePr = &( outRe[n][slot][0] ); + outSlotImPr = &( outIm[n][slot][0] ); + cldfbSynthesis( &outSlotRePr, &outSlotImPr, &( rendered_objects[n][nBins * slot] ), nBins, st_ivas->cldfbSynDec[n + CPE_CHANNELS] ); + } + } + + /* Combine the rendered objects with the separated objects */ + for ( n = 0; n < nchan_ism; n++ ) + { + v_add( output[CPE_CHANNELS + n], rendered_objects[n], output[CPE_CHANNELS + n], output_frame ); + } + + hExtData->prev_idx_separated_ism = st_ivas->hMasaIsmData->idx_separated_ism; + + return; +} +#endif diff --git a/lib_dec/ivas_output_config.c b/lib_dec/ivas_output_config.c index dbdb0df16..c44b59e8c 100644 --- a/lib_dec/ivas_output_config.c +++ b/lib_dec/ivas_output_config.c @@ -388,7 +388,23 @@ void ivas_renderer_select( } else if ( output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) { +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + *renderer_type = RENDERER_OMASA_OBJECT_EXT; + } + else if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ ) + { + *renderer_type = RENDERER_OMASA_MIX_EXT; + } + else + { + *renderer_type = RENDERER_DISABLE; + } +#else // NONBE_FIX_984_OMASA_EXT_OUTPUT *renderer_type = RENDERER_DISABLE; +#endif } } else if ( st_ivas->ivas_format == MC_FORMAT ) diff --git a/lib_dec/ivas_sba_dec.c b/lib_dec/ivas_sba_dec.c index cff1eef75..6f5a67999 100644 --- a/lib_dec/ivas_sba_dec.c +++ b/lib_dec/ivas_sba_dec.c @@ -667,7 +667,11 @@ void ivas_sba_dec_digest_tc( int16_t ch_idx, nchan_transport; /* set the md map */ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( st_ivas->hDirAC || st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) +#else if ( st_ivas->hDirAC ) +#endif { ivas_dirac_dec_set_md_map( st_ivas, nCldfbSlots ); } diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index de3589fb1..83ab0a70d 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -877,6 +877,20 @@ typedef struct ivas_masa_decoder_struct } MASA_DECODER, *MASA_DECODER_HANDLE; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT +/* Data structure for MASA_ISM EXT rendering */ +typedef struct ivas_masa_ism_ext_data_structure +{ + int16_t prev_idx_separated_ism; + float prev_panning_gains[MAX_NUM_OBJECTS][2]; + float ism_render_proto_energy[MAX_NUM_OBJECTS][CLDFB_NO_CHANNELS_MAX]; + float ism_render_target_energy[MAX_NUM_OBJECTS][CLDFB_NO_CHANNELS_MAX]; + float masa_render_proto_energy[CLDFB_NO_CHANNELS_MAX]; + float masa_render_target_energy[CLDFB_NO_CHANNELS_MAX]; + float masa_render_masa_to_total[MAX_PARAM_SPATIAL_SUBFRAMES + DELAY_MASA_PARAM_DEC_SFR][CLDFB_NO_CHANNELS_MAX]; +} MASA_ISM_EXT_DATA, *MASA_ISM_EXT_DATA_HANDLE; +#endif + /* Data structure for MASA_ISM rendering */ typedef struct ivas_masa_ism_data_structure { @@ -907,6 +921,9 @@ typedef struct ivas_masa_ism_data_structure int16_t delayBuffer_size; int16_t delayBuffer_nchan; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + MASA_ISM_EXT_DATA_HANDLE hExtData; +#endif } MASA_ISM_DATA, *MASA_ISM_DATA_HANDLE; diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index e08b838e3..72b99d2be 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -1480,11 +1480,26 @@ ivas_error IVAS_DEC_GetNumObjects( uint16_t *numObjects /* o : number of objects for which the decoder has been configured */ ) { +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + int16_t is_masa_ism; + is_masa_ism = 0; +#endif if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( hIvasDec->st_ivas->hMasa != NULL ) + { + if ( hIvasDec->st_ivas->hMasa->config.input_ivas_format == MASA_ISM_FORMAT ) + { + is_masa_ism = 1; + } + } + if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT || is_masa_ism ) +#else if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT ) +#endif { *numObjects = hIvasDec->st_ivas->nchan_ism; } @@ -1517,7 +1532,12 @@ ivas_error IVAS_DEC_GetFormat( { *format = IVAS_DEC_BS_UNKOWN; } - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( *format == IVAS_DEC_BS_MASA && hIvasDec->st_ivas->hMasa->config.input_ivas_format == MASA_ISM_FORMAT ) + { + *format = IVAS_DEC_BS_MASA_ISM; + } +#endif return IVAS_ERR_OK; } @@ -1625,14 +1645,28 @@ ivas_error IVAS_DEC_GetObjectMetadata( { Decoder_Struct *st_ivas; ISM_METADATA_HANDLE hIsmMeta; - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + int16_t is_masa_ism; + is_masa_ism = 0; +#endif if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } st_ivas = hIvasDec->st_ivas; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( hIvasDec->st_ivas->hMasa != NULL ) + { + if ( hIvasDec->st_ivas->hMasa->config.input_ivas_format == MASA_ISM_FORMAT ) + { + is_masa_ism = 1; + } + } + if ( st_ivas->ivas_format != ISM_FORMAT && st_ivas->ivas_format != MASA_ISM_FORMAT && st_ivas->ivas_format != SBA_ISM_FORMAT && is_masa_ism == 0 ) +#else if ( st_ivas->ivas_format != ISM_FORMAT && st_ivas->ivas_format != MASA_ISM_FORMAT && st_ivas->ivas_format != SBA_ISM_FORMAT ) +#endif { return IVAS_ERR_WRONG_MODE; } @@ -1643,8 +1677,11 @@ ivas_error IVAS_DEC_GetObjectMetadata( } hIsmMeta = st_ivas->hIsmMetaData[objectIdx]; - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( hIsmMeta == NULL || zero_flag || ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ ) ) +#else if ( hIsmMeta == NULL || zero_flag ) +#endif { metadata->azimuth = 0.f; metadata->elevation = 0.f; @@ -2215,8 +2252,19 @@ ivas_error IVAS_DEC_GetDelay( st_ivas = hIvasDec->st_ivas; hDecoderConfig = st_ivas->hDecoderConfig; - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT +#ifdef SPLIT_REND_WITH_HEAD_ROT + nSamples[1] = NS2SA( hDecoderConfig->output_Fs, get_delay( DEC, hDecoderConfig->output_Fs, st_ivas->ivas_format, st_ivas->cldfbSynDec[0], hDecoderConfig->output_config ) ); +#else + nSamples[1] = NS2SA( hDecoderConfig->output_Fs, get_delay( DEC, hDecoderConfig->output_Fs, st_ivas->ivas_format, st_ivas->cldfbSynDec[0], hDecoderConfig->output_config ) ); +#endif +#else +#ifdef SPLIT_REND_WITH_HEAD_ROT nSamples[1] = NS2SA( hDecoderConfig->output_Fs, get_delay( DEC, hDecoderConfig->output_Fs, st_ivas->ivas_format, st_ivas->cldfbAnaDec[0], hDecoderConfig->output_config ) ); +#else + nSamples[1] = NS2SA( hDecoderConfig->output_Fs, get_delay( DEC, hDecoderConfig->output_Fs, st_ivas->ivas_format, st_ivas->cldfbAnaDec[0], hDecoderConfig->output_config ) ); +#endif +#endif nSamples[2] = (int16_t) roundf( (float) st_ivas->binaural_latency_ns * hDecoderConfig->output_Fs / 1000000000.f ); nSamples[0] = nSamples[1] + nSamples[2]; diff --git a/lib_enc/ivas_masa_enc.c b/lib_enc/ivas_masa_enc.c index ee6be5042..3c9eb20d2 100644 --- a/lib_enc/ivas_masa_enc.c +++ b/lib_enc/ivas_masa_enc.c @@ -418,18 +418,49 @@ ivas_error ivas_masa_encode( } else { - /* write the number of MASA transport channels */ - push_next_indice( hMetaData, nchan_transport - 1, MASA_TRANSP_BITS ); +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( ivas_format == MASA_ISM_FORMAT && ism_mode == ISM_MODE_NONE ) + { + /* use the MASA number of transport channels bit to signal if there are 3 or 4 objects */ + if ( nchan_ism == 4 ) + { + push_next_indice( hMetaData, 1, MASA_TRANSP_BITS ); + } + else + { + push_next_indice( hMetaData, 0, MASA_TRANSP_BITS ); + } + } + else + { +#endif + /* write the number of MASA transport channels */ + push_next_indice( hMetaData, nchan_transport - 1, MASA_TRANSP_BITS ); +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + } +#endif hQMetaData->metadata_max_bits -= MASA_TRANSP_BITS; } if ( ivas_format == MASA_ISM_FORMAT && ism_mode == ISM_MODE_NONE ) { +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( nchan_ism <= 3 ) + { + push_next_indice( hMetaData, nchan_ism, MASA_HEADER_BITS ); + } + else + { + push_next_indice( hMetaData, nchan_ism - 1, MASA_HEADER_BITS ); + } + hQMetaData->metadata_max_bits -= MASA_HEADER_BITS; +#else /* signal MASA_ISM_FORMAT to decoder */ push_next_indice( hMetaData, 1, 1 ); /* write reserved bit */ push_next_indice( hMetaData, 0, MASA_HEADER_BITS - 1 ); hQMetaData->metadata_max_bits -= MASA_HEADER_BITS; +#endif } else { diff --git a/lib_rend/ivas_output_init.c b/lib_rend/ivas_output_init.c index 24e619722..39ff3d95b 100644 --- a/lib_rend/ivas_output_init.c +++ b/lib_rend/ivas_output_init.c @@ -333,6 +333,12 @@ int16_t ivas_get_nchan_buffers_dec( nchan_out_buff = max( nchan_out_buff, audioCfg2channels( st_ivas->intern_config ) ); nchan_out_buff = max( nchan_out_buff, audioCfg2channels( output_config ) ); } +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + if ( output_config == IVAS_AUDIO_CONFIG_EXTERNAL && st_ivas->nchan_ism > 0 ) + { + nchan_out_buff = st_ivas->nchan_ism + CPE_CHANNELS; + } +#endif } else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) { diff --git a/lib_util/masa_file_writer.c b/lib_util/masa_file_writer.c index 0939d0829..c7dd7df86 100644 --- a/lib_util/masa_file_writer.c +++ b/lib_util/masa_file_writer.c @@ -41,12 +41,20 @@ typedef struct masaMetaDelayStorage { MASA_DECRIPTIVE_META descriptiveMeta; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + uint16_t directionIndex[MASA_MAXIMUM_DIRECTIONS][DELAY_MASA_PARAM_DEC_SFR + 1][MASA_FREQUENCY_BANDS]; + uint8_t directToTotalRatio[MASA_MAXIMUM_DIRECTIONS][DELAY_MASA_PARAM_DEC_SFR + 1][MASA_FREQUENCY_BANDS]; + uint8_t spreadCoherence[MASA_MAXIMUM_DIRECTIONS][DELAY_MASA_PARAM_DEC_SFR + 1][MASA_FREQUENCY_BANDS]; + uint8_t surroundCoherence[DELAY_MASA_PARAM_DEC_SFR + 1][MASA_FREQUENCY_BANDS]; + uint8_t diffuseToTotalRatio[DELAY_MASA_PARAM_DEC_SFR + 1][MASA_FREQUENCY_BANDS]; + uint8_t prevDelay; +#else uint16_t directionIndex[MASA_MAXIMUM_DIRECTIONS][DELAY_MASA_PARAM_DEC_SFR][MASA_FREQUENCY_BANDS]; uint8_t directToTotalRatio[MASA_MAXIMUM_DIRECTIONS][DELAY_MASA_PARAM_DEC_SFR][MASA_FREQUENCY_BANDS]; uint8_t spreadCoherence[MASA_MAXIMUM_DIRECTIONS][DELAY_MASA_PARAM_DEC_SFR][MASA_FREQUENCY_BANDS]; uint8_t surroundCoherence[DELAY_MASA_PARAM_DEC_SFR][MASA_FREQUENCY_BANDS]; uint8_t diffuseToTotalRatio[DELAY_MASA_PARAM_DEC_SFR][MASA_FREQUENCY_BANDS]; - +#endif } MASA_META_DELAY_STORAGE; struct MasaFileWriter @@ -88,14 +96,94 @@ static void getExtMasaMetadataFileName( static void delayMasaMetadata( MASA_DECODER_EXT_OUT_META_HANDLE extOutMeta, /* i/o: New input metadata which is inplace replaced with delayed metadata frame */ - MASA_META_DELAY_STORAGE *delayStorage /* i/o: Storage for 10 ms of metadata and related descriptive metadata */ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + MASA_META_DELAY_STORAGE *delayStorage, /* i/o: Storage for 15 ms of metadata and related descriptive metadata */ + uint8_t delayNsf +#else + MASA_META_DELAY_STORAGE *delayStorage /* i/o: Storage for 10 ms of metadata and related descriptive metadata */ +#endif ) { int16_t dir, sf, band; uint8_t currentNumberOfDirections; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + int16_t storeReadOffset; +#endif /* Move meta to delay and output. Always use two directions as the metadata is prepared to contain zero energy second direction * if there is 1dir meta. */ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + /* stable state expected results delay change expected results + delayNsf = 2 delayNsf = 3 delayNsf = 3 (from 2) delayNsf = 2 (from 3) + ext[0] = delayStorage[0] (prev ext[2]) ext[0] = delayStorage[0] (prev ext[1]) ext[0] = delayStorage[0] (prev ext[2]) ext[0] = delayStorage[1] (prev ext[2]) + ext[1] = delayStorage[1] (prev ext[3]) ext[1] = delayStorage[1] (prev ext[2]) ext[1] = delayStorage[1] (prev ext[3]) ext[1] = delayStorage[2] (prev ext[3]) + ext[2] = ext[0] ext[2] = delayStorage[2] (prev ext[3]) ext[2] = delayStorage[1] (prev ext[3]) ext[2] = ext[0] + ext[3] = ext[1] ext[3] = ext[0] ext[3] = ext[0] ext[3] = ext[0] + delayStorage[0] = ext[2] delayStorage[0] = ext[1] delayStorage[0] = ext[1] delayStorage[0] = ext[2] + delayStorage[1] = ext[3] delayStorage[1] = ext[2] delayStorage[1] = ext[2] delayStorage[1] = ext[3] + delayStorage[2] = ext[3] delayStorage[2] = ext[3] + */ + storeReadOffset = delayStorage->prevDelay > delayNsf ? delayStorage->prevDelay - delayNsf : 0; /* delay decreases: read later from storage -> discard one sf */ + + for ( sf = 0; sf < delayNsf; sf++ ) + { + for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) + { + uint16_t temp_u16; + uint8_t temp_u8; + int16_t toStoreIdx, reOrgIdx, storeReadIdx; + + toStoreIdx = sf + MAX_PARAM_SPATIAL_SUBFRAMES - delayNsf; + reOrgIdx = sf + delayNsf; + /* when switching to longer delay, repeat the last valid sf to fill the gap */ + storeReadIdx = ( ( sf + storeReadOffset ) < delayStorage->prevDelay ) ? sf + storeReadOffset : delayStorage->prevDelay - 1; + + for ( dir = 0; dir < MASA_MAXIMUM_DIRECTIONS; dir++ ) + { + temp_u16 = delayStorage->directionIndex[dir][storeReadIdx][band]; + delayStorage->directionIndex[dir][sf][band] = extOutMeta->directionIndex[dir][toStoreIdx][band]; + if ( reOrgIdx < MAX_PARAM_SPATIAL_SUBFRAMES ) + { + extOutMeta->directionIndex[dir][reOrgIdx][band] = extOutMeta->directionIndex[dir][sf][band]; + } + extOutMeta->directionIndex[dir][sf][band] = temp_u16; + + temp_u8 = delayStorage->directToTotalRatio[dir][storeReadIdx][band]; + delayStorage->directToTotalRatio[dir][sf][band] = extOutMeta->directToTotalRatio[dir][toStoreIdx][band]; + if ( reOrgIdx < MAX_PARAM_SPATIAL_SUBFRAMES ) + { + extOutMeta->directToTotalRatio[dir][reOrgIdx][band] = extOutMeta->directToTotalRatio[dir][sf][band]; + } + extOutMeta->directToTotalRatio[dir][sf][band] = temp_u8; + + temp_u8 = delayStorage->spreadCoherence[dir][storeReadIdx][band]; + delayStorage->spreadCoherence[dir][sf][band] = extOutMeta->spreadCoherence[dir][toStoreIdx][band]; + if ( reOrgIdx < MAX_PARAM_SPATIAL_SUBFRAMES ) + { + extOutMeta->spreadCoherence[dir][reOrgIdx][band] = extOutMeta->spreadCoherence[dir][sf][band]; + } + extOutMeta->spreadCoherence[dir][sf][band] = temp_u8; + } + + temp_u8 = delayStorage->surroundCoherence[storeReadIdx][band]; + delayStorage->surroundCoherence[sf][band] = extOutMeta->surroundCoherence[toStoreIdx][band]; + if ( reOrgIdx < MAX_PARAM_SPATIAL_SUBFRAMES ) + { + extOutMeta->surroundCoherence[reOrgIdx][band] = extOutMeta->surroundCoherence[sf][band]; + } + extOutMeta->surroundCoherence[sf][band] = temp_u8; + + temp_u8 = delayStorage->diffuseToTotalRatio[storeReadIdx][band]; + delayStorage->diffuseToTotalRatio[sf][band] = extOutMeta->diffuseToTotalRatio[toStoreIdx][band]; + if ( reOrgIdx < MAX_PARAM_SPATIAL_SUBFRAMES ) + { + extOutMeta->diffuseToTotalRatio[reOrgIdx][band] = extOutMeta->diffuseToTotalRatio[sf][band]; + } + extOutMeta->diffuseToTotalRatio[sf][band] = temp_u8; + } + } + +#else for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES - DELAY_MASA_PARAM_DEC_SFR; sf++ ) { for ( band = 0; band < MASA_FREQUENCY_BANDS; band++ ) @@ -131,7 +219,7 @@ static void delayMasaMetadata( extOutMeta->diffuseToTotalRatio[sf][band] = temp_u8; } } - +#endif /* Finalize descriptive meta by using new frame except for number of directions which is the larger of the two */ currentNumberOfDirections = extOutMeta->descriptiveMeta.numberOfDirections; if ( delayStorage->descriptiveMeta.numberOfDirections > extOutMeta->descriptiveMeta.numberOfDirections ) @@ -139,7 +227,9 @@ static void delayMasaMetadata( extOutMeta->descriptiveMeta.numberOfDirections = delayStorage->descriptiveMeta.numberOfDirections; } delayStorage->descriptiveMeta.numberOfDirections = currentNumberOfDirections; - +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + delayStorage->prevDelay = delayNsf; +#endif return; } @@ -182,6 +272,9 @@ ivas_error MasaFileWriter_open( if ( !delayCompensationEnabled ) { self->delayStorage = calloc( sizeof( MASA_META_DELAY_STORAGE ), 1 ); +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + self->delayStorage->prevDelay = DELAY_MASA_PARAM_DEC_SFR; +#endif } *masaWriter = self; @@ -197,8 +290,13 @@ ivas_error MasaFileWriter_open( *---------------------------------------------------------------------*/ ivas_error MasaFileWriter_writeFrame( - MasaFileWriter *self, /* i/o: MasaFileWriter handle */ + MasaFileWriter *self, /* i/o: MasaFileWriter handle */ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta, /* i/o: MASA ext out meta handle to be written */ + const float *decDelay /* i : decoding audio delay */ +#else MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta /* i/o: MASA ext out meta handle to be written */ +#endif ) { if ( self == NULL ) @@ -209,7 +307,17 @@ ivas_error MasaFileWriter_writeFrame( uint16_t descMetaTemp = 0; int16_t i, sf, dir, numDirections; uint8_t writeTempOther[MASA_FREQUENCY_BANDS]; +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + /* If delay storage has been reserved, then we are in the normal mode for the decoder + * (i.e., no delay compensation for PCM) which means that metadata should be delayed + * by two or three subframes (10 or 15 ms). Descriptive metadata is a combined result. */ + if ( self->delayStorage ) + { + uint8_t delayFrames = (uint8_t) ( (int32_t) ( ( *decDelay ) * 48000 ) / L_SPATIAL_SUBFR_48k ); + delayMasaMetadata( hMasaExtOutMeta, self->delayStorage, delayFrames ); + } +#else /* If delay storage has been reserved, then we are in the normal mode for the decoder * (i.e., no delay compensation for PCM) which means that metadata should be delayed * by two subframes (10 ms). Descriptive metadata is a combined result. */ @@ -217,6 +325,7 @@ ivas_error MasaFileWriter_writeFrame( { delayMasaMetadata( hMasaExtOutMeta, self->delayStorage ); } +#endif numDirections = hMasaExtOutMeta->descriptiveMeta.numberOfDirections + 1; diff --git a/lib_util/masa_file_writer.h b/lib_util/masa_file_writer.h index 2d476e28f..c3e142254 100644 --- a/lib_util/masa_file_writer.h +++ b/lib_util/masa_file_writer.h @@ -47,8 +47,13 @@ ivas_error MasaFileWriter_open( ); ivas_error MasaFileWriter_writeFrame( - MasaFileWriter *self, /* i/o: MasaFileWriter handle */ + MasaFileWriter *self, /* i/o: MasaFileWriter handle */ +#ifdef NONBE_FIX_984_OMASA_EXT_OUTPUT + IVAS_MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta, /* i/o: MASA ext out meta handle to be written */ + const float *decDelay /* i : decoding audio delay */ +#else IVAS_MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta /* i/o: MASA ext out meta handle to be written */ +#endif ); void MasaFileWriter_close( -- GitLab From 83a160a070863adfb462616452ccfce5c5817d0d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 1 Aug 2025 09:09:43 +0200 Subject: [PATCH 2/2] dummy commit to trigger CI -- GitLab