diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index 3eb310192dbd9ee9b139901ccd0730703fca3ebc..98229a668d8e11f1b576e22ad1a41182f48d932c 100644 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -5026,7 +5026,13 @@ void ivas_masa_enc_reconfigure( ); ivas_error ivas_masa_dec_reconfigure( +#ifdef MASA_AND_OBJECTS + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ + int16_t *data /* o : flushed PCM samples */ +#else Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +#endif ); ivas_error ivas_masa_encode( @@ -5723,7 +5729,13 @@ ivas_error ivas_omasa_enc_config( ); ivas_error ivas_omasa_dec_config( +#ifdef MASA_AND_OBJECTS + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ + int16_t *data /* o : flushed PCM samples */ +#else Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +#endif ); void ivas_omasa_set_config( @@ -5835,12 +5847,34 @@ ivas_error ivas_omasa_dirac_td_binaural( const int16_t output_frame /* i : output frame length per channel */ ); +#ifdef MASA_AND_OBJECTS +ivas_error ivas_omasa_dirac_td_binaural_jbm( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of samples requested */ + uint16_t *nSamplesRendered, /* o : number of samples rendered */ + uint16_t *nSamplesAvailable, /* o : number of samples still to render */ + const int16_t nchan_transport, /* i : number of transport channels */ + float *output_f[] /* o : rendered time signal */ +); +#endif + void ivas_omasa_dirac_rend( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ float output[][L_FRAME48k], /* o : output synthesis signal */ const int16_t output_frame /* i : output frame length per channel */ ); +#ifdef MASA_AND_OBJECTS +void ivas_omasa_dirac_rend_jbm( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of samples requested */ + uint16_t *nSamplesRendered, /* o : number of samples rendered */ + uint16_t *nSamplesAvailable, /* o : number of samples still to render */ + const int16_t nchan_transport, /* i : number of transport channels */ + float *output_f[] /* o : rendered time signal */ +); +#endif + void ivas_omasa_preProcessStereoTransportsForMovedObjects( Decoder_Struct *st_ivas, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], @@ -5863,6 +5897,17 @@ void ivas_omasa_separate_object_render( float output_f[][L_FRAME48k], /* i/o: output signals */ const int16_t output_frame /* i : output frame length per channel */ ); + +#ifdef MASA_AND_OBJECTS +void ivas_omasa_separate_object_render_jbm( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + const uint16_t nSamplesRendered, /* i : number of samples rendered */ + float *output_f[], /* o : rendered time signal */ + const int16_t subframes_rendered, /* i : number of subframes rendered */ + const int16_t slots_rendered /* i : number of CLDFB slots rendered */ +); +#endif + #ifndef FIX__657_REMOVE_EDITING void ivas_omasa_set_edited_objects( Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 2734c9e9ac4d93f69ce560826c9af8af202fb8f2..353eaddc6009679793223144e767407eada76519 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -326,7 +326,11 @@ ivas_error ivas_dec_setup( } else { - if ( ( error = ivas_masa_dec_reconfigure( st_ivas ) ) != IVAS_ERR_OK ) +#ifdef MASA_AND_OBJECTS + if ( ( error = ivas_masa_dec_reconfigure( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) +#else + if ( ( error = ivas_masa_dec_reconfigure( st_ivas ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -335,7 +339,7 @@ ivas_error ivas_dec_setup( } else { - if ( ( error = ivas_omasa_dec_config( st_ivas ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_omasa_dec_config( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) { return error; } @@ -358,7 +362,7 @@ ivas_error ivas_dec_setup( /* reconfigure in case a change of operation mode is detected */ if ( ( ivas_total_brate > IVAS_SID_5k2 && ivas_total_brate != st_ivas->hDecoderConfig->last_ivas_total_brate ) || ( st_ivas->ini_active_frame == 0 ) ) { - if ( ( error = ivas_omasa_dec_config( st_ivas ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_omasa_dec_config( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) { return error; } diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index ced15186f930e800f73f1e565a394b486a5d7a67..0a40ab325b7e7b893973cb68cdfdb2bcb9be72f4 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -371,7 +371,6 @@ ivas_error ivas_omasa_separate_object_renderer_open( set_f( st_ivas->hIsmRendererData->prev_gains[i], 0.0f, MAX_OUTPUT_CHANNELS ); } - // Todo OMASA JBM: This needs touches for VOIP path at least. Current version is mostly an adapted copy from ivas_ism_renderer_open() if ( st_ivas->hDecoderConfig->voip_active ) { init_interpolator_length = NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_CLDFB_TIMESLOTS * CLDFB_SLOT_NS ); @@ -388,6 +387,7 @@ ivas_error ivas_omasa_separate_object_renderer_open( { st_ivas->hIsmRendererData->interpolator[i] = (float) i / ( (float) interpolator_length ); } + st_ivas->hIsmRendererData->interpolator_length = interpolator_length; st_ivas->hMasaIsmData->delayBuffer_size = (int16_t) ( ( st_ivas->hDecoderConfig->output_Fs / 50 ) / MAX_PARAM_SPATIAL_SUBFRAMES ); @@ -470,7 +470,6 @@ void ivas_omasa_separate_object_renderer_close( * Rendering separated objects and mixing them to the parametrically rendered signals *-------------------------------------------------------------------------*/ -// Todo OMASA JBM: This might need adjustments void ivas_omasa_separate_object_render( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ float input_f[][L_FRAME48k], /* i : separated object signal */ @@ -575,4 +574,161 @@ void ivas_omasa_separate_object_render( return; } + +/*-------------------------------------------------------------------------* + * ivas_omasa_separate_object_render_jbm() + * + * Rendering separated objects and mixing them to the parametrically rendered signals for JBM + *-------------------------------------------------------------------------*/ + +void ivas_omasa_separate_object_render_jbm( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + const uint16_t nSamplesRendered, /* i : number of samples rendered */ + float *output_f[], /* o : rendered time signal */ + const int16_t subframes_rendered, /* i : number of subframes rendered */ + const int16_t slots_rendered /* i : number of CLDFB slots rendered */ +) +{ + VBAP_HANDLE hVBAPdata; + DIRAC_REND_HANDLE hDirACRend; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + int16_t nchan_out_woLFE, num_lfe; + ISM_RENDERER_HANDLE hRendererData; + int16_t j, k, j2; + int16_t obj; + float gains[MAX_OUTPUT_CHANNELS]; + float g1, g2; + int16_t lfe_index; + int16_t azimuth, elevation; + int16_t num_objects; + uint8_t single_separated; + float *input_f[MAX_TRANSPORT_CHANNELS]; + float *output_f_local[MAX_OUTPUT_CHANNELS]; + int16_t offsetSamples; + int16_t n_samples_sf, md_idx; + int16_t slots_to_render, first_sf, last_sf, subframe_idx; + + hVBAPdata = st_ivas->hVBAPdata; + hDirACRend = st_ivas->hDirACRend; + hSpatParamRendCom = st_ivas->hSpatParamRendCom; + nchan_out_woLFE = st_ivas->hIntSetup.nchan_out_woLFE; + num_lfe = st_ivas->hIntSetup.num_lfe; + hRendererData = st_ivas->hIsmRendererData; + lfe_index = hDirACRend->hOutSetup.index_lfe[0]; + + if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + single_separated = 1; + num_objects = 1; + } + else + { + single_separated = 0; + num_objects = st_ivas->nchan_ism; + } + + offsetSamples = slots_rendered * hSpatParamRendCom->slot_size; + + for ( j = 0; j < nchan_out_woLFE + num_lfe; j++ ) + { + output_f_local[j] = output_f[j]; + } + + for ( obj = 0; obj < num_objects; obj++ ) + { + input_f[obj] = &st_ivas->hTcBuffer->tc[obj + 2][offsetSamples]; + } + + slots_to_render = nSamplesRendered / hSpatParamRendCom->slot_size; + first_sf = subframes_rendered; + last_sf = first_sf; + + while ( slots_to_render > 0 ) + { + slots_to_render -= hSpatParamRendCom->subframe_nbslots[last_sf]; + last_sf++; + } + + for ( obj = 0; obj < num_objects; obj++ ) + { + /* Delay the signal to match CLDFB delay. Delay the whole buffer with the first rendering call of the stretched buffer. */ + if ( slots_rendered == 0 ) + { + int16_t tcBufferSize; + + tcBufferSize = hSpatParamRendCom->num_slots * hSpatParamRendCom->slot_size; + delay_signal( input_f[obj], tcBufferSize, st_ivas->hMasaIsmData->delayBuffer[obj], st_ivas->hMasaIsmData->delayBuffer_size ); + } + + offsetSamples = 0; + + for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ ) + { + n_samples_sf = hSpatParamRendCom->subframe_nbslots[subframe_idx] * hSpatParamRendCom->slot_size; + if ( n_samples_sf != hRendererData->interpolator_length ) + { + for ( k = 0; k < n_samples_sf; k++ ) + { + hRendererData->interpolator[k] = (float) k / ( (float) n_samples_sf ); + } + hRendererData->interpolator_length = n_samples_sf; + } + + md_idx = hSpatParamRendCom->render_to_md_map[subframe_idx]; + + if ( single_separated ) + { + azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[md_idx]; + elevation = st_ivas->hMasaIsmData->elevation_separated_ism[md_idx]; + } + else + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism[obj][md_idx]; + elevation = st_ivas->hMasaIsmData->elevation_ism[obj][md_idx]; + } + + if ( st_ivas->hOutSetup.is_planar_setup ) + { + /* If no elevation support in output format, then rendering should be done with zero elevation */ + elevation = 0; + } + + if ( hVBAPdata != NULL ) + { + vbap_determine_gains( hVBAPdata, gains, azimuth, elevation, 1 ); + } + else + { + ivas_dirac_dec_get_response( azimuth, elevation, gains, hDirACRend->hOutSetup.ambisonics_order ); + } + + for ( j = 0; j < nchan_out_woLFE; j++ ) + { + if ( hDirACRend->hOutSetup.num_lfe > 0 ) + { + j2 = j + ( j >= lfe_index ); + } + else + { + j2 = j; + } + + if ( fabsf( gains[j] ) > 0.0f || fabsf( hRendererData->prev_gains[obj][j] ) > 0.0f ) + { + for ( k = 0; k < n_samples_sf; k++ ) + { + g1 = hRendererData->interpolator[k]; + g2 = 1.0f - g1; + output_f_local[j2][k + offsetSamples] += ( g1 * gains[j] + g2 * hRendererData->prev_gains[obj][j] ) * input_f[obj][k + offsetSamples]; + } + } + hRendererData->prev_gains[obj][j] = gains[j]; + } + + offsetSamples += n_samples_sf; + } + } + + return; +} #endif diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 2adbb5a25d98f0a659ade2a7bb879a780062063e..55af4b59059225f8e53718761ce7d4315cc48bad 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -77,7 +77,11 @@ ivas_error ivas_jbm_dec_tc( float output[MAX_TRANSPORT_CHANNELS][L_FRAME48k]; /* 'float' buffer for transport channels, MAX_TRANSPORT_CHANNELS channels */ int16_t nchan_remapped; float output_lfe_ch[L_FRAME48k]; +#ifdef MASA_AND_OBJECTS + int16_t nb_bits_metadata[MAX_SCE + 1]; +#else int16_t nb_bits_metadata[MAX_SCE]; +#endif int32_t output_Fs, ivas_total_brate; AUDIO_CONFIG output_config; ivas_error error; @@ -85,6 +89,9 @@ ivas_error ivas_jbm_dec_tc( #ifdef VLBR_20MS_MD int16_t num_md_sub_frames; #endif +#ifdef MASA_AND_OBJECTS + int32_t ism_total_brate; +#endif error = IVAS_ERR_OK; @@ -331,6 +338,82 @@ ivas_error ivas_jbm_dec_tc( #endif } } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + int16_t nchan_ism, nchan_transport_ism; + int16_t dirac_bs_md_write_idx; + + st = st_ivas->hCPE[0]->hCoreCoder[0]; + set_s( nb_bits_metadata, 0, MAX_SCE + 1 ); + + /* Set the number of objects for the parametric rendering */ + dirac_bs_md_write_idx = 0; + if ( st_ivas->hDirAC != NULL ) + { + st_ivas->hSpatParamRendCom->numIsmDirections = 0; + if ( st_ivas->ism_mode != ISM_MASA_MODE_DISC && st_ivas->ism_mode != ISM_MASA_MODE_MASA_ONE_OBJ ) + { + st_ivas->hSpatParamRendCom->numIsmDirections = st_ivas->nchan_ism; + } + + dirac_bs_md_write_idx = st_ivas->hSpatParamRendCom->dirac_bs_md_write_idx; /* Store the write-index for this frame */ + } + + /* MASA metadata decoding */ + if ( ( error = ivas_masa_decode( st_ivas, st, &nb_bits_metadata[0] ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Configuration of combined-format bit-budget distribution */ + ivas_set_surplus_brate_dec( st_ivas, &ism_total_brate ); + + st->bit_stream = &( st_ivas->bit_stream[( ism_total_brate / FRAMES_PER_SEC )] ); + + /* set ISM parameters and decode ISM metadata in OMASA format */ + if ( ( error = ivas_omasa_ism_metadata_dec( st_ivas, ism_total_brate, &nchan_ism, &nchan_transport_ism, dirac_bs_md_write_idx, &nb_bits_metadata[1] ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* decode ISM channels */ + for ( n = 0; n < nchan_transport_ism; n++ ) + { + if ( ( error = ivas_sce_dec( st_ivas, n, &output[st_ivas->nchan_transport + n], output_frame, nb_bits_metadata[1] ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + /* decode MASA channels */ + if ( ( error = ivas_cpe_dec( st_ivas, 0, output, output_frame, nb_bits_metadata[0] ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( st_ivas->hCPE[0]->nchan_out == 1 ) + { + mvr2r( output[0], output[1], output_frame ); /* Copy mono signal to stereo output channels */ + } + + /* HP filtering */ + for ( n = 0; n < getNumChanSynthesis( st_ivas ); n++ ) + { + hp20( output[n], output_frame, st_ivas->mem_hp20_out[n], output_Fs ); + } + + if ( st_ivas->renderer_type == RENDERER_MONO_DOWNMIX ) + { + ivas_mono_downmix_render_passive( st_ivas, output, output_frame ); + } + +#ifndef FIX_657_REMOVE_EDITING + /* Set edited object positions, if editing enabled */ + ivas_omasa_set_edited_objects( st_ivas ); +#endif + } +#endif else if ( st_ivas->ivas_format == MC_FORMAT ) { st = ( st_ivas->nSCE > 0 ) ? st_ivas->hSCE[0]->hCoreCoder[0] : st_ivas->hCPE[0]->hCoreCoder[0]; @@ -642,6 +725,21 @@ ivas_error ivas_jbm_dec_feed_tc_to_renderer( { ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + n_render_timeslots *= ( st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size ); + } + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + ivas_ism_dec_digest_tc( st_ivas ); + } + } +#endif else if ( st_ivas->ivas_format == MC_FORMAT ) { if ( st_ivas->mc_mode == MC_MODE_MCT ) @@ -878,6 +976,31 @@ ivas_error ivas_jbm_dec_render( ivas_sba_dec_render( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, p_output ); } } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + nchan_remapped = st_ivas->nchan_transport; + + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC ) + { + if ( ( ivas_omasa_dirac_td_binaural_jbm( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + ivas_dirac_dec_binaural_render( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output ); + } + } + else if ( st_ivas->renderer_type == RENDERER_DIRAC ) + { + ivas_omasa_dirac_rend_jbm( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output ); + } + } +#endif else if ( st_ivas->ivas_format == MC_FORMAT ) { if ( st_ivas->mc_mode == MC_MODE_MCT ) @@ -1198,6 +1321,38 @@ ivas_error ivas_jbm_dec_flush_renderer( return IVAS_ERROR( IVAS_ERR_WRONG_MODE, "Wrong MC_MODE in VoIP renderer flushing!" ); } } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT || st_ivas->ivas_format == MASA_FORMAT ) + { + if ( ism_mode_old == ISM_MASA_MODE_DISC ) + { + float *tc_local[MAX_TRANSPORT_CHANNELS]; + + for ( ch_idx = 0; ch_idx < st_ivas->nchan_ism; ch_idx++ ) + { + tc_local[ch_idx] = &st_ivas->hTcBuffer->tc[ch_idx + 2][hTcBuffer->n_samples_rendered]; + mvr2r( st_ivas->hMasaIsmData->delayBuffer[ch_idx], tc_local[ch_idx], st_ivas->hMasaIsmData->delayBuffer_size ); + } + + if ( st_ivas->nchan_ism > 0 ) + { + if ( ( ivas_td_binaural_renderer_sf( st_ivas, p_output, hTcBuffer->n_samples_granularity ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + for ( ch_idx = 0; ch_idx < st_ivas->hDecoderConfig->nchan_out; ch_idx++ ) + { + set_zero( p_output[ch_idx], (int16_t) ( *nSamplesRendered ) ); + } + st_ivas->hTcBuffer->slots_rendered += 1; + st_ivas->hTcBuffer->subframes_rendered += 1; + } + } + } +#endif else { return IVAS_ERROR( IVAS_ERR_WRONG_MODE, "Wrong IVAS format in VoIP renderer flushing!" ); @@ -1516,6 +1671,19 @@ int16_t ivas_jbm_dec_get_num_tc_channels( } } } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + num_tc++; + } + else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + num_tc += st_ivas->nchan_ism; + } + } +#endif else if ( st_ivas->ivas_format == MC_FORMAT ) { if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO ) diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index aa11b9797b29024cc829d709ac581dcffaf46fb5..5090cf8a0db92f6ff0bbc4673f6df5b5a85a8589 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -674,8 +674,21 @@ ivas_error ivas_masa_dec_open( { buffer_mode = TC_BUFFER_MODE_BUFFER; } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_MONO_DOWNMIX ) + { + buffer_mode = TC_BUFFER_MODE_BUFFER; + } +#endif nchan_to_allocate = ivas_jbm_dec_get_num_tc_channels( st_ivas ); +#ifdef MASA_AND_OBJECTS + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_MONO_DOWNMIX ) + { + nchan_to_allocate = 1; + } +#endif + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_to_allocate, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) { return error; @@ -1274,7 +1287,13 @@ static int16_t decode_lfe_to_total_energy_ratio( *-------------------------------------------------------------------*/ ivas_error ivas_masa_dec_reconfigure( +#ifdef MASA_AND_OBJECTS + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ + int16_t *data /* o : flushed PCM samples */ +#else Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +#endif ) { int16_t n, tmp, num_bits; @@ -1296,6 +1315,18 @@ ivas_error ivas_masa_dec_reconfigure( ivas_total_brate = st_ivas->hDecoderConfig->ivas_total_brate; last_ivas_total_brate = st_ivas->hDecoderConfig->last_ivas_total_brate; +#ifdef MASA_AND_OBJECTS + if ( st_ivas->hDecoderConfig->voip_active == 1 ) + { + if ( st_ivas->hSpatParamRendCom != NULL && st_ivas->hSpatParamRendCom->slot_size == st_ivas->hTcBuffer->n_samples_granularity ) + { + mvs2s( st_ivas->hSpatParamRendCom->subframe_nbslots, st_ivas->hTcBuffer->subframe_nbslots, MAX_JBM_SUBFRAMES_5MS ); + st_ivas->hTcBuffer->nb_subframes = st_ivas->hSpatParamRendCom->nb_subframes; + st_ivas->hTcBuffer->subframes_rendered = st_ivas->hSpatParamRendCom->subframes_rendered; + } + } +#endif + ivas_init_dec_get_num_cldfb_instances( st_ivas, &numCldfbAnalyses_old, &numCldfbSyntheses_old ); /* renderer might have changed, reselect */ @@ -1315,7 +1346,11 @@ ivas_error ivas_masa_dec_reconfigure( return error; } } +#ifdef MASA_AND_OBJECTS + else if ( st_ivas->renderer_type == RENDERER_DISABLE || st_ivas->renderer_type == RENDERER_MONO_DOWNMIX ) +#else else if ( st_ivas->renderer_type == RENDERER_DISABLE ) +#endif { if ( st_ivas->hDirAC != NULL ) { @@ -1497,23 +1532,84 @@ ivas_error ivas_masa_dec_reconfigure( int16_t tc_nchan_to_allocate; int16_t tc_nchan_transport; TC_BUFFER_MODE buffer_mode_new; +#ifdef MASA_AND_OBJECTS + int16_t n_samples_granularity; + n_samples_granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ); +#endif buffer_mode_new = ivas_jbm_dec_get_tc_buffer_mode( st_ivas ); tc_nchan_transport = ivas_jbm_dec_get_num_tc_channels( st_ivas ); tc_nchan_to_allocate = tc_nchan_transport; if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { +#ifdef MASA_AND_OBJECTS + if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + tc_nchan_to_allocate = 2 * BINAURAL_CHANNELS + 2; + } + else + { + tc_nchan_to_allocate = 2 * BINAURAL_CHANNELS; + } + + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + n_samples_granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); /* Use the same granularity as tdrend */ + + if ( n_samples_granularity > st_ivas->hTcBuffer->n_samples_granularity ) + { + if ( ( error = ivas_jbm_dec_set_discard_samples( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } + } + } + else if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode != ISM_MASA_MODE_DISC ) + { + if ( n_samples_granularity < st_ivas->hTcBuffer->n_samples_granularity ) + { + if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, n_samples_granularity, st_ivas->renderer_type, st_ivas->intern_config, &st_ivas->hIntSetup, MC_MODE_NONE, ISM_MASA_MODE_DISC, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + { + return error; + } + } + } +#else tc_nchan_to_allocate = 2 * BINAURAL_CHANNELS; +#endif } if ( tc_nchan_transport != st_ivas->hTcBuffer->nchan_transport_jbm || tc_nchan_to_allocate != st_ivas->hTcBuffer->nchan_transport_internal || buffer_mode_new != st_ivas->hTcBuffer->tc_buffer_mode ) { +#ifdef MASA_AND_OBJECTS + if ( ( error = ivas_jbm_dec_tc_buffer_reconfigure( st_ivas, buffer_mode_new, tc_nchan_transport, tc_nchan_to_allocate, tc_nchan_to_allocate, n_samples_granularity ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_jbm_dec_tc_buffer_reconfigure( st_ivas, buffer_mode_new, tc_nchan_transport, tc_nchan_to_allocate, tc_nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) +#endif { return error; } } + +#ifdef MASA_AND_OBJECTS + if ( st_ivas->hSpatParamRendCom != NULL && st_ivas->hSpatParamRendCom->slot_size == st_ivas->hTcBuffer->n_samples_granularity ) + { + mvs2s( st_ivas->hTcBuffer->subframe_nbslots, st_ivas->hSpatParamRendCom->subframe_nbslots, MAX_JBM_SUBFRAMES_5MS ); + st_ivas->hSpatParamRendCom->nb_subframes = st_ivas->hTcBuffer->nb_subframes; + st_ivas->hSpatParamRendCom->subframes_rendered = st_ivas->hTcBuffer->subframes_rendered; + } + + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + int16_t granularityMultiplier = st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size; + + for ( n = 0; n < MAX_JBM_SUBFRAMES_5MS; n++ ) + { + st_ivas->hSpatParamRendCom->subframe_nbslots[n] = st_ivas->hTcBuffer->subframe_nbslots[n] * granularityMultiplier; + } + } +#endif } return error; diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index b2d7f3f4b5b581fc93a71032ea1714ecc1df14bd..e8a9c58efb3c423ef2ff4423e7ce2d653a382a11 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -159,6 +159,23 @@ ivas_error ivas_td_binaural_renderer_sf( int16_t ism_md_subframe_update_jbm; int16_t c_indx, nS; +#ifdef MASA_AND_OBJECTS + int16_t nchan_ism_internal, nchan_ism, ch_offset; + + /* Set the number of ISMs */ + if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + nchan_ism_internal = st_ivas->nchan_ism; + nchan_ism = st_ivas->nchan_ism; + ch_offset = 2; + } + else + { + nchan_ism_internal = st_ivas->hTcBuffer->nchan_transport_internal; + nchan_ism = st_ivas->nchan_transport; + ch_offset = 0; + } +#endif /* Number of subframes to delay metadata to sync with audio */ if ( st_ivas->hDecoderConfig->Opt_delay_comp ) @@ -170,15 +187,29 @@ ivas_error ivas_td_binaural_renderer_sf( ism_md_subframe_update_jbm = st_ivas->hTcBuffer->nb_subframes - 2; } +#ifdef MASA_AND_OBJECTS + if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + ism_md_subframe_update_jbm = max( 0, st_ivas->hTcBuffer->nb_subframes - 4 ); /* Todo Nokia: Update this value to match the value of nonJBM rendering. */ + } +#endif + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) { p_reverb_signal[ch] = reverb_signal[ch]; } +#ifdef MASA_AND_OBJECTS + for ( ch = 0; ch < nchan_ism_internal; ch++ ) + { + tc_local[ch] = st_ivas->hTcBuffer->tc[ch + ch_offset] + st_ivas->hTcBuffer->n_samples_rendered; + } +#else for ( ch = 0; ch < st_ivas->hTcBuffer->nchan_transport_internal; ch++ ) { tc_local[ch] = st_ivas->hTcBuffer->tc[ch] + st_ivas->hTcBuffer->n_samples_rendered; } +#endif for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ ) { @@ -206,7 +237,11 @@ ivas_error ivas_td_binaural_renderer_sf( /* Update object position(s) */ c_indx = 0; +#ifdef MASA_AND_OBJECTS + for ( nS = 0; nS < nchan_ism; nS++ ) +#else for ( nS = 0; nS < st_ivas->nchan_transport; nS++ ) +#endif { if ( !( st_ivas->ivas_format == MC_FORMAT && nS == LFE_CHANNEL ) ) /* Skip LFE for MC */ { @@ -217,7 +252,11 @@ ivas_error ivas_td_binaural_renderer_sf( } if ( subframe_idx == ism_md_subframe_update_jbm ) { +#ifdef MASA_AND_OBJECTS + TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, st_ivas->hIsmMetaData ); +#else TDREND_Update_object_positions( st_ivas->hBinRendererTd, st_ivas->nchan_transport, st_ivas->ivas_format, st_ivas->hIsmMetaData ); +#endif } /* Update the listener's location/orientation */ @@ -248,7 +287,11 @@ ivas_error ivas_td_binaural_renderer_sf( } +#ifdef MASA_AND_OBJECTS + for ( ch = 0; ch < nchan_ism_internal; ch++ ) +#else for ( ch = 0; ch < st_ivas->hTcBuffer->nchan_transport_internal; ch++ ) +#endif { tc_local[ch] += output_frame; } diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index 832243883e923638fbbc29b173adfff56802127f..356bdc3ff18525176516a59caaa41d891a81d4c0 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -44,6 +44,13 @@ #ifdef MASA_AND_OBJECTS +/*------------------------------------------------------------------------- + * Local constants + *------------------------------------------------------------------------*/ + +#define OMASA_TDREND_MATCHING_GAIN 0.7943f + + /*-------------------------------------------------------------------* * ivas_omasa_data_open() * @@ -154,7 +161,9 @@ void ivas_omasa_data_close( *--------------------------------------------------------------------------*/ ivas_error ivas_omasa_dec_config( - Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ + uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ + int16_t *data /* o : flushed PCM samples */ ) { int16_t k, sce_id, nSCE_old, nchan_hp20_old, numCldfbAnalyses_old, numCldfbSyntheses_old, n_MD; @@ -195,7 +204,7 @@ ivas_error ivas_omasa_dec_config( { st_ivas->hCPE[0]->nchan_out = 1; } - else if ( ( error = ivas_masa_dec_reconfigure( st_ivas ) ) != IVAS_ERR_OK ) + else if ( ( error = ivas_masa_dec_reconfigure( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) { return error; } @@ -569,7 +578,6 @@ ivas_error ivas_omasa_ism_metadata_dec( * Rendering in OMASA format *--------------------------------------------------------------------------*/ -// Todo OMASA JBM: This might need adjustments void ivas_omasa_dirac_rend( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ float output[][L_FRAME48k], /* o : output synthesis signal */ @@ -603,6 +611,35 @@ void ivas_omasa_dirac_rend( } +/*--------------------------------------------------------------------------* + * ivas_omasa_dirac_rend_jbm() + * + * Rendering in OMASA format for JBM + *--------------------------------------------------------------------------*/ + +void ivas_omasa_dirac_rend_jbm( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of samples requested */ + uint16_t *nSamplesRendered, /* o : number of samples rendered */ + uint16_t *nSamplesAvailable, /* o : number of samples still to render */ + const int16_t nchan_transport, /* i : number of transport channels */ + float *output_f[] /* o : rendered time signal */ +) +{ + int16_t subframes_rendered; + int16_t slots_rendered; + + subframes_rendered = st_ivas->hSpatParamRendCom->subframes_rendered; + slots_rendered = st_ivas->hSpatParamRendCom->slots_rendered; + + ivas_dirac_dec_render( st_ivas, nchan_transport, nSamplesAsked, nSamplesRendered, nSamplesAvailable, output_f ); + + ivas_omasa_separate_object_render_jbm( st_ivas, *nSamplesRendered, output_f, subframes_rendered, slots_rendered ); + + return; +} + + /*--------------------------------------------------------------------------* * ivas_omasa_dirac_td_binaural() * @@ -617,7 +654,7 @@ ivas_error ivas_omasa_dirac_td_binaural( { int16_t n; float data_separated_objects[MAX_NUM_OBJECTS][L_FRAME48k]; - float gain = 0.7943f; /* Todo Nokia: Temporary gain for roughly matching the loudness of other processing paths. */ + float gain = OMASA_TDREND_MATCHING_GAIN; ivas_error error; float *p_sepobj[MAX_NUM_OBJECTS]; @@ -651,4 +688,62 @@ ivas_error ivas_omasa_dirac_td_binaural( return IVAS_ERR_OK; } + + +/*--------------------------------------------------------------------------* + * ivas_omasa_dirac_td_binaural_render() + * + * Binaural rendering in OMASA format for JBM + *--------------------------------------------------------------------------*/ + +ivas_error ivas_omasa_dirac_td_binaural_jbm( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of samples requested */ + uint16_t *nSamplesRendered, /* o : number of samples rendered */ + uint16_t *nSamplesAvailable, /* o : number of samples still to render */ + const int16_t nchan_transport, /* i : number of transport channels */ + float *output_f[] /* o : rendered time signal */ +) +{ + int16_t n; + float data_separated_objects[MAX_NUM_OBJECTS][L_FRAME48k]; + float gain = OMASA_TDREND_MATCHING_GAIN; + ivas_error error; + float *p_sepobj[MAX_NUM_OBJECTS]; + float *tc_local[MAX_TRANSPORT_CHANNELS]; + + for ( n = 0; n < MAX_NUM_OBJECTS; n++ ) + { + p_sepobj[n] = &data_separated_objects[n][0]; + } + + /* Delay the object signals to match the CLDFB delay. Delay the whole buffer with the first rendering call of the stretched buffer. */ + if ( st_ivas->hSpatParamRendCom->slots_rendered == 0 ) + { + int16_t tcBufferSize; + + tcBufferSize = st_ivas->hSpatParamRendCom->num_slots * st_ivas->hSpatParamRendCom->slot_size; + + for ( n = 0; n < st_ivas->nchan_ism; n++ ) + { + tc_local[n] = st_ivas->hTcBuffer->tc[n + 2]; + v_multc( tc_local[n], gain, tc_local[n], tcBufferSize ); + delay_signal( tc_local[n], tcBufferSize, st_ivas->hMasaIsmData->delayBuffer[n], st_ivas->hMasaIsmData->delayBuffer_size ); + } + } + + ivas_dirac_dec_binaural_render( st_ivas, nSamplesAsked, nSamplesRendered, nSamplesAvailable, nchan_transport, output_f ); + + if ( ( error = ivas_td_binaural_renderer_sf( st_ivas, p_sepobj, *nSamplesRendered ) ) != IVAS_ERR_OK ) + { + return error; + } + + for ( n = 0; n < BINAURAL_CHANNELS; n++ ) + { + v_add( output_f[n], p_sepobj[n], output_f[n], *nSamplesRendered ); + } + + return IVAS_ERR_OK; +} #endif diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index c4212bd0d0960901c732d83add31b710c1307d0a..7848e09fe066693b3bd3ac850a841fa26e1b4d68 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -777,6 +777,9 @@ typedef struct renderer_struct { float prev_gains[MAX_CICP_CHANNELS - 1][MAX_OUTPUT_CHANNELS]; float *interpolator; +#ifdef MASA_AND_OBJECTS + int16_t interpolator_length; +#endif float gains[MAX_CICP_CHANNELS - 1][MAX_OUTPUT_CHANNELS]; } ISM_RENDERER_DATA, *ISM_RENDERER_HANDLE; diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index c04ff82cbf7cdf9d87f13a51fde1987e20c817bb..35ad9794872c2633879bf23f666509d595f8f02b 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -124,7 +124,7 @@ static void ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matric #endif #ifdef MASA_AND_OBJECTS -static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t isHeadtracked, const int16_t nchanSeparateChannels, const MASA_ISM_DATA_HANDLE hMasaIsmData ); +static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t subframe, const int16_t isHeadtracked, const int16_t nchanSeparateChannels, const MASA_ISM_DATA_HANDLE hMasaIsmData ); #else static void ivas_dirac_dec_binaural_determine_processing_matrices( DIRAC_DEC_BIN_HANDLE hDiracDecBin, SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom, PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], const int16_t isHeadtracked ); #endif @@ -342,9 +342,27 @@ ivas_error ivas_dirac_dec_init_binaural_data( if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) { int16_t nchan_to_allocate; +#ifdef MASA_AND_OBJECTS + int16_t n_samples_granularity; +#endif nchan_to_allocate = 2 * BINAURAL_CHANNELS; +#ifdef MASA_AND_OBJECTS + if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + nchan_to_allocate = 2 * BINAURAL_CHANNELS + 2; + } + + n_samples_granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ); + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + n_samples_granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); /* Use the same granularity as tdrend */ + } + + if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, ivas_jbm_dec_get_num_tc_channels( st_ivas ), nchan_to_allocate, nchan_to_allocate, n_samples_granularity ) ) != IVAS_ERR_OK ) +#else if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, ivas_jbm_dec_get_num_tc_channels( st_ivas ), nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) +#endif { return error; } @@ -953,7 +971,7 @@ static void ivas_dirac_dec_binaural_internal( { nchanSeparateChannels = (uint8_t) st_ivas->nchan_ism; } - ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, + ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, nchanSeparateChannels, st_ivas->hMasaIsmData ); #else @@ -1034,7 +1052,7 @@ static void ivas_dirac_dec_binaural_internal( subFrameTotalEne, IIReneLimiter, st_ivas->hMasaIsmData ); - ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, + ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, subframe, hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, nchanSeparateChannels, st_ivas->hMasaIsmData ); #else @@ -2226,6 +2244,9 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( PARAMBIN_REND_CONFIG_HANDLE hConfig, const int16_t max_band_decorr, float Rmat[3][3], +#ifdef MASA_AND_OBJECTS + const int16_t subframe, +#endif const int16_t isHeadtracked #ifdef MASA_AND_OBJECTS , @@ -2266,8 +2287,7 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( #ifdef MASA_AND_OBJECTS ism_mode = hConfig->ism_mode; - // Todo OMASA JBM: This is probably not correct now. We should get the index from JBM - dirac_read_idx = hSpatParamRendCom->dirac_read_idx; + dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; #endif #ifdef MASA_AND_OBJECTS