diff --git a/apps/decoder.c b/apps/decoder.c index 1b6b7e5d57614343574b70e43718c0ba3877eb19..5eb6d9f16ff3ccc62ad84092f83a79a877ebf7a7 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -54,6 +54,11 @@ #include "debug.h" #endif #include "wmc_auto.h" +#ifdef OBJ_EDITING_API +#ifdef OBJ_EDITING_EXAMPLE +#include +#endif +#endif #define WMC_TOOL_SKIP @@ -150,6 +155,9 @@ typedef struct AcousticEnvironmentSequence aeSequence; bool dpidEnabled; uint16_t directivityPatternId[IVAS_MAX_NUM_OBJECTS]; +#ifdef OBJ_EDITING_EXAMPLE + bool objEditEnabled; +#endif } DecArguments; @@ -1091,6 +1099,12 @@ static bool parseCmdlIVAS_dec( arg->directivityPatternId[i] = 65535; } +#ifdef OBJ_EDITING_EXAMPLE + arg->objEditEnabled = false; +#endif +# + + /*-----------------------------------------------------------------* * Initialization *-----------------------------------------------------------------*/ @@ -1544,6 +1558,14 @@ static bool parseCmdlIVAS_dec( i += tmp; } +#ifdef OBJ_EDITING_EXAMPLE + else if ( strcmp( argv_to_upper, "-OBJ_EDIT" ) == 0 ) + { + arg->objEditEnabled = true; + i++; + } +#endif + /*-----------------------------------------------------------------* * Option not recognized *-----------------------------------------------------------------*/ @@ -1748,6 +1770,9 @@ static void usage_dec( void ) fprintf( stdout, " Currently, all values default to level 3 (full functionality).\n" ); fprintf( stdout, "-q : Quiet mode, no frame counter\n" ); fprintf( stdout, " default is deactivated\n" ); +#ifdef OMASA_OBJECT_EDITING + fprintf( stdout, "-obj_edit : enable object editing\n" ); +#endif #ifdef DEBUG_MODE_INFO #ifdef DEBUG_MODE_INFO_TWEAK fprintf( stdout, "-info : specify subfolder name for debug output\n" ); @@ -2115,6 +2140,10 @@ static ivas_error decodeG192( return error; } +#ifdef OBJ_EDITING_API + IVAS_EDITABLE_PARAMETERS editableParameters; +#endif + IVAS_RENDER_CONFIG_DATA renderConfig; RenderConfigReader *renderConfigReader = NULL; @@ -2416,11 +2445,84 @@ static ivas_error decodeG192( } #endif /* Feed into decoder */ - if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, bit_stream, num_bits, bfi ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, bit_stream, num_bits, bfi +#ifdef OBJ_EDITING_API + , + isSplitRend, + splitRendBits +#endif + ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError: could not feed frame to decoder: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } +#ifdef OBJ_EDITING_API +#ifdef OBJ_EDITING_EXAMPLE + if ( arg.objEditEnabled ) + { + + /* Do object info editing here */ + /* get object parameters */ + if ( ( error = IVAS_DEC_GetEditableParameters( hIvasDec, &editableParameters ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: could not get the editable parameters: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + /* edit object parameters...*/ + + /* put the objects equally spaced at the horizontal plane */ + /* and play a little bit with the gains... */ + int16_t obj_idx, non_diegetic_obj_idx; + int16_t num_nondiegetic_objects; + + num_nondiegetic_objects = 0; + for ( obj_idx = 0; obj_idx < editableParameters.num_obj; obj_idx++ ) + { + if ( !editableParameters.ism_metadata[obj_idx].non_diegetic_flag ) + { + num_nondiegetic_objects++; + } + } + if ( num_nondiegetic_objects ) + { + float start_angle, angle_inc; + angle_inc = 360.0f / (float) num_nondiegetic_objects; + start_angle = angle_inc / 2.0f; + for ( obj_idx = 0, non_diegetic_obj_idx = 0; obj_idx < editableParameters.num_obj; obj_idx++ ) + { + if ( !editableParameters.ism_metadata[obj_idx].non_diegetic_flag ) + { + editableParameters.ism_metadata[obj_idx].elevation = 0.0f; + editableParameters.ism_metadata[obj_idx].azimuth = start_angle + (float) non_diegetic_obj_idx * angle_inc; + non_diegetic_obj_idx++; + } + } + } + + /* breakover object gains */ + for ( obj_idx = 0; obj_idx < editableParameters.num_obj; obj_idx++ ) + { + editableParameters.ism_metadata[obj_idx].gain = 0.5f + (float) ( ( frame + obj_idx * 50 ) % 250 ) / 250.0f; + } + + editableParameters.gain_bed = 0.5f; + + /* set new object parameters*/ + if ( ( error = IVAS_DEC_SetEditableParameters( hIvasDec, editableParameters ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: could not set the editable parameters: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } +#endif + /* Do the final preparations needed for rendering */ + if ( ( error = IVAS_DEC_PrepareRenderer( hIvasDec ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: could not prepare the renderer: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } +#endif } if ( isSplitRend ) @@ -2970,6 +3072,11 @@ static ivas_error decodeVoIP( int16_t vec_pos_update, vec_pos_len; int16_t nOutSamples = 0; +#ifdef OBJ_EDITING_API + bool parameterAvailableForEditing = false; + uint16_t nSamplesRendered = 0; +#endif + vec_pos_update = 0; if ( ( error = IVAS_DEC_GetRenderFramesizeMs( hIvasDec, &systemTimeInc_ms ) ) != IVAS_ERR_OK ) { @@ -3086,6 +3193,9 @@ static ivas_error decodeVoIP( while ( 1 ) { +#ifdef OBJ_EDITING_API + nSamplesRendered = 0; +#endif /* reference vector */ if ( arg.enableReferenceVectorTracking && vec_pos_update == 0 ) @@ -3248,15 +3358,37 @@ static ivas_error decodeVoIP( /* decode and get samples */ +#ifdef OBJ_EDITING_API + while ( nSamplesRendered < nOutSamples ) + { +#endif + #ifdef SUPPORT_JBM_TRACEFILE - if ( ( error = IVAS_DEC_VoIP_GetSamples( hIvasDec, nOutSamples, IVAS_DEC_PCM_INT16, (void *) pcmBuf, systemTime_ms, writeJbmTraceFileFrameWrapper, jbmTraceWriter ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_DEC_VoIP_GetSamples( hIvasDec, nOutSamples, IVAS_DEC_PCM_INT16, (void *) pcmBuf, systemTime_ms, writeJbmTraceFileFrameWrapper, jbmTraceWriter +#ifdef OBJ_EDITING_API + , + &nSamplesRendered, ¶meterAvailableForEditing +#endif + ) ) != IVAS_ERR_OK ) #else - if ( ( error = IVAS_DEC_VoIP_GetSamples( hIvasDec, nOutSamples, IVAS_DEC_PCM_INT16, (void *) pcmBuf, systemTime_ms ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_DEC_VoIP_GetSamples( hIvasDec, nOutSamples, IVAS_DEC_PCM_INT16, (void *) pcmBuf, systemTime_ms, &nSamplesRendered, ¶meterAvailableForEditing +#ifdef OBJ_EDITING_API + & nSamplesRendered, + ¶meterAvailableForEditing +#endif + ) ) != IVAS_ERR_OK ) +#endif + { + fprintf( stderr, "\nError in IVAS_DEC_VoIP_GetSamples: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } +#ifdef OBJ_EDITING_API + if ( parameterAvailableForEditing == true ) + { + /* do the object editing here */ + } + } /* while ( nSamplesRendered < nOutSamples ) */ #endif - { - fprintf( stderr, "\nError in IVAS_DEC_VoIP_GetSamples: %s\n", IVAS_DEC_GetErrorMessage( error ) ); - goto cleanup; - } /* write JBM Offset file entry */ if ( jbmOffsetWriter != NULL ) diff --git a/lib_com/common_api_types.h b/lib_com/common_api_types.h index 279e00b986243a795ce04cdc80cf1f852fdd4601..8cbdfcb3cd362f76c6514b67a6c10d9653ca76b9 100644 --- a/lib_com/common_api_types.h +++ b/lib_com/common_api_types.h @@ -128,9 +128,21 @@ typedef struct _IVAS_ISM_METADATA float yaw; float pitch; int16_t non_diegetic_flag; +#ifdef OBJ_EDITING_API + float gain; +#endif } IVAS_ISM_METADATA; +#ifdef OBJ_EDITING_API +typedef struct _IVAS_EDITABLE_PARAMETERS +{ + int16_t num_obj; + IVAS_ISM_METADATA ism_metadata[IVAS_MAX_NUM_OBJECTS]; + float gain_bed; +} IVAS_EDITABLE_PARAMETERS; +#endif + typedef struct { float w, x, y, z; diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index 8c72cf280832915c449e188d673effd07a338a6c..4e5d9d8ae5a910470368eab79e75a77b20fa329f 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -1215,6 +1215,11 @@ enum #define MASA_MAXIMUM_TWO_DIR_BANDS 24 #define NBITS_HR_COH 4 #define OMASA_TDREND_MATCHING_GAIN 0.7943f +#ifdef OMASA_OBJECT_EDITING +#define OMASA_GAIN_EDIT_THR 0.06f /* OMASA gain change threshold */ +#define OMASA_AZI_EDIT_THR 1.0f /* OMASA-DISC azimuth change threshold */ +#define OMASA_ELE_EDIT_THR 2.0f /* OMASA-DISC elevation change threshold */ +#endif #define MASA_JBM_RINGBUFFER_FRAMES 3 diff --git a/lib_com/ivas_ism_com.c b/lib_com/ivas_ism_com.c index 55f6016fcf5f36c6d6d25fbea386fd39351410a0..99131d5e0f3a10f7c585bc8c49d6cddd9a6d3357 100644 --- a/lib_com/ivas_ism_com.c +++ b/lib_com/ivas_ism_com.c @@ -418,6 +418,17 @@ void ivas_ism_reset_metadata( hIsmMeta->ism_metadata_flag = 0; hIsmMeta->non_diegetic_flag = 0; +#ifdef OBJ_EDITING_API + hIsmMeta->edited_gain = 1.0f; + hIsmMeta->edited_azimuth = 0.0f; + hIsmMeta->edited_elevation = 0.0f; + hIsmMeta->edited_pitch = 0.0f; + hIsmMeta->edited_yaw = 0.0f; + hIsmMeta->edited_radius = 1.0f; + hIsmMeta->gain = 1.0f; + hIsmMeta->non_diegetic_flag = 0; +#endif + return; } diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index db43d4fe2f90d9d3c66ffd525c5259bec898c9d5..da1f94daed5a69eaa5f80a59373a24e12a5611e9 100755 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -809,6 +809,12 @@ void ivas_jbm_dec_feed_tc_to_renderer( float *data /* i/o: transport channels/output synthesis signal */ ); +#ifdef OBJ_EDITING_API +void ivas_jbm_dec_prepare_renderer( + Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +); +#endif + ivas_error ivas_jbm_dec_set_discard_samples( Decoder_Struct *st_ivas /* i/o: main IVAS decoder structre */ ); @@ -1018,6 +1024,11 @@ ivas_error ivas_ism_metadata_dec( DEC_CORE_HANDLE st0 /* i : core-coder handle */ ); +#ifdef OBJ_EDITING_API +void ivas_ism_renderer_update_md( + Decoder_Struct *st_ivas /* i/o: main IVAS decoder handle */ +); +#endif /*----------------------------------------------------------------------------------* * Parametric ISM prototypes @@ -1092,6 +1103,17 @@ void ivas_param_ism_dec_digest_tc( float *transport_channels_f[] /* i : synthesized core-coder transport channels/DirAC output */ ); +#ifdef OBJ_EDITING_API +void ivas_param_ism_dec_dequant_md( + Decoder_Struct *st_ivas +); + +void ivas_param_ism_dec_prepare_renderer( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ +); +#endif + void ivas_ism_param_dec_tc_gain_ajust( Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ const uint16_t nSamples, /* i : number of samples to be compensate */ @@ -3845,6 +3867,13 @@ void ivas_param_mc_dec_digest_tc( float *transport_channels_f[] /* i : synthesized core-coder transport channels/DirAC output*/ ); +#ifdef OBJ_EDITING_API +void ivas_param_mc_dec_prepare_renderer( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ +); +#endif + void ivas_param_mc_dec_render( Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ @@ -5775,7 +5804,11 @@ void ivas_omasa_dirac_rend_jbm( float *output_f[] /* o : rendered time signal */ ); +#ifdef OMASA_OBJECT_EDITING +void ivas_omasa_preProcessStereoTransportsForEditedObjects( +#else void ivas_omasa_preProcessStereoTransportsForMovedObjects( +#endif Decoder_Struct *st_ivas, float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], diff --git a/lib_com/ivas_stat_com.h b/lib_com/ivas_stat_com.h index 23645a77dc88154235066ef245a229ee81eeebb9..594e30ff26c77055da92f6e6ec598c14384a7400 100644 --- a/lib_com/ivas_stat_com.h +++ b/lib_com/ivas_stat_com.h @@ -65,6 +65,18 @@ typedef struct float yaw; /* yaw value read from the input metadata file */ float pitch; /* pitch value read from the input metadata file */ +#ifdef OBJ_EDITING_API + float gain; + + float edited_azimuth; + float edited_elevation; + float edited_radius; + + float edited_yaw; + float edited_pitch; + float edited_gain; +#endif + int16_t non_diegetic_flag; /* Non-diegetic (non-headtracked) object flag */ ISM_METADATA_ANGLE position_angle; /* Angle structs for azimuth and elevation */ diff --git a/lib_com/options.h b/lib_com/options.h index d9eb13e60edbe0c58866fb62d8801ef45d4028ca..f768a074b028e73fb958b93bccf0c6df5bff1b3f 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -154,6 +154,16 @@ /* ################## Start DEVELOPMENT switches ######################### */ +#define OBJ_EDITING_INTERFACE /* Interface for object editing */ +#ifdef OBJ_EDITING_INTERFACE +#define OBJ_EDITING_API /* object editing changes related to the API */ +#define OBJ_EDITING_EXAMPLE /* obj editing example code in decoder.c */ +#endif + +#define OMASA_OBJECT_EDITING /* Nokia: object editing interface for OMASA */ +#define OBJ_EDITING_PARAMISM_BIN /* Nokia: object editing for ParamISM to binaural */ + + /* ################### Start BE switches ################################# */ /* only BE switches wrt selection floating point code */ diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index 7afe0de9f6c65a31a47fec72b3fff9f1dccbcf87..f79bd86ad4acde0e38a879acd91b01999d1aba1e 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -1839,10 +1839,17 @@ void ivas_dirac_dec_render_sf( } } +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + ivas_omasa_preProcessStereoTransportsForEditedObjects( st_ivas, Cldfb_RealBuffer_Temp, Cldfb_ImagBuffer_Temp, hSpatParamRendCom->num_freq_bands, subframe_idx ); + } +#else if ( st_ivas->ism_mode != ISM_MASA_MODE_DISC && st_ivas->ism_mode != ISM_MASA_MODE_MASA_ONE_OBJ ) { ivas_omasa_preProcessStereoTransportsForMovedObjects( st_ivas, Cldfb_RealBuffer_Temp, Cldfb_ImagBuffer_Temp, hSpatParamRendCom->num_freq_bands, subframe_idx ); } +#endif } for ( slot_idx = 0; slot_idx < hSpatParamRendCom->subframe_nbslots[subframe_idx]; slot_idx++ ) @@ -2269,6 +2276,9 @@ void ivas_dirac_dec_render_sf( if ( st_ivas->hEFAPdata != NULL ) { efap_determine_gains( st_ivas->hEFAPdata, st_ivas->hIsmRendererData->gains[i], az1, el1, EFAP_MODE_EFAP ); +#ifdef OBJ_EDITING_API + v_multc( st_ivas->hIsmRendererData->gains[i], st_ivas->hIsmMetaData[i]->edited_gain, st_ivas->hIsmRendererData->gains[i], nchan_out_woLFE ); +#endif } } diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 907f5619ab0235d7f0684355f3221013fdf76e50..3930c3b2c1ce17a8fde3c3b81fce7a620cc517c5 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -1136,6 +1136,17 @@ ivas_error ivas_init_decoder( if ( st_ivas->ism_mode == ISM_MODE_PARAM ) { st_ivas->hSCE[0]->hCoreCoder[0]->hFdCngDec->hFdCngCom->seed2 = st_ivas->hSCE[0]->hCoreCoder[0]->hFdCngDec->hFdCngCom->seed3; + +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM ) + { + /* reusing OMASA function for allocating and initializing MASA_ISM rendering handle (even though not in OMASA) */ + if ( ( error = ivas_omasa_data_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif } else if ( st_ivas->ism_mode == ISM_MODE_DISC ) { @@ -1453,12 +1464,15 @@ ivas_error ivas_init_decoder( reset_indices_dec( st_ivas->hSCE[0]->hCoreCoder[0] ); +#ifndef OMASA_OBJECT_EDITING if ( hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) { +#endif if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) { return error; } +#ifndef OMASA_OBJECT_EDITING } else { @@ -1467,6 +1481,7 @@ ivas_error ivas_init_decoder( return error; } } +#endif } else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) { @@ -1890,7 +1905,8 @@ ivas_error ivas_init_decoder( } } - if ( st_ivas->renderer_type == RENDERER_DIRAC && ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) + if ( ( st_ivas->renderer_type == RENDERER_DIRAC ) && + ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) { /* Allocate 'hIsmRendererData' handle and memory for delay buffer within 'hMasaIsmData' */ if ( ( error = ivas_omasa_separate_object_renderer_open( st_ivas ) ) != IVAS_ERR_OK ) diff --git a/lib_dec/ivas_ism_dec.c b/lib_dec/ivas_ism_dec.c index 4f5dc179dd06a36d4d677c3c7ada2108629a6d7a..36945f81550f347f6e8e3813bd2456fca27036bb 100644 --- a/lib_dec/ivas_ism_dec.c +++ b/lib_dec/ivas_ism_dec.c @@ -177,6 +177,10 @@ static ivas_error ivas_ism_bitrate_switching_dec( { /* close the parametric binaural renderer */ ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#ifdef OBJ_EDITING_PARAMISM_BIN + /* Close omasa data struct (used for object editing) */ + ivas_omasa_data_close( &st_ivas->hMasaIsmData ); +#endif /* Open the TD Binaural renderer */ if ( st_ivas->hHrtfTD == NULL || st_ivas->hBinRendererTd == NULL ) { @@ -209,6 +213,11 @@ static ivas_error ivas_ism_bitrate_switching_dec( /* close the parametric binaural renderer */ ivas_dirac_dec_close_binaural_data( st_ivas->hDiracDecBin ); +#ifdef OBJ_EDITING_PARAMISM_BIN + /* Close omasa data struct (used for object editing) */ + ivas_omasa_data_close( &st_ivas->hMasaIsmData ); +#endif + /* Open Crend Binaural renderer */ if ( ( error = ivas_rend_openCrend( &( st_ivas->hCrendWrapper ), st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hRenderConfig, st_ivas->hSetOfHRTF, st_ivas->hHrtfStatistics, st_ivas->hDecoderConfig->output_Fs, ( st_ivas->hSplitBinRend == NULL ) ? 1 : st_ivas->hSplitBinRend->splitrend.multiBinPoseData.num_poses ) ) != IVAS_ERR_OK ) { @@ -241,6 +250,14 @@ static ivas_error ivas_ism_bitrate_switching_dec( return error; } +#ifdef OBJ_EDITING_PARAMISM_BIN + /* Open omasa data struct (used for object editing) */ + if ( ( error = ivas_omasa_data_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif + /* Close the TD Binaural renderer */ ivas_td_binaural_close( &st_ivas->hBinRendererTd ); @@ -268,6 +285,14 @@ static ivas_error ivas_ism_bitrate_switching_dec( return error; } +#ifdef OBJ_EDITING_PARAMISM_BIN + /* Open omasa data struct (used for object editing) */ + if ( ( error = ivas_omasa_data_open( st_ivas ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif + /* close the crend binaural renderer */ ivas_rend_closeCrend( &( st_ivas->hCrendWrapper ), ( st_ivas->hSplitBinRend == NULL ) ? 1 : st_ivas->hSplitBinRend->splitrend.multiBinPoseData.num_poses ); } diff --git a/lib_dec/ivas_ism_param_dec.c b/lib_dec/ivas_ism_param_dec.c index 534f4971d6ea6acb98d0df9d687362a0909eda07..b4a37dddf609f22008d0f6c241407b05f37f3263 100644 --- a/lib_dec/ivas_ism_param_dec.c +++ b/lib_dec/ivas_ism_param_dec.c @@ -746,13 +746,23 @@ void ivas_ism_dec_digest_tc( if ( st_ivas->intern_config == IVAS_AUDIO_CONFIG_STEREO ) { +#ifdef OBJ_EDITING_API + ivas_ism_get_stereo_gains( st_ivas->hIsmMetaData[i]->edited_azimuth, st_ivas->hIsmMetaData[i]->edited_elevation, &st_ivas->hIsmRendererData->gains[i][0], &st_ivas->hIsmRendererData->gains[i][1] ); + v_multc( st_ivas->hIsmRendererData->gains[i], st_ivas->hIsmMetaData[i]->edited_gain, st_ivas->hIsmRendererData->gains[i], CPE_CHANNELS ); +#else ivas_ism_get_stereo_gains( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &st_ivas->hIsmRendererData->gains[i][0], &st_ivas->hIsmRendererData->gains[i][1] ); +#endif } else { // TODO tmu review when #215 is resolved +#ifdef OBJ_EDITING_API + azimuth = (int16_t) floorf( st_ivas->hIsmMetaData[i]->edited_azimuth + 0.5f ); + elevation = (int16_t) floorf( st_ivas->hIsmMetaData[i]->edited_elevation + 0.5f ); +#else azimuth = (int16_t) floorf( st_ivas->hIsmMetaData[i]->azimuth + 0.5f ); elevation = (int16_t) floorf( st_ivas->hIsmMetaData[i]->elevation + 0.5f ); +#endif if ( ( st_ivas->renderer_type == RENDERER_TD_PANNING || st_ivas->renderer_type == RENDERER_OSBA_LS || @@ -772,6 +782,9 @@ void ivas_ism_dec_digest_tc( if ( st_ivas->hEFAPdata != NULL ) { efap_determine_gains( st_ivas->hEFAPdata, st_ivas->hIsmRendererData->gains[i], azimuth, elevation, EFAP_MODE_EFAP ); +#ifdef OBJ_EDITING_API + v_multc( st_ivas->hIsmRendererData->gains[i], st_ivas->hIsmMetaData[i]->edited_gain, st_ivas->hIsmRendererData->gains[i], st_ivas->hEFAPdata->numSpk ); +#endif } } else if ( st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC || @@ -780,6 +793,9 @@ void ivas_ism_dec_digest_tc( { /*get HOA gets for direction (ACN/SN3D)*/ ivas_dirac_dec_get_response( azimuth, elevation, st_ivas->hIsmRendererData->gains[i], st_ivas->hIntSetup.ambisonics_order ); +#ifdef OBJ_EDITING_API + v_multc( st_ivas->hIsmRendererData->gains[i], st_ivas->hIsmMetaData[i]->edited_gain, st_ivas->hIsmRendererData->gains[i], ivas_sba_get_nchan( st_ivas->hIntSetup.ambisonics_order, 0 ) ); +#endif } } } @@ -801,14 +817,21 @@ void ivas_param_ism_dec_digest_tc( float *transport_channels_f[] /* i : synthesized core-coder transport channels/DirAC output */ ) { +#ifndef OBJ_EDITING_API int16_t ch, nchan_transport, nchan_out, nchan_out_woLFE, i; int16_t slot_idx, bin_idx; int32_t ivas_total_brate; +#else + int16_t ch, nchan_transport; + int16_t slot_idx; +#endif int16_t output_frame; +#ifndef OBJ_EDITING_API float ref_power[CLDFB_NO_CHANNELS_MAX]; float cx_diag[CLDFB_NO_CHANNELS_MAX][PARAM_ISM_MAX_DMX]; /* Direct Response/EFAP Gains */ float direct_response[MAX_NUM_OBJECTS][PARAM_ISM_MAX_CHAN]; +#endif PARAM_ISM_DEC_HANDLE hParamIsmDec; SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; int16_t fade_len; @@ -822,6 +845,8 @@ void ivas_param_ism_dec_digest_tc( fade_len = output_frame / 2; nchan_transport = st_ivas->nchan_transport; + +#ifndef OBJ_EDITING_API ivas_total_brate = st_ivas->hDecoderConfig->ivas_total_brate; if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) @@ -835,9 +860,11 @@ void ivas_param_ism_dec_digest_tc( nchan_out = st_ivas->hIntSetup.nchan_out_woLFE + st_ivas->hIntSetup.num_lfe; nchan_out_woLFE = st_ivas->hIntSetup.nchan_out_woLFE; } +#endif push_wmops( "ivas_param_ism_dec_digest_tc" ); +#ifndef OBJ_EDITING_API /* general setup */ ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hParamIsmDec->hParamIsmRendering->interpolator ); @@ -912,6 +939,7 @@ void ivas_param_ism_dec_digest_tc( } } } +#endif if ( st_ivas->hDecoderConfig->Opt_tsm ) { @@ -935,10 +963,13 @@ void ivas_param_ism_dec_digest_tc( mvr2r( ImagBuffer, &hParamIsmDec->hParamIsmRendering->Cldfb_ImagBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], hSpatParamRendCom->num_freq_bands ); } +#ifndef OBJ_EDITING_API ivas_param_ism_collect_slot( hParamIsmDec, &hParamIsmDec->hParamIsmRendering->Cldfb_RealBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], &hParamIsmDec->hParamIsmRendering->Cldfb_ImagBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], ch, ref_power, cx_diag ); +#endif } } +#ifndef OBJ_EDITING_API /* Obtain Mixing Matrix on a frame-level */ for ( bin_idx = 0; bin_idx < hSpatParamRendCom->num_freq_bands; bin_idx++ ) { @@ -947,6 +978,7 @@ void ivas_param_ism_dec_digest_tc( /* Compute mixing matrix */ ivas_param_ism_compute_mixing_matrix( st_ivas->nchan_ism, hParamIsmDec, st_ivas->hISMDTX, direct_response, nchan_transport, nchan_out_woLFE, cx_diag, ref_power, hParamIsmDec->hParamIsmRendering->mixing_matrix_lin ); +#endif pop_wmops(); @@ -954,6 +986,164 @@ void ivas_param_ism_dec_digest_tc( } +#ifdef OBJ_EDITING_API +/*-------------------------------------------------------------------------* + * ivas_param_ism_dec_prepare_renderer() + * + * + *-------------------------------------------------------------------------*/ + +void ivas_param_ism_dec_dequant_md( + Decoder_Struct *st_ivas ) +{ + /* De-quantization */ + if ( !( st_ivas->hDecoderConfig->ivas_total_brate == IVAS_SID_5k2 || st_ivas->hDecoderConfig->ivas_total_brate == FRAME_NO_DATA ) ) + { + ivas_param_ism_dec_dequant_DOA( st_ivas->hParamIsmDec, st_ivas->nchan_ism ); + ivas_param_ism_dec_dequant_powrat( st_ivas->hParamIsmDec ); + st_ivas->hISMDTX.dtx_flag = 0; + } + else + { + st_ivas->hISMDTX.dtx_flag = 1; + } + + return; +} + + +/*-------------------------------------------------------------------------* + * ivas_param_ism_dec_prepare_renderer() + * + * + *-------------------------------------------------------------------------*/ + +void ivas_param_ism_dec_prepare_renderer( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ +) +{ + int16_t ch, nchan_transport, nchan_out, nchan_out_woLFE, i; + int16_t slot_idx, bin_idx; + float ref_power[CLDFB_NO_CHANNELS_MAX]; + float cx_diag[CLDFB_NO_CHANNELS_MAX][PARAM_ISM_MAX_DMX]; + /* Direct Response/EFAP Gains */ + float direct_response[MAX_NUM_OBJECTS][PARAM_ISM_MAX_CHAN]; + PARAM_ISM_DEC_HANDLE hParamIsmDec; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + + /* Initialization */ + hParamIsmDec = st_ivas->hParamIsmDec; + assert( hParamIsmDec ); + hSpatParamRendCom = st_ivas->hSpatParamRendCom; + assert( hSpatParamRendCom ); + + nchan_transport = st_ivas->nchan_transport; + + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) + { + nchan_out = st_ivas->nchan_ism; + nchan_out_woLFE = nchan_out; + st_ivas->hDecoderConfig->nchan_out = nchan_out; + } + else + { + nchan_out = st_ivas->hIntSetup.nchan_out_woLFE + st_ivas->hIntSetup.num_lfe; + nchan_out_woLFE = st_ivas->hIntSetup.nchan_out_woLFE; + } + + push_wmops( "ivas_param_ism_dec_digest_tc" ); + + /* general setup */ + ivas_jbm_dec_get_adapted_linear_interpolator( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbSlots, hParamIsmDec->hParamIsmRendering->interpolator ); + + ivas_dirac_dec_set_md_map( st_ivas, nCldfbSlots ); + + /* set buffers to zero */ + for ( bin_idx = 0; bin_idx < CLDFB_NO_CHANNELS_MAX; bin_idx++ ) + { + set_zero( cx_diag[bin_idx], PARAM_ISM_MAX_DMX ); + } + set_zero( ref_power, CLDFB_NO_CHANNELS_MAX ); + + /* obtain the direct response using EFAP */ + if ( !( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) ) + { + for ( i = 0; i < st_ivas->nchan_ism; i++ ) + { +#ifdef OBJ_EDITING_API + efap_determine_gains( st_ivas->hEFAPdata, direct_response[i], hParamIsmDec->edited_azimuth_values[i], hParamIsmDec->edited_elevation_values[i], EFAP_MODE_EFAP ); +#else + efap_determine_gains( st_ivas->hEFAPdata, direct_response[i], hParamIsmDec->azimuth_values[i], hParamIsmDec->elevation_values[i], EFAP_MODE_EFAP ); +#endif + } + } + else + { + int16_t j; + + for ( i = 0; i < st_ivas->nchan_ism; i++ ) + { + for ( j = 0; j < nchan_out_woLFE; j++ ) + { + if ( i == j ) + { + direct_response[i][j] = 1.0f; + } + else + { + direct_response[i][j] = 0.0f; + } + } + } + + for ( j = 0; j < nchan_out_woLFE; j++ ) + { + if ( hParamIsmDec->azimuth_values[j] > 0.0f ) + { + hParamIsmDec->hParamIsmRendering->proto_matrix[j] = 1.0f; + hParamIsmDec->hParamIsmRendering->proto_matrix[nchan_out_woLFE + j] = 0.0f; + } + else + { + if ( hParamIsmDec->azimuth_values[j] < 0.0f ) + { + hParamIsmDec->hParamIsmRendering->proto_matrix[j] = 0.0f; + hParamIsmDec->hParamIsmRendering->proto_matrix[nchan_out_woLFE + j] = 1.0f; + } + else /* == 0.0f */ + { + hParamIsmDec->hParamIsmRendering->proto_matrix[j] = 0.5f; + hParamIsmDec->hParamIsmRendering->proto_matrix[nchan_out_woLFE + j] = 0.5f; + } + } + } + } + + for ( ch = 0; ch < nchan_transport; ch++ ) + { + /* CLDFB Analysis */ + for ( slot_idx = 0; slot_idx < nCldfbSlots; slot_idx++ ) + { + ivas_param_ism_collect_slot( hParamIsmDec, &hParamIsmDec->hParamIsmRendering->Cldfb_RealBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], &hParamIsmDec->hParamIsmRendering->Cldfb_ImagBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], ch, ref_power, cx_diag ); + } + } + + /* Obtain Mixing Matrix on a frame-level */ + for ( bin_idx = 0; bin_idx < hSpatParamRendCom->num_freq_bands; bin_idx++ ) + { + set_f( hParamIsmDec->hParamIsmRendering->mixing_matrix_lin[bin_idx], 0.0f, nchan_transport * nchan_out_woLFE ); + } + + /* Compute mixing matrix */ + ivas_param_ism_compute_mixing_matrix( st_ivas->nchan_ism, hParamIsmDec, st_ivas->hISMDTX, direct_response, nchan_transport, nchan_out_woLFE, cx_diag, ref_power, hParamIsmDec->hParamIsmRendering->mixing_matrix_lin ); + + pop_wmops(); + + return; +} +#endif + /*-------------------------------------------------------------------------* * ivas_ism_param_dec_tc_gain_ajust() * @@ -1192,6 +1382,17 @@ void ivas_param_ism_dec_render( /* store MetaData parameters */ for ( ch = 0; ch < st_ivas->nchan_ism; ch++ ) { +#ifdef OBJ_EDITING_API + if ( st_ivas->hParamIsmDec->azimuth_values[ch] > 180.0f ) + { + st_ivas->hIsmMetaData[ch]->azimuth = st_ivas->hParamIsmDec->edited_azimuth_values[ch] - 360.0f; + } + else + { + st_ivas->hIsmMetaData[ch]->azimuth = st_ivas->hParamIsmDec->edited_azimuth_values[ch]; + } + st_ivas->hIsmMetaData[ch]->elevation = st_ivas->hParamIsmDec->edited_elevation_values[ch]; +#else if ( st_ivas->hParamIsmDec->azimuth_values[ch] > 180.0f ) { st_ivas->hIsmMetaData[ch]->azimuth = st_ivas->hParamIsmDec->azimuth_values[ch] - 360.0f; @@ -1202,6 +1403,7 @@ void ivas_param_ism_dec_render( } st_ivas->hIsmMetaData[ch]->elevation = st_ivas->hParamIsmDec->elevation_values[ch]; +#endif } } @@ -1229,12 +1431,23 @@ void ivas_param_ism_params_to_masa_param_mapping( int16_t azimuth[2]; int16_t elevation[2]; float power_ratio[2]; +#ifdef OBJ_EDITING_PARAMISM_BIN + MASA_ISM_DATA_HANDLE hMasaIsmData; + int16_t obj; + int16_t obj_idx; +#endif +#ifndef OBJ_EDITING_API int32_t ivas_total_brate; +#endif hParamIsmDec = st_ivas->hParamIsmDec; hSpatParamRendCom = st_ivas->hSpatParamRendCom; +#ifdef OBJ_EDITING_PARAMISM_BIN + hMasaIsmData = st_ivas->hMasaIsmData; +#endif nBins = hSpatParamRendCom->num_freq_bands; +#ifndef OBJ_EDITING_API ivas_total_brate = st_ivas->hDecoderConfig->ivas_total_brate; if ( !( ivas_total_brate == IVAS_SID_5k2 || ivas_total_brate == FRAME_NO_DATA ) ) @@ -1247,6 +1460,7 @@ void ivas_param_ism_params_to_masa_param_mapping( { st_ivas->hISMDTX.dtx_flag = 1; } +#endif if ( st_ivas->hISMDTX.dtx_flag ) { @@ -1279,12 +1493,22 @@ void ivas_param_ism_params_to_masa_param_mapping( brange[0] = hParamIsmDec->hParamIsm->band_grouping[band_idx]; brange[1] = hParamIsmDec->hParamIsm->band_grouping[band_idx + 1]; +#ifdef OBJ_EDITING_API + azimuth[0] = (int16_t) roundf( hParamIsmDec->edited_azimuth_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][0]] ); + elevation[0] = (int16_t) roundf( hParamIsmDec->edited_elevation_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][0]] ); +#else azimuth[0] = (int16_t) roundf( hParamIsmDec->azimuth_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][0]] ); elevation[0] = (int16_t) roundf( hParamIsmDec->elevation_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][0]] ); +#endif power_ratio[0] = hParamIsmDec->power_ratios[band_idx][0][0]; +#ifdef OBJ_EDITING_API + azimuth[1] = (int16_t) roundf( hParamIsmDec->edited_azimuth_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][1]] ); + elevation[1] = (int16_t) roundf( hParamIsmDec->edited_elevation_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][1]] ); +#else azimuth[1] = (int16_t) roundf( hParamIsmDec->azimuth_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][1]] ); elevation[1] = (int16_t) roundf( hParamIsmDec->elevation_values[hParamIsmDec->hParamIsm->obj_indices[band_idx][0][1]] ); +#endif power_ratio[1] = hParamIsmDec->power_ratios[band_idx][0][1]; for ( sf_idx = 0; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) @@ -1310,6 +1534,44 @@ void ivas_param_ism_params_to_masa_param_mapping( hSpatParamRendCom->surroundingCoherence[sf_idx][bin_idx] = 0.0; } } +#ifdef OBJ_EDITING_PARAMISM_BIN + for ( obj = 0; obj < st_ivas->nchan_ism; obj++ ) + { + for ( sf_idx = 0; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) + { + hMasaIsmData->azimuth_ism[obj][sf_idx] = (int16_t) roundf( hParamIsmDec->azimuth_values[obj] ); + hMasaIsmData->elevation_ism[obj][sf_idx] = (int16_t) roundf( hParamIsmDec->elevation_values[obj] ); + } + + hMasaIsmData->azimuth_ism_edited[obj] = (int16_t) roundf( hParamIsmDec->edited_azimuth_values[obj] ); + hMasaIsmData->elevation_ism_edited[obj] = (int16_t) roundf( hParamIsmDec->edited_elevation_values[obj] ); + + for ( sf_idx = 0; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) + { + set_zero( hMasaIsmData->energy_ratio_ism[obj][sf_idx], CLDFB_NO_CHANNELS_MAX ); + } + } + + for ( obj = 0; obj < MAX_PARAM_ISM_WAVE; obj++ ) + { + for ( band_idx = 0; band_idx < hParamIsmDec->hParamIsm->nbands; band_idx++ ) + { + brange[0] = hParamIsmDec->hParamIsm->band_grouping[band_idx]; + brange[1] = hParamIsmDec->hParamIsm->band_grouping[band_idx + 1]; + + obj_idx = hParamIsmDec->hParamIsm->obj_indices[band_idx][0][obj]; + power_ratio[obj] = hParamIsmDec->power_ratios[band_idx][0][obj]; + + for ( sf_idx = 0; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) + { + for ( bin_idx = brange[0]; bin_idx < brange[1]; bin_idx++ ) + { + hMasaIsmData->energy_ratio_ism[obj_idx][sf_idx][bin_idx] = power_ratio[obj]; + } + } + } + } +#endif } return; diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index 11b8c5a2029854c41c3ac70ebe1cda90a000f23d..ae6e4523137882bfa65a821847b20815ce7c2530 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -233,15 +233,26 @@ void ivas_ism_render_sf( { if ( subframe_idx >= ism_md_subframe_update_jbm ) { +#ifdef OBJ_EDITING_API + rotateAziEle( st_ivas->hIsmMetaData[i]->edited_azimuth, st_ivas->hIsmMetaData[i]->edited_elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat[st_ivas->hCombinedOrientationData->subframe_idx], st_ivas->hIntSetup.is_planar_setup ); +#else rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat[st_ivas->hCombinedOrientationData->subframe_idx], st_ivas->hIntSetup.is_planar_setup ); +#endif } else { +#ifdef OBJ_EDITING_API + rotateAziEle( st_ivas->hIsmMetaData[i]->edited_azimuth, st_ivas->hIsmMetaData[i]->edited_elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat[st_ivas->hCombinedOrientationData->subframe_idx], st_ivas->hIntSetup.is_planar_setup ); +#else rotateAziEle( st_ivas->hIsmMetaData[i]->last_azimuth, st_ivas->hIsmMetaData[i]->last_elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat[st_ivas->hCombinedOrientationData->subframe_idx], st_ivas->hIntSetup.is_planar_setup ); +#endif } if ( st_ivas->hEFAPdata != NULL ) { efap_determine_gains( st_ivas->hEFAPdata, st_ivas->hIsmRendererData->gains[i], azimuth, elevation, EFAP_MODE_EFAP ); +#ifdef OBJ_EDITING_API + v_multc( st_ivas->hIsmRendererData->gains[i], st_ivas->hIsmMetaData[i]->edited_gain, st_ivas->hIsmRendererData->gains[i], nchan_out_woLFE ); +#endif } } @@ -546,13 +557,39 @@ void ivas_omasa_separate_object_render_jbm( if ( single_separated ) { +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && st_ivas->hMasaIsmData->ism_dir_is_edited[st_ivas->hMasaIsmData->idx_separated_ism] ) + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism_edited[st_ivas->hMasaIsmData->idx_separated_ism]; + elevation = st_ivas->hMasaIsmData->elevation_ism_edited[st_ivas->hMasaIsmData->idx_separated_ism]; + } + else + { + azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[md_idx]; + elevation = st_ivas->hMasaIsmData->elevation_separated_ism[md_idx]; + } +#else azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[md_idx]; elevation = st_ivas->hMasaIsmData->elevation_separated_ism[md_idx]; +#endif } else { +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->hMasaIsmData->ism_dir_is_edited[obj] ) + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism_edited[obj]; + elevation = st_ivas->hMasaIsmData->elevation_ism_edited[obj]; + } + else + { + azimuth = st_ivas->hMasaIsmData->azimuth_ism[obj][md_idx]; + elevation = st_ivas->hMasaIsmData->elevation_ism[obj][md_idx]; + } +#else azimuth = st_ivas->hMasaIsmData->azimuth_ism[obj][md_idx]; elevation = st_ivas->hMasaIsmData->elevation_ism[obj][md_idx]; +#endif } if ( st_ivas->hOutSetup.is_planar_setup ) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index b005723168d36b95b028a523256550cca299cdd3..6eea0419a29f31a65394034831ec423ed42af106 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -150,6 +150,9 @@ ivas_error ivas_jbm_dec_tc( } ivas_ism_dtx_limit_noise_energy_for_near_silence( st_ivas->hSCE, st_ivas->hISMDTX.sce_id_dtx, st_ivas->nchan_transport ); +#ifdef OBJ_EDITING_API + ivas_param_ism_dec_dequant_md( st_ivas ); +#endif } else if ( st_ivas->ism_mode == ISM_MODE_PARAM ) { @@ -157,6 +160,9 @@ ivas_error ivas_jbm_dec_tc( { return error; } +#ifdef OBJ_EDITING_API + ivas_param_ism_dec_dequant_md( st_ivas ); +#endif } else /* ISM_MODE_DISC */ { @@ -810,6 +816,7 @@ void ivas_jbm_dec_feed_tc_to_renderer( } n_render_timeslots = st_ivas->hTcBuffer->n_samples_available / st_ivas->hTcBuffer->n_samples_granularity; +#ifndef OBJ_EDITING_API if ( st_ivas->hTcBuffer->tc_buffer_mode == TC_BUFFER_MODE_BUFFER ) { ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); @@ -823,27 +830,35 @@ void ivas_jbm_dec_feed_tc_to_renderer( { ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); } - else if ( st_ivas->ivas_format == ISM_FORMAT ) + else +#endif + if ( st_ivas->ivas_format == ISM_FORMAT ) { /* Rendering */ if ( st_ivas->ism_mode == ISM_MODE_PARAM ) { +#ifndef OBJ_EDITING_API if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { ivas_dirac_dec_set_md_map( st_ivas, n_render_timeslots ); ivas_param_ism_params_to_masa_param_mapping( st_ivas ); } - else if ( st_ivas->renderer_type == RENDERER_PARAM_ISM || st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) + else +#endif + if ( st_ivas->renderer_type == RENDERER_PARAM_ISM || st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) { ivas_param_ism_dec_digest_tc( st_ivas, n_render_timeslots, p_data_f ); } } +#ifndef OBJ_EDITING_API else /* ISM_MODE_DISC */ { ivas_ism_dec_digest_tc( st_ivas ); } +#endif } +#ifndef OBJ_EDITING_API else if ( st_ivas->ivas_format == SBA_FORMAT || st_ivas->ivas_format == MASA_FORMAT ) { ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); @@ -959,6 +974,15 @@ void ivas_jbm_dec_feed_tc_to_renderer( ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); } } +#else + else if ( st_ivas->ivas_format == MC_FORMAT ) + { + if ( st_ivas->mc_mode == MC_MODE_PARAMMC && st_ivas->hTcBuffer->tc_buffer_mode == TC_BUFFER_MODE_RENDERER ) + { + ivas_param_mc_dec_digest_tc( st_ivas, (uint8_t) n_render_timeslots, p_data_f ); + } + } +#endif pop_wmops(); return; @@ -1223,9 +1247,36 @@ ivas_error ivas_jbm_dec_render( ivas_ism_render_sf( st_ivas, st_ivas->renderer_type, p_output, *nSamplesRendered ); /* add already rendered SBA part */ - for ( n = 0; n < nchan_out; n++ ) +#ifdef OBJ_EDITING_API + if ( st_ivas->ivas_format == SBA_ISM_FORMAT && st_ivas->ism_mode == ISM_SBA_MODE_DISC ) { - v_add( p_output[n], p_tc[n + st_ivas->nchan_ism], p_output[n], *nSamplesRendered ); + float gain = st_ivas->hSbaIsmData->gain_bed; + if ( gain != 1.0f && gain >= 0.0f ) + { + int16_t i; + for ( n = 0; n < nchan_out; n++ ) + { + for ( i = 0; i < *nSamplesRendered; i++ ) + { + p_output[n][i] += p_tc[n + st_ivas->nchan_ism][i] * gain; + } + } + } + else + { + for ( n = 0; n < nchan_out; n++ ) + { + v_add( p_output[n], p_tc[n + st_ivas->nchan_ism], p_output[n], *nSamplesRendered ); + } + } + } + else +#endif + { + for ( n = 0; n < nchan_out; n++ ) + { + v_add( p_output[n], p_tc[n + st_ivas->nchan_ism], p_output[n], *nSamplesRendered ); + } } } else if ( st_ivas->renderer_type == RENDERER_OSBA_AMBI || st_ivas->renderer_type == RENDERER_OSBA_LS || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) @@ -2947,3 +2998,199 @@ void ivas_jbm_masa_sf_to_sf_map( return; } + +#ifdef OBJ_EDITING_API +/*--------------------------------------------------------------------------* + * ivas_jbm_dec_prepare_renderer() + * + * prepare IVAS JBM renderer routine + *--------------------------------------------------------------------------*/ + +void ivas_jbm_dec_prepare_renderer( + Decoder_Struct *st_ivas /* i/o: IVAS decoder structure */ +) +{ + int16_t n, n_render_timeslots; + + push_wmops( "ivas_jbm_dec_feed_tc_to_rendererer" ); + + n_render_timeslots = st_ivas->hTcBuffer->n_samples_available / st_ivas->hTcBuffer->n_samples_granularity; + + if ( st_ivas->hTcBuffer->tc_buffer_mode == TC_BUFFER_MODE_BUFFER ) + { + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + + if ( ( st_ivas->ivas_format == MASA_FORMAT || st_ivas->ivas_format == MASA_ISM_FORMAT ) && st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL && st_ivas->hDecoderConfig->Opt_tsm ) + { + ivas_jbm_masa_sf_to_slot_map( st_ivas, n_render_timeslots ); + } + } + else if ( st_ivas->ivas_format == STEREO_FORMAT ) + { + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + } + else if ( st_ivas->ivas_format == ISM_FORMAT ) + { + /* Rendering */ + if ( st_ivas->ism_mode == ISM_MODE_PARAM ) + { + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + ivas_dirac_dec_set_md_map( st_ivas, n_render_timeslots ); + ivas_param_ism_params_to_masa_param_mapping( st_ivas ); + } + else if ( st_ivas->renderer_type == RENDERER_PARAM_ISM || st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) + { + ivas_param_ism_dec_prepare_renderer( st_ivas, n_render_timeslots ); + } + } + else /* ISM_MODE_DISC */ + { + ivas_ism_dec_digest_tc( st_ivas ); + } + } + else if ( st_ivas->ivas_format == SBA_FORMAT || st_ivas->ivas_format == MASA_FORMAT ) + { + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + } + else if ( st_ivas->ivas_format == SBA_ISM_FORMAT ) + { + + if ( st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + ivas_ism_dec_digest_tc( st_ivas ); + + /* delay the objects here for all renderers where it is needed */ + if ( ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || + st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM || + st_ivas->renderer_type == RENDERER_OSBA_AMBI || + st_ivas->renderer_type == RENDERER_OSBA_LS || + st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) && + ( st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + for ( n = 0; n < st_ivas->nchan_ism; n++ ) + { + delay_signal( st_ivas->hTcBuffer->tc[n], st_ivas->hTcBuffer->n_samples_available, st_ivas->hSbaIsmData->delayBuffer[n], st_ivas->hSbaIsmData->delayBuffer_size ); + } + } + + if ( !st_ivas->sba_dirac_stereo_flag ) + { + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV ) + { + n_render_timeslots *= ( st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size ); + } + + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + } + } + else + { + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + } + } + else if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + if ( st_ivas->renderer_type == RENDERER_OMASA_MIX_EXT || st_ivas->renderer_type == RENDERER_OMASA_OBJECT_EXT ) + { + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + + if ( st_ivas->hDecoderConfig->Opt_tsm ) + { + ivas_jbm_masa_sf_to_slot_map( st_ivas, n_render_timeslots ); + } + } + else + { + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + n_render_timeslots *= ( st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size ); + } + + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + ivas_ism_dec_digest_tc( st_ivas ); + } + + if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_DIRAC ) + { + int16_t num_objects; + /* Delay the signal to match CLDFB delay. Delay the whole buffer. */ + num_objects = 0; + if ( ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC ) + { + num_objects = 1; + } + else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + num_objects = st_ivas->nchan_ism; + } + for ( n = 0; n < num_objects; n++ ) + { + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC ) + { +#ifdef OMASA_OBJECT_EDITING + float gainIsm; + gainIsm = OMASA_TDREND_MATCHING_GAIN; + + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[n] ) + { + gainIsm *= st_ivas->hMasaIsmData->gain_ism_edited[n]; + } + + v_multc( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], gainIsm, st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available ); +#else + v_multc( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], OMASA_TDREND_MATCHING_GAIN, st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available ); +#endif + } + if ( st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED && st_ivas->hDecoderConfig->output_config != IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + delay_signal( st_ivas->hTcBuffer->tc[CPE_CHANNELS + n], st_ivas->hTcBuffer->n_samples_available, st_ivas->hMasaIsmData->delayBuffer[n], st_ivas->hMasaIsmData->delayBuffer_size ); + } + } +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC ) + { + int16_t tcBufferSize; + tcBufferSize = st_ivas->hSpatParamRendCom->num_slots * st_ivas->hSpatParamRendCom->slot_size; + + if ( st_ivas->hMasaIsmData->masa_gain_is_edited ) + { + for ( int16_t ch = 0; ch < 2; ch++ ) + { + v_multc( st_ivas->hTcBuffer->tc[ch], st_ivas->hMasaIsmData->gain_masa_edited, st_ivas->hTcBuffer->tc[ch], tcBufferSize ); + } + } + } +#endif + } + } + } + else if ( st_ivas->ivas_format == MC_FORMAT ) + { + if ( st_ivas->mc_mode == MC_MODE_MCT ) + { + ivas_jbm_dec_td_renderers_adapt_subframes( st_ivas ); + } + else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) + { + ivas_mc_paramupmix_dec_digest_tc( st_ivas, (uint8_t) n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + } + else if ( st_ivas->mc_mode == MC_MODE_PARAMMC ) + { + ivas_param_mc_dec_prepare_renderer( st_ivas, (uint8_t) n_render_timeslots ); + } + else if ( st_ivas->mc_mode == MC_MODE_MCMASA ) + { + ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available ); + } + } + + pop_wmops(); + return; +} +#endif diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index ef7d51fc37af13c3e8e4055ea5a6bffb3dc6c1ab..436517639dd6c8ae14c11e8cacda9395099caae8 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -634,7 +634,11 @@ ivas_error ivas_masa_dec_open( ism_total_brate = 0; /* ISM metadata */ +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->ivas_format == MASA_FORMAT && st_ivas->hIsmMetaData[0] != NULL ) +#else if ( st_ivas->ivas_format == MASA_FORMAT && st_ivas->hIsmMetaData[0] != NULL && st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) +#endif { /* these are not needed -> clean. EXT metafile writer in OMASA needs only the number of ISMs and writes default null-data */ ivas_ism_metadata_close( st_ivas->hIsmMetaData, 0 ); @@ -2427,6 +2431,9 @@ static int16_t ivas_decode_masaism_metadata( for ( obj = 0; obj < nchan_ism; obj++ ) { +#ifdef OMASA_OBJECT_EDITING + hMasaIsmData->bits_ism[obj] = bits_ism[obj]; +#endif index = 0; if ( bits_ism[obj] < 8 ) /* if low resolution, can look to the past */ { diff --git a/lib_dec/ivas_mc_param_dec.c b/lib_dec/ivas_mc_param_dec.c index 54b6438d01e9368fe6a0e4c05677a34588b02c86..8b26bfbaba485b9f980f2123af87104567aa4c4c 100644 --- a/lib_dec/ivas_mc_param_dec.c +++ b/lib_dec/ivas_mc_param_dec.c @@ -1331,6 +1331,10 @@ void ivas_param_mc_dec_digest_tc( ) { PARAM_MC_DEC_HANDLE hParamMC; +#ifdef OBJ_EDITING_API + int16_t ch, slot_idx; + int16_t nchan_transport; +#else int16_t i, ch; #ifdef FIX_835_PARAMMC_BUFFER_VALUES int16_t is_next_band, skip_next_band; @@ -1338,6 +1342,9 @@ void ivas_param_mc_dec_digest_tc( int16_t slot_idx, param_band_idx; int16_t nchan_transport, nchan_out_transport, nchan_out_cldfb; int16_t nchan_out_cov; +#endif + +#ifndef OBJ_EDITING_API /*CLDFB*/ #ifdef FIX_835_PARAMMC_BUFFER_VALUES float *pCx, *pCx_imag; @@ -1353,14 +1360,18 @@ void ivas_param_mc_dec_digest_tc( /* format converter */ int16_t channel_active[MAX_OUTPUT_CHANNELS]; IVAS_OUTPUT_SETUP *hSynthesisOutputSetup; +#endif hParamMC = st_ivas->hParamMC; assert( hParamMC ); push_wmops( "param_mc_dec_digest_tc" ); +#ifndef OBJ_EDITING_API set_s( channel_active, 0, MAX_CICP_CHANNELS ); +#endif nchan_transport = st_ivas->nchan_transport; +#ifndef OBJ_EDITING_API nchan_out_transport = st_ivas->hTransSetup.nchan_out_woLFE + st_ivas->hTransSetup.num_lfe; if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) @@ -1424,6 +1435,7 @@ void ivas_param_mc_dec_digest_tc( set_zero( cx[param_band_idx], PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); set_zero( cx_imag[param_band_idx], PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); } +#endif #endif /* slot loop for gathering the input data */ @@ -1431,7 +1443,7 @@ void ivas_param_mc_dec_digest_tc( { if ( st_ivas->hDecoderConfig->Opt_tsm ) { -#ifdef FIX_835_PARAMMC_BUFFER_VALUES +#if defined( FIX_835_PARAMMC_BUFFER_VALUES ) && !defined( OBJ_EDITING_API ) if ( param_band_idx == 0 ) /* only run cldfbAna once */ { #endif @@ -1446,10 +1458,247 @@ void ivas_param_mc_dec_digest_tc( mvr2r( RealBuffer, &hParamMC->Cldfb_RealBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport + ch * hParamMC->num_freq_bands], hParamMC->num_freq_bands ); mvr2r( ImagBuffer, &hParamMC->Cldfb_ImagBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport + ch * hParamMC->num_freq_bands], hParamMC->num_freq_bands ); } +#if defined( FIX_835_PARAMMC_BUFFER_VALUES ) && !defined( OBJ_EDITING_API ) + } +#endif + } +#ifndef OBJ_EDITING_API + if ( slot_idx >= 2 * hParamMC->hMetadataPMC->attackIndex ) + { +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + for ( is_next_band = 0; is_next_band < 2; is_next_band++ ) + { + if ( is_next_band && skip_next_band ) + { + continue; + } +#endif + ivas_dirac_dec_output_synthesis_cov_param_mc_collect_slot( &hParamMC->Cldfb_RealBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport], + &hParamMC->Cldfb_ImagBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport], +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + is_next_band ? cx_next_band : cx, + is_next_band ? cx_imag_next_band : cx_imag, + param_band_idx + is_next_band, +#else + cx, + cx_imag, +#endif + hParamMC, + nchan_transport ); #ifdef FIX_835_PARAMMC_BUFFER_VALUES } #endif } +#endif + } + +#ifndef OBJ_EDITING_API + /* map from complex input covariance to real values */ +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + for ( is_next_band = 0; is_next_band < 2; is_next_band++ ) + { + if ( is_next_band && skip_next_band ) + { + continue; + } +#else + for ( param_band_idx = 0; param_band_idx < hParamMC->num_param_bands_synth; param_band_idx++ ) + { +#endif + /* Cx for transport channels */ +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + pCx = is_next_band ? &cx_next_band[0] : &cx[0]; + pCx_imag = is_next_band ? &cx_imag_next_band[0] : &cx_imag[0]; +#endif + for ( i = 0; i < nchan_transport * nchan_transport; i++ ) + { +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + real_part = pCx[i]; + imag_part = pCx_imag[i]; +#else + real_part = cx[param_band_idx][i]; + imag_part = cx_imag[param_band_idx][i]; +#endif + + /* (a-ib)(c+id) = ac + bd + i(ad-bc) */ + if ( param_band_idx < hParamMC->max_param_band_abs_cov ) + { +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + pCx[i] = sqrtf( real_part * real_part + imag_part * imag_part ); +#else + cx[param_band_idx][i] = sqrtf( real_part * real_part + imag_part * imag_part ); +#endif + } + else + { +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + pCx[i] = real_part; +#else + cx[param_band_idx][i] = real_part; +#endif + } + } + } + + /* we have to do it similar to the encoder in case of attacks (i.e. accumulate two bands) to ensure correct DMX of the target covariance*/ + if ( hParamMC->hMetadataPMC->bAttackPresent && ( hParamMC->synthesis_conf == PARAM_MC_SYNTH_LS_CONV_COV || hParamMC->synthesis_conf == PARAM_MC_SYNTH_MONO_STEREO ) ) + { +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + v_add( cx, cx_next_band, cx, nchan_transport * nchan_transport ); + mvr2r( cx, cx_next_band, nchan_transport * nchan_transport ); +#else + for ( param_band_idx = 0; param_band_idx < hParamMC->num_param_bands_synth; param_band_idx += 2 ) + { + v_add( cx[param_band_idx], cx[param_band_idx + 1], cx[param_band_idx], nchan_transport * nchan_transport ); + mvr2r( cx[param_band_idx], cx[param_band_idx + 1], nchan_transport * nchan_transport ); + } +#endif + } + +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + for ( is_next_band = 0; is_next_band < 2; is_next_band++ ) + { + if ( is_next_band && skip_next_band ) + { + continue; + } +#endif + /* generate mixing matrices */ + ivas_param_mc_get_mixing_matrices( hParamMC, + hSynthesisOutputSetup, +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + is_next_band ? cx_next_band : cx, + param_band_idx + is_next_band, +#else + cx, +#endif + hParamMC->h_output_synthesis_cov_state.mixing_matrix, + hParamMC->h_output_synthesis_cov_state.mixing_matrix_res, + nchan_out_transport, + hParamMC->synthesis_conf, + nchan_transport, + nchan_out_cov ); +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + } + } +#endif +#endif + pop_wmops(); + + return; +} + +#ifdef OBJ_EDITING_API +/*------------------------------------------------------------------------- + * ivas_param_mc_dec_prepare_renderer() + * + * + *------------------------------------------------------------------------*/ + +void ivas_param_mc_dec_prepare_renderer( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint8_t nCldfbSlots /* i : number of CLFBS slots in the transport channels */ +) +{ + PARAM_MC_DEC_HANDLE hParamMC; + int16_t i; +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + int16_t is_next_band, skip_next_band; +#endif + int16_t slot_idx, param_band_idx; + int16_t nchan_transport, nchan_out_transport, nchan_out_cldfb; + int16_t nchan_out_cov; + /*CLDFB*/ +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + float *pCx, *pCx_imag; + float cx[PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS]; + float cx_imag[PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS]; + float cx_next_band[PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS]; + float cx_imag_next_band[PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS]; +#else + float cx[PARAM_MC_MAX_PARAMETER_BANDS][PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS]; + float cx_imag[PARAM_MC_MAX_PARAMETER_BANDS][PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS]; +#endif + float real_part, imag_part; + /* format converter */ + int16_t channel_active[MAX_OUTPUT_CHANNELS]; + IVAS_OUTPUT_SETUP *hSynthesisOutputSetup; + + hParamMC = st_ivas->hParamMC; + assert( hParamMC ); + + push_wmops( "param_mc_dec_digest_tc" ); + + set_s( channel_active, 0, MAX_CICP_CHANNELS ); + nchan_transport = st_ivas->nchan_transport; + nchan_out_transport = st_ivas->hTransSetup.nchan_out_woLFE + st_ivas->hTransSetup.num_lfe; + + if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV || st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV_ROOM ) + { + nchan_out_cldfb = BINAURAL_CHANNELS; + set_s( channel_active, 1, nchan_out_cldfb ); + nchan_out_cov = st_ivas->hTransSetup.nchan_out_woLFE + st_ivas->hTransSetup.num_lfe; + hSynthesisOutputSetup = &st_ivas->hTransSetup; + } + else if ( hParamMC->synthesis_conf == PARAM_MC_SYNTH_LS_CONV_CLDFB ) + { + nchan_out_cov = nchan_out_transport; + nchan_out_cldfb = st_ivas->hOutSetup.nchan_out_woLFE + st_ivas->hOutSetup.num_lfe; + hSynthesisOutputSetup = &st_ivas->hTransSetup; + } + else if ( hParamMC->synthesis_conf == PARAM_MC_SYNTH_LS_CONV_COV || hParamMC->synthesis_conf == PARAM_MC_SYNTH_MONO_STEREO ) + { + nchan_out_cov = st_ivas->hOutSetup.nchan_out_woLFE + st_ivas->hOutSetup.num_lfe; + nchan_out_cldfb = nchan_out_cov; + set_s( channel_active, 1, nchan_out_cov ); + hSynthesisOutputSetup = &st_ivas->hOutSetup; + } + else + { + nchan_out_cov = nchan_out_transport; + nchan_out_cldfb = nchan_out_transport; + set_s( channel_active, 1, nchan_out_cov ); + hSynthesisOutputSetup = &st_ivas->hTransSetup; + } + + /* adapt transient position */ + if ( hParamMC->hMetadataPMC->bAttackPresent ) + { + hParamMC->hMetadataPMC->attackIndex = (int16_t) max( 0, hParamMC->hMetadataPMC->attackIndex + ( ( nCldfbSlots - DEFAULT_JBM_CLDFB_TIMESLOTS ) / 2 ) ); + } + /* adapt subframes */ + hParamMC->num_slots = nCldfbSlots; + hParamMC->slots_rendered = 0; + hParamMC->subframes_rendered = 0; + ivas_jbm_dec_get_adapted_subframes( nCldfbSlots, hParamMC->subframe_nbslots, &hParamMC->nb_subframes ); + st_ivas->hTcBuffer->nb_subframes = hParamMC->nb_subframes; + mvs2s( hParamMC->subframe_nbslots, st_ivas->hTcBuffer->subframe_nbslots, hParamMC->nb_subframes ); + + ivas_param_mc_dec_compute_interpolator( hParamMC->hMetadataPMC->bAttackPresent, hParamMC->hMetadataPMC->attackIndex, nCldfbSlots, hParamMC->h_output_synthesis_params.interpolator ); + + +#ifdef FIX_835_PARAMMC_BUFFER_VALUES + /* loop over two bands at a time */ + for ( param_band_idx = 0; param_band_idx < hParamMC->num_param_bands_synth; param_band_idx += 2 ) + { + /* don't process next band if it exceeds the limit */ + skip_next_band = ( ( param_band_idx + 1 ) == hParamMC->num_param_bands_synth ) ? 1 : 0; + + set_zero( cx, PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); + set_zero( cx_imag, PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); + set_zero( cx_next_band, PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); + set_zero( cx_imag_next_band, PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); +#else + for ( param_band_idx = 0; param_band_idx < PARAM_MC_MAX_PARAMETER_BANDS; param_band_idx++ ) + { + set_zero( cx[param_band_idx], PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); + set_zero( cx_imag[param_band_idx], PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_TRANSPORT_CHANS ); + } +#endif + + /* slot loop for gathering the input data */ + for ( slot_idx = 0; slot_idx < nCldfbSlots; slot_idx++ ) + { if ( slot_idx >= 2 * hParamMC->hMetadataPMC->attackIndex ) { #ifdef FIX_835_PARAMMC_BUFFER_VALUES @@ -1573,6 +1822,7 @@ void ivas_param_mc_dec_digest_tc( return; } +#endif /*------------------------------------------------------------------------- diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index 7dcab1ad082e19dfe563e1478659a0183c4c25bc..77d89242a923d89bf808127443ea22cb32a839fa 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -173,9 +173,35 @@ ivas_error ivas_td_binaural_renderer_sf( } if ( subframe_idx == ism_md_subframe_update_jbm ) { - if ( ( error = TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, st_ivas->hIsmMetaData ) ) != IVAS_ERR_OK ) +#ifdef OBJ_EDITING_API + if ( st_ivas->ivas_format == ISM_FORMAT || st_ivas->ivas_format == SBA_ISM_FORMAT ) { - return error; + ISM_METADATA_FRAME ismMetaData[MAX_NUM_OBJECTS]; + ISM_METADATA_HANDLE hIsmMetaData[MAX_NUM_OBJECTS]; + for ( nS = 0; nS < nchan_ism; nS++ ) + { + ismMetaData[nS].azimuth = st_ivas->hIsmMetaData[nS]->edited_azimuth; + ismMetaData[nS].elevation = st_ivas->hIsmMetaData[nS]->edited_elevation; + ismMetaData[nS].radius = st_ivas->hIsmMetaData[nS]->edited_radius; + ismMetaData[nS].yaw = st_ivas->hIsmMetaData[nS]->edited_yaw; + ismMetaData[nS].pitch = st_ivas->hIsmMetaData[nS]->edited_pitch; + ismMetaData[nS].non_diegetic_flag = st_ivas->hIsmMetaData[nS]->non_diegetic_flag; + ismMetaData[nS].gain = st_ivas->hIsmMetaData[nS]->edited_gain; + hIsmMetaData[nS] = &ismMetaData[nS]; + } + + if ( ( error = TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, hIsmMetaData ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else +#endif + { + if ( ( error = TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, st_ivas->hIsmMetaData ) ) != IVAS_ERR_OK ) + { + return error; + } } } diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index 465f013317990d726414c8a15ff69994d35fb084..8164c2aa656d8a5c586ebdc4cc0ea77ddf71f3c2 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -60,7 +60,11 @@ ivas_error ivas_omasa_data_open( ) { MASA_ISM_DATA_HANDLE hMasaIsmData; +#ifdef OMASA_OBJECT_EDITING + int16_t ch; +#else int16_t ch, bin; +#endif int16_t sf, obj_idx; if ( ( hMasaIsmData = (MASA_ISM_DATA_HANDLE) malloc( sizeof( MASA_ISM_DATA ) ) ) == NULL ) @@ -68,6 +72,23 @@ ivas_error ivas_omasa_data_open( return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for MASA ISM data\n" ) ); } +#ifdef OMASA_OBJECT_EDITING + for ( int16_t band_idx = 0; band_idx < MASA_FREQUENCY_BANDS; band_idx++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + hMasaIsmData->ismPreprocMatrix[ch][ch][band_idx] = 1.0f; + hMasaIsmData->ismPreprocMatrix[1 - ch][ch][band_idx] = 0.0f; + hMasaIsmData->eneMoveIIR[ch][band_idx] = 0.0f; + hMasaIsmData->enePreserveIIR[ch][band_idx] = 0.0f; + } + hMasaIsmData->eneOrigIIR[band_idx] = 0.0f; + hMasaIsmData->preprocEneTarget[band_idx] = 0.0f; + hMasaIsmData->preprocEneRealized[band_idx] = 0.0f; + } + + hMasaIsmData->objectsEdited = 0; +#else for ( bin = 0; bin < CLDFB_NO_CHANNELS_MAX; bin++ ) { for ( ch = 0; ch < 2; ch++ ) @@ -82,14 +103,24 @@ ivas_error ivas_omasa_data_open( } hMasaIsmData->objectsMoved = 0; +#endif hMasaIsmData->delayBuffer = NULL; for ( ch = 0; ch < MAX_NUM_OBJECTS; ch++ ) { +#ifdef OMASA_OBJECT_EDITING + hMasaIsmData->ism_dir_is_edited[ch] = 0; + hMasaIsmData->ism_gain_is_edited[ch] = 0; +#else hMasaIsmData->ism_is_edited[ch] = 0; +#endif hMasaIsmData->q_elevation_old[ch] = 0.0f; hMasaIsmData->q_azimuth_old[ch] = 0.0f; } +#ifdef OMASA_OBJECT_EDITING + hMasaIsmData->masa_gain_is_edited = 0; + hMasaIsmData->idx_separated_ism = -1; +#endif for ( obj_idx = 0; obj_idx < MAX_NUM_OBJECTS; obj_idx++ ) { @@ -195,7 +226,11 @@ ivas_error ivas_omasa_dec_config( void *data /* o : output synthesis signal */ ) { +#ifdef OMASA_OBJECT_EDITING + int16_t k, sce_id, nSCE_old, nchan_hp20_old, numCldfbAnalyses_old, numCldfbSyntheses_old; +#else int16_t k, sce_id, nSCE_old, nchan_hp20_old, numCldfbAnalyses_old, numCldfbSyntheses_old, n_MD; +#endif int32_t ivas_total_brate, ism_total_brate, cpe_brate; ISM_MODE ism_mode_old; IVAS_FORMAT ivas_format_orig; @@ -287,6 +322,22 @@ ivas_error ivas_omasa_dec_config( if ( ism_mode_old != st_ivas->ism_mode ) { /* ISM MD reconfig. */ +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->hIsmMetaData[0] == NULL ) + { + if ( ( error = ivas_ism_metadata_dec_create( st_ivas, st_ivas->nchan_ism, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + } + else + { + for ( k = 0; k < st_ivas->nchan_ism; k++ ) + { + ivas_ism_reset_metadata_handle_dec( st_ivas->hIsmMetaData[k] ); + } + } +#else n_MD = 0; if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) @@ -337,6 +388,7 @@ ivas_error ivas_omasa_dec_config( ivas_ism_metadata_close( st_ivas->hIsmMetaData, n_MD ); } +#endif st_ivas->hCPE[0]->element_brate = ivas_total_brate - ism_total_brate; @@ -700,13 +752,44 @@ void ivas_omasa_dirac_rend_jbm( if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ || st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) { mvr2r( &output_f[CPE_CHANNELS][st_ivas->hTcBuffer->n_samples_rendered], data_separated_objects[0], *nSamplesRendered ); + +#ifdef OMASA_OBJECT_EDITING + if ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + /* Gain separated object, if edited */ + for ( n = 0; n < st_ivas->nchan_ism; n++ ) + { + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[n] && st_ivas->hMasaIsmData->idx_separated_ism == n ) + { + v_multc( data_separated_objects[0], st_ivas->hMasaIsmData->gain_ism_edited[n], data_separated_objects[0], *nSamplesRendered ); + } + } + } +#endif } else { for ( n = 0; n < st_ivas->nchan_ism; n++ ) { mvr2r( &output_f[n + CPE_CHANNELS][st_ivas->hTcBuffer->n_samples_rendered], data_separated_objects[n], *nSamplesRendered ); +#ifdef OMASA_OBJECT_EDITING + /* Gain discrete objects, if edited */ + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[n] ) + { + v_multc( data_separated_objects[n], st_ivas->hMasaIsmData->gain_ism_edited[n], data_separated_objects[n], *nSamplesRendered ); + } +#endif } +#ifdef OMASA_OBJECT_EDITING + /* Gain MASA part, if edited */ + if ( st_ivas->hMasaIsmData->masa_gain_is_edited ) + { + for ( int16_t ch = 0; ch < 2; ch++ ) + { + v_multc( output_f[ch], st_ivas->hMasaIsmData->gain_masa_edited, output_f[ch], *nSamplesRendered ); + } + } +#endif } } diff --git a/lib_dec/ivas_sba_dec.c b/lib_dec/ivas_sba_dec.c index d13268858cf3ed733371e2e8e81e3c8794032fba..9a6df407d4ed569dfd5ab38276ea27287616c2e8 100644 --- a/lib_dec/ivas_sba_dec.c +++ b/lib_dec/ivas_sba_dec.c @@ -795,6 +795,25 @@ ivas_error ivas_sba_dec_render( int16_t n_samples_sf = slot_size * hSpar->subframe_nbslots[subframe_idx]; ivas_spar_dec_upmixer_sf( st_ivas, output_f_local, nchan_internal ); + +#ifdef OBJ_EDITING_API + if ( st_ivas->ivas_format == SBA_ISM_FORMAT && st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + float gain = st_ivas->hSbaIsmData->gain_bed; + if ( gain != 1.0f && gain >= 0.0f ) + { + for ( ch = 0; ch < nchan_out; ch++ ) + { + int16_t i; + for ( i = 0; i < n_samples_sf; i++ ) + { + output_f_local[ch][i] *= gain; + } + } + } + } +#endif + for ( ch = 0; ch < nchan_out; ch++ ) { output_f_local[ch] += n_samples_sf; diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index b100848685f9edc2a3b2961318fc7e9f4e6c8002..c7d03b3fc272f8d65ffe2dd27cd93ca4e0c402fe 100644 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -422,6 +422,11 @@ typedef struct ivas_param_ism_dec_data_structure float elevation_values[MAX_PARAM_SPATIAL_SUBFRAMES * IVAS_MAX_NUM_BANDS]; float power_ratios[MAX_PARAM_ISM_NBANDS][MAX_PARAM_ISM_NBLOCKS][MAX_PARAM_ISM_WAVE]; +#ifdef OBJ_EDITING_API + float edited_azimuth_values[MAX_PARAM_SPATIAL_SUBFRAMES * IVAS_MAX_NUM_BANDS]; + float edited_elevation_values[MAX_PARAM_SPATIAL_SUBFRAMES * IVAS_MAX_NUM_BANDS]; +#endif + /*sub-modules*/ PARAM_ISM_CONFIG_HANDLE hParamIsm; /* Parametric ISM common handle */ PARAM_ISM_RENDERING_HANDLE hParamIsmRendering; /* ParamISM rendering handle */ @@ -674,6 +679,9 @@ typedef struct ivas_osba_data float **delayBuffer; int16_t delayBuffer_size; int16_t delayBuffer_nchan; +#ifdef OBJ_EDITING_API + float gain_bed; +#endif } SBA_ISM_DATA, *SBA_ISM_DATA_HANDLE; @@ -902,7 +910,22 @@ typedef struct ivas_masa_ism_data_structure int16_t azimuth_ism_edited[MAX_NUM_OBJECTS]; int16_t elevation_ism_edited[MAX_NUM_OBJECTS]; +#ifdef OMASA_OBJECT_EDITING + uint8_t ism_dir_is_edited[MAX_NUM_OBJECTS]; +#else uint8_t ism_is_edited[MAX_NUM_OBJECTS]; +#endif +#ifdef OBJ_EDITING_API + float gain_ism[MAX_NUM_OBJECTS]; + float gain_masa; +#endif + +#ifdef OMASA_OBJECT_EDITING + float gain_ism_edited[MAX_NUM_OBJECTS]; + uint8_t ism_gain_is_edited[MAX_NUM_OBJECTS]; + float gain_masa_edited; + uint8_t masa_gain_is_edited; +#endif int16_t idx_separated_ism; int16_t azimuth_separated_ism[MAX_PARAM_SPATIAL_SUBFRAMES + DELAY_MASA_PARAM_DEC_SFR]; @@ -911,12 +934,23 @@ typedef struct ivas_masa_ism_data_structure float q_azimuth_old[MAX_NUM_OBJECTS]; float q_elevation_old[MAX_NUM_OBJECTS]; +#ifdef OMASA_OBJECT_EDITING + uint8_t objectsEdited; + float preprocEneTarget[MASA_FREQUENCY_BANDS]; + float preprocEneRealized[MASA_FREQUENCY_BANDS]; + float ismPreprocMatrix[2][2][MASA_FREQUENCY_BANDS]; + float eneMoveIIR[2][MASA_FREQUENCY_BANDS]; + float enePreserveIIR[2][MASA_FREQUENCY_BANDS]; + float eneOrigIIR[MASA_FREQUENCY_BANDS]; + int16_t bits_ism[MAX_NUM_OBJECTS]; +#else float ismPreprocMatrix[2][2][CLDFB_NO_CHANNELS_MAX]; uint8_t objectsMoved; float eneMoveIIR[2][CLDFB_NO_CHANNELS_MAX]; float enePreserveIIR[2][CLDFB_NO_CHANNELS_MAX]; float preprocEneTarget[CLDFB_NO_CHANNELS_MAX]; float preprocEneRealized[CLDFB_NO_CHANNELS_MAX]; +#endif float **delayBuffer; int16_t delayBuffer_size; diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index cf62708e8709034fbedddc6edd0407d2c57b388f..61ba65f590ba14418ab94a8b1e8fe9faa05d5b65 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -35,6 +35,9 @@ #include "ivas_prot.h" #include "ivas_prot_rend.h" #include "isar_prot.h" +#ifdef OMASA_OBJECT_EDITING +#include "ivas_rom_com.h" +#endif #include "lib_isar_pre_rend.h" #include "prot.h" #include "jbm_jb4sb.h" @@ -95,6 +98,15 @@ struct IVAS_DEC int16_t sdp_hf_only; /* RTP payload format parameter: only Header-Full format without zero padding for size collision avoidance */ int16_t prev_ft_speech; /* RXDTX handler: previous frametype flag for G.192 format AMRWB SID_FIRST detection */ int16_t CNG; /* RXDTX handler: CNG=1, nonCNG=0 */ + +#ifdef OBJ_EDITING_API + uint16_t nSamplesFlushed; + void *flushbuffer; + IVAS_DEC_PCM_TYPE pcmType; + bool hasEditableParameters; + bool enableParameterEditing; + bool hasBeenPreparedRendering; +#endif }; @@ -172,6 +184,14 @@ ivas_error IVAS_DEC_Open( hIvasDec->hasDecodedFirstGoodFrame = false; hIvasDec->isInitialized = false; hIvasDec->updateOrientation = false; +#ifdef OBJ_EDITING_API + hIvasDec->flushbuffer = NULL; + hIvasDec->pcmType = IVAS_DEC_PCM_INVALID; + hIvasDec->nSamplesFlushed = 0; + hIvasDec->hasEditableParameters = false; + hIvasDec->enableParameterEditing = false; + hIvasDec->hasBeenPreparedRendering = false; +#endif hIvasDec->mode = mode; @@ -345,6 +365,12 @@ void IVAS_DEC_Close( { free( ( *phIvasDec )->apaExecBuffer ); } +#ifdef OBJ_EDITING_API + if ( ( *phIvasDec )->flushbuffer != NULL ) + { + free( ( *phIvasDec )->flushbuffer ); + } +#endif free( *phIvasDec ); *phIvasDec = NULL; phIvasDec = NULL; @@ -518,6 +544,16 @@ ivas_error IVAS_DEC_Configure( hIvasDec->tsm_max_scaling = 0; hIvasDec->tsm_quality = 1.0f; +#ifdef OBJ_EDITING_API + /* init flush buffer if necessary (only needed for binaural)*/ + if ( tsmEnabled && ( outputConfig == IVAS_AUDIO_CONFIG_BINAURAL || outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM || outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR || outputConfig == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) ) + { + hIvasDec->flushbuffer = (void *) malloc( CPE_CHANNELS * hIvasDec->nSamplesFrame / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES * sizeof( int16_t ) ); + hIvasDec->pcmType = IVAS_DEC_PCM_INT16; + set_s( (int16_t *) hIvasDec->flushbuffer, 0, CPE_CHANNELS * hIvasDec->nSamplesFrame / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ); + } +#endif + return error; } @@ -774,6 +810,16 @@ ivas_error IVAS_DEC_EnableVoIP( return IVAS_ERR_FAILED_ALLOC; } +#ifdef OBJ_EDITING_API + /* init flush buffer if necessary (only needed for binaural)*/ + if ( hIvasDec->flushbuffer == NULL && ( hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL || hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_IR || hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) ) + { + hIvasDec->flushbuffer = (void *) malloc( CPE_CHANNELS * hIvasDec->nSamplesFrame / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES * sizeof( int16_t ) ); + hIvasDec->pcmType = IVAS_DEC_PCM_INT16; + set_s( (int16_t *) hIvasDec->flushbuffer, 0, CPE_CHANNELS * hIvasDec->nSamplesFrame / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ); + } +#endif + return error; } @@ -789,6 +835,11 @@ ivas_error IVAS_DEC_FeedFrame_Serial( uint16_t *serial, /* i : buffer containing serial input bitstream. Each bit should be stored as a single uint16_t value */ const uint16_t num_bits, /* i : number of bits in input bitstream */ int16_t bfi /* i : bad frame indicator flag */ +#ifdef OBJ_EDITING_API + , + int16_t isSplitRend, + ISAR_SPLIT_REND_BITS_DATA *splitRendBits +#endif ) { ivas_error error; @@ -866,10 +917,499 @@ ivas_error IVAS_DEC_FeedFrame_Serial( hIvasDec->nSamplesRendered = 0; hIvasDec->nSamplesAvailableNext = hIvasDec->nSamplesFrame; +#ifdef OBJ_EDITING_API + /* decode TCs, do TSM and feed to renderer */ + /* setup */ + if ( hIvasDec->hasBeenFedFirstGoodFrame ) + { + uint16_t l_ts, nTimeScalerOutSamples; + uint8_t nTransportChannels, nOutChannels; + int16_t nResidualSamples, nSamplesTcsScaled, nOutSamplesElse; + + if ( isSplitRend ) + { + if ( ( error = isar_set_split_rend_setup( hIvasDec->st_ivas->hSplitBinRend, &hIvasDec->st_ivas->hRenderConfig->split_rend_config, hIvasDec->st_ivas->hCombinedOrientationData, splitRendBits ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + if ( ( error = IVAS_DEC_Setup( hIvasDec, &l_ts, &nTransportChannels, &nOutChannels, &hIvasDec->nSamplesFlushed, hIvasDec->pcmType, hIvasDec->flushbuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm && nTransportChannels != hIvasDec->nTransportChannelsOld ) + { + if ( ( error = IVAS_DEC_VoIP_reconfigure( hIvasDec, nTransportChannels, l_ts ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + /* IVAS TC decoder */ + if ( ( error = IVAS_DEC_GetTcSamples( hIvasDec, hIvasDec->apaExecBuffer, &nOutSamplesElse ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* JBM */ + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm ) + { + if ( apa_set_scale( hIvasDec->hTimeScaler, hIvasDec->tsm_scale ) != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + + if ( apa_exec( hIvasDec->hTimeScaler, hIvasDec->apaExecBuffer, hIvasDec->nSamplesFrame * nTransportChannels, (uint16_t) hIvasDec->tsm_max_scaling, hIvasDec->apaExecBuffer, &nTimeScalerOutSamples ) != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + + assert( nTimeScalerOutSamples <= APA_BUF ); + nSamplesTcsScaled = nTimeScalerOutSamples / nTransportChannels; + hIvasDec->timeScalingDone = 1; + } + else + { + nSamplesTcsScaled = hIvasDec->nSamplesFrame; + } + + /* Feed decoded transport channels samples to the renderer */ + if ( ( error = IVAS_DEC_RendererFeedTcSamples( hIvasDec, nSamplesTcsScaled, &nResidualSamples, hIvasDec->apaExecBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm ) + { + /* feed residual samples to TSM for the next call */ + if ( apa_set_renderer_residual_samples( hIvasDec->hTimeScaler, (uint16_t) nResidualSamples ) != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + } + } + hIvasDec->hasBeenPreparedRendering = false; + + if ( hIvasDec->st_ivas->hIsmMetaData[0] ) + { + if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT ) + { +#ifdef OMASA_OBJECT_EDITING + if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_DISC || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) +#else + if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_DISC || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC || hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) +#endif + { + int16_t obj; + ISM_METADATA_HANDLE *hIsmMetaData = hIvasDec->st_ivas->hIsmMetaData; + for ( obj = 0; obj < hIvasDec->st_ivas->nchan_ism; obj++ ) + { + hIsmMetaData[obj]->edited_azimuth = hIsmMetaData[obj]->azimuth; + hIsmMetaData[obj]->edited_elevation = hIsmMetaData[obj]->elevation; + hIsmMetaData[obj]->edited_yaw = hIsmMetaData[obj]->yaw; + hIsmMetaData[obj]->edited_pitch = hIsmMetaData[obj]->pitch; + hIsmMetaData[obj]->edited_radius = hIsmMetaData[obj]->radius; + hIsmMetaData[obj]->edited_gain = 1.0f; + } + + if ( hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + hIvasDec->st_ivas->hSbaIsmData->gain_bed = 1.0f; + } + } + } + } + + if ( hIvasDec->st_ivas->hParamIsmDec != NULL ) + { + if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_PARAM ) + { + int16_t obj = 0; + PARAM_ISM_DEC_HANDLE hParamIsmDec = hIvasDec->st_ivas->hParamIsmDec; + for ( obj = 0; obj < hIvasDec->st_ivas->nchan_ism; obj++ ) + { + hParamIsmDec->edited_azimuth_values[obj] = hParamIsmDec->azimuth_values[obj]; + hParamIsmDec->edited_elevation_values[obj] = hParamIsmDec->elevation_values[obj]; + } + } + } +#endif + return IVAS_ERR_OK; } +#ifdef OBJ_EDITING_API +/*---------------------------------------------------------------------* + * IVAS_DEC_GetEditableParameters( ) + * + * + *---------------------------------------------------------------------*/ +ivas_error IVAS_DEC_GetEditableParameters( + IVAS_DEC_HANDLE hIvasDec, + IVAS_EDITABLE_PARAMETERS *hIvasEditableParameters ) +{ + ivas_error error; + int16_t dirac_read_idx; + + if ( !( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || + ( hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT && hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) || + ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) ) ) + { + return IVAS_ERROR( IVAS_ERR_WRONG_MODE, "Object editing no supported in this operation mode." ); + } + + if ( hIvasEditableParameters == NULL || hIvasDec == NULL || hIvasDec->st_ivas == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + hIvasEditableParameters->gain_bed = 1.0f; + hIvasEditableParameters->num_obj = hIvasDec->st_ivas->nchan_ism; + if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT ) + { + if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_DISC || hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + int16_t obj; + for ( obj = 0; obj < hIvasEditableParameters->num_obj; obj++ ) + { + hIvasEditableParameters->ism_metadata[obj].azimuth = hIvasDec->st_ivas->hIsmMetaData[obj]->azimuth; + hIvasEditableParameters->ism_metadata[obj].elevation = hIvasDec->st_ivas->hIsmMetaData[obj]->elevation; + hIvasEditableParameters->ism_metadata[obj].yaw = hIvasDec->st_ivas->hIsmMetaData[obj]->yaw; + hIvasEditableParameters->ism_metadata[obj].pitch = hIvasDec->st_ivas->hIsmMetaData[obj]->pitch; + hIvasEditableParameters->ism_metadata[obj].radius = hIvasDec->st_ivas->hIsmMetaData[obj]->radius; + hIvasEditableParameters->ism_metadata[obj].gain = hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain; + hIvasEditableParameters->ism_metadata[obj].non_diegetic_flag = hIvasDec->st_ivas->hIsmMetaData[obj]->non_diegetic_flag; + } + if ( hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + hIvasEditableParameters->gain_bed = 1.0f; + } + } + else if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_PARAM ) + { + int16_t obj; + for ( obj = 0; obj < hIvasEditableParameters->num_obj; obj++ ) + { + hIvasEditableParameters->ism_metadata[obj].azimuth = hIvasDec->st_ivas->hParamIsmDec->azimuth_values[obj]; + hIvasEditableParameters->ism_metadata[obj].elevation = hIvasDec->st_ivas->hParamIsmDec->elevation_values[obj]; + hIvasEditableParameters->ism_metadata[obj].yaw = 0.0f; + hIvasEditableParameters->ism_metadata[obj].pitch = 0.0f; + hIvasEditableParameters->ism_metadata[obj].radius = 0.0f; + hIvasEditableParameters->ism_metadata[obj].gain = 1.0f; + hIvasEditableParameters->ism_metadata[obj].non_diegetic_flag = 0; + } + } + else + { + assert( 0 && "This should never happen!" ); + } + } +#ifdef OMASA_OBJECT_EDITING + else if ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) + { + /* object editing possible only in two highest OMASA modes */ + int16_t obj; + + if ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + for ( obj = 0; obj < hIvasEditableParameters->num_obj; obj++ ) + { + hIvasEditableParameters->ism_metadata[obj].azimuth = hIvasDec->st_ivas->hIsmMetaData[obj]->azimuth; + hIvasEditableParameters->ism_metadata[obj].elevation = hIvasDec->st_ivas->hIsmMetaData[obj]->elevation; + hIvasEditableParameters->ism_metadata[obj].yaw = hIvasDec->st_ivas->hIsmMetaData[obj]->yaw; + hIvasEditableParameters->ism_metadata[obj].pitch = hIvasDec->st_ivas->hIsmMetaData[obj]->pitch; + hIvasEditableParameters->ism_metadata[obj].radius = hIvasDec->st_ivas->hIsmMetaData[obj]->radius; + + /* reset the otherwise unused "gain" field for the object */ + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain = 1.0f; + hIvasEditableParameters->ism_metadata[obj].gain = hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain; + hIvasEditableParameters->ism_metadata[obj].non_diegetic_flag = hIvasDec->st_ivas->hIsmMetaData[obj]->non_diegetic_flag; + } + } + else + { + /* Handle MONO output */ + if ( hIvasDec->st_ivas->renderer_type != RENDERER_MONO_DOWNMIX ) + { + dirac_read_idx = hIvasDec->st_ivas->hSpatParamRendCom->render_to_md_map[hIvasDec->st_ivas->hSpatParamRendCom->subframes_rendered]; + } + else + { + dirac_read_idx = 0; + } + + for ( obj = 0; obj < hIvasEditableParameters->num_obj; obj++ ) + { + hIvasEditableParameters->ism_metadata[obj].azimuth = hIvasDec->st_ivas->hMasaIsmData->azimuth_ism[obj][dirac_read_idx]; + hIvasEditableParameters->ism_metadata[obj].elevation = hIvasDec->st_ivas->hMasaIsmData->elevation_ism[obj][dirac_read_idx]; + + hIvasEditableParameters->ism_metadata[obj].yaw = hIvasDec->st_ivas->hIsmMetaData[obj]->yaw; + hIvasEditableParameters->ism_metadata[obj].pitch = hIvasDec->st_ivas->hIsmMetaData[obj]->pitch; + hIvasEditableParameters->ism_metadata[obj].radius = hIvasDec->st_ivas->hIsmMetaData[obj]->radius; + + /* reset the otherwise unused "gain" field for the object */ + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain = 1.0f; + hIvasEditableParameters->ism_metadata[obj].gain = hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain; + hIvasEditableParameters->ism_metadata[obj].non_diegetic_flag = hIvasDec->st_ivas->hIsmMetaData[obj]->non_diegetic_flag; + } + } + } +#endif + + error = IVAS_ERR_OK; + + return error; +} + +/*---------------------------------------------------------------------* + * IVAS_DEC_SetEditableParameters( ) + * + * Main function to decode to PCM data + *---------------------------------------------------------------------*/ +ivas_error IVAS_DEC_SetEditableParameters( + IVAS_DEC_HANDLE hIvasDec, + IVAS_EDITABLE_PARAMETERS hIvasEditableParameters ) +{ + ivas_error error; + int16_t dirac_read_idx; + + if ( !( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || + ( hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT && hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) || + ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) ) || + hIvasDec->st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_EXTERNAL ) + { + return IVAS_ERROR( IVAS_ERR_WRONG_MODE, "Object editing no supported in this operation mode." ); + } + + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + error = IVAS_ERR_OK; + +#ifdef DEBUGGING + assert( hIvasEditableParameters.num_obj == hIvasDec->st_ivas->nchan_ism ); +#endif + + if ( hIvasDec->st_ivas->ivas_format == ISM_FORMAT || hIvasDec->st_ivas->ivas_format == SBA_ISM_FORMAT ) + { + if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_DISC || hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + int16_t obj; + for ( obj = 0; obj < hIvasEditableParameters.num_obj; obj++ ) + { + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_azimuth = hIvasEditableParameters.ism_metadata[obj].azimuth; + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_elevation = hIvasEditableParameters.ism_metadata[obj].elevation; + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_radius = hIvasEditableParameters.ism_metadata[obj].radius; + + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain = hIvasEditableParameters.ism_metadata[obj].gain; + + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_yaw = hIvasEditableParameters.ism_metadata[obj].yaw; + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_pitch = hIvasEditableParameters.ism_metadata[obj].pitch; + + hIvasDec->st_ivas->hIsmMetaData[obj]->non_diegetic_flag = hIvasEditableParameters.ism_metadata[obj].non_diegetic_flag; + } + + if ( hIvasDec->st_ivas->ism_mode == ISM_SBA_MODE_DISC ) + { + hIvasDec->st_ivas->hSbaIsmData->gain_bed = hIvasEditableParameters.gain_bed; + } + } + else if ( hIvasDec->st_ivas->ism_mode == ISM_MODE_PARAM ) + { + int16_t obj; + for ( obj = 0; obj < hIvasEditableParameters.num_obj; obj++ ) + { +#ifdef OBJ_EDITING_PARAMISM_BIN + hIvasDec->st_ivas->hParamIsmDec->edited_azimuth_values[obj] = hIvasEditableParameters.ism_metadata[obj].azimuth; + hIvasDec->st_ivas->hParamIsmDec->edited_elevation_values[obj] = hIvasEditableParameters.ism_metadata[obj].elevation; + hIvasDec->st_ivas->hMasaIsmData->gain_ism_edited[obj] = hIvasEditableParameters.ism_metadata[obj].gain; + + /* Detect direction editing in Param-ISM mode */ + if ( fabsf( hIvasDec->st_ivas->hParamIsmDec->azimuth_values[obj] - hIvasEditableParameters.ism_metadata[obj].azimuth ) > OMASA_AZI_EDIT_THR || + fabsf( hIvasDec->st_ivas->hParamIsmDec->elevation_values[obj] - hIvasEditableParameters.ism_metadata[obj].elevation ) > OMASA_ELE_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[obj] = 1u; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[obj] = 0u; + } + + /* Detect gain editing in Param-ISM mode */ + if ( fabsf( 1.0f - hIvasEditableParameters.ism_metadata[obj].gain ) > OMASA_GAIN_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->ism_gain_is_edited[obj] = 1u; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->ism_gain_is_edited[obj] = 0u; + } +#endif + } +#ifdef OBJ_EDITING_PARAMISM_BIN + /* MASA is not present with the ISM format */ + hIvasDec->st_ivas->hMasaIsmData->masa_gain_is_edited = 0u; +#endif + } + else + { + assert( 0 && "This should never happen!" ); + } + } +#ifdef OMASA_OBJECT_EDITING + else if ( hIvasDec->st_ivas->ivas_format == MASA_ISM_FORMAT && ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_DISC ) ) + { + int16_t obj; + int32_t id_th; + float threshold_azi, threshold_ele; + + for ( obj = 0; obj < hIvasEditableParameters.num_obj; obj++ ) + { + if ( hIvasDec->st_ivas->hMasaIsmData != NULL ) + { + /* copy relevant fields also to OMASA structs, but only if the value has been changed. original values are in st_ivas->hIsmMetaData */ + /* first, need to convert float values to ints used internally */ + int16_t new_azi, new_ele; + if ( hIvasEditableParameters.ism_metadata[obj].azimuth > 0.0f ) + { + new_azi = (int16_t) ( hIvasEditableParameters.ism_metadata[obj].azimuth + 0.5f ); + } + else + { + new_azi = (int16_t) ( hIvasEditableParameters.ism_metadata[obj].azimuth - 0.5f ); + } + + if ( hIvasEditableParameters.ism_metadata[obj].elevation > 0.0f ) + { + new_ele = (int16_t) ( hIvasEditableParameters.ism_metadata[obj].elevation + 0.5f ); + } + else + { + new_ele = (int16_t) ( hIvasEditableParameters.ism_metadata[obj].elevation - 0.5f ); + } + + if ( hIvasDec->st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ ) + { + /* Handle MONO output */ + if ( hIvasDec->st_ivas->renderer_type != RENDERER_MONO_DOWNMIX ) + { + dirac_read_idx = hIvasDec->st_ivas->hSpatParamRendCom->render_to_md_map[hIvasDec->st_ivas->hSpatParamRendCom->subframes_rendered]; + } + else + { + dirac_read_idx = 0; + } + + /* determine thresholds for detecting object metadata edit for direction based on quantization resolution of the spatial direction parameters. + * these depend from the number of bits used to transmit the directions, + * which in turn depends from the object priority and importance: + * importance -> priority -> number of bits -> elevation resolution -> elevation ring index -> azimuth resolution. + * leading to elevation_resolution -> elevation threshold and azimuth resolution -> azimuth threshold */ + id_th = (int) ( fabsf( (float) hIvasDec->st_ivas->hMasaIsmData->elevation_ism[obj][dirac_read_idx] ) / delta_theta_masa[hIvasDec->st_ivas->hMasaIsmData->bits_ism[obj] - 3] + 0.5f ); + threshold_azi = 360.0f / (float) no_phi_masa[hIvasDec->st_ivas->hMasaIsmData->bits_ism[obj] - 1][id_th]; + threshold_ele = delta_theta_masa[hIvasDec->st_ivas->hMasaIsmData->bits_ism[obj] - 3]; + + if ( ( (float) abs( new_azi - hIvasDec->st_ivas->hMasaIsmData->azimuth_ism[obj][dirac_read_idx] ) > threshold_azi ) || + ( (float) abs( new_ele - hIvasDec->st_ivas->hMasaIsmData->elevation_ism[obj][dirac_read_idx] ) > threshold_ele ) ) + { + /* at least one of the threshold is exceeded, so use new direction value and set editing detection flag */ + hIvasDec->st_ivas->hMasaIsmData->azimuth_ism_edited[obj] = new_azi; + hIvasDec->st_ivas->hMasaIsmData->elevation_ism_edited[obj] = new_ele; + + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[obj] = 1u; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[obj] = 0u; + } + } + else + { + /* detect editing in ISM_MASA_MODE_DISC. optionally, add quantization-resolution -based thresholds */ + if ( fabsf( hIvasDec->st_ivas->hIsmMetaData[obj]->azimuth - hIvasEditableParameters.ism_metadata[obj].azimuth ) > OMASA_AZI_EDIT_THR || + fabsf( hIvasDec->st_ivas->hIsmMetaData[obj]->elevation - hIvasEditableParameters.ism_metadata[obj].elevation ) > OMASA_ELE_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->azimuth_ism_edited[obj] = new_azi; + hIvasDec->st_ivas->hMasaIsmData->elevation_ism_edited[obj] = new_ele; + + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[obj] = 1u; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->ism_dir_is_edited[obj] = 0u; + } + } + + /* compare pre-edit gain and the edited one to detect editing */ + if ( fabsf( hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain - hIvasEditableParameters.ism_metadata[obj].gain ) > OMASA_GAIN_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->ism_gain_is_edited[obj] = 1u; + hIvasDec->st_ivas->hMasaIsmData->gain_ism_edited[obj] = hIvasEditableParameters.ism_metadata[obj].gain; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->ism_gain_is_edited[obj] = 0u; + } + } + + /* Copy edited values to hIsmMetaData struct */ + if ( hIvasDec->st_ivas->hIsmMetaData[obj] != NULL ) + { + hIvasDec->st_ivas->hIsmMetaData[obj]->azimuth = hIvasEditableParameters.ism_metadata[obj].azimuth; + hIvasDec->st_ivas->hIsmMetaData[obj]->elevation = hIvasEditableParameters.ism_metadata[obj].elevation; + hIvasDec->st_ivas->hIsmMetaData[obj]->yaw = hIvasEditableParameters.ism_metadata[obj].yaw; + hIvasDec->st_ivas->hIsmMetaData[obj]->pitch = hIvasEditableParameters.ism_metadata[obj].pitch; + hIvasDec->st_ivas->hIsmMetaData[obj]->radius = hIvasEditableParameters.ism_metadata[obj].radius; + hIvasDec->st_ivas->hIsmMetaData[obj]->edited_gain = hIvasEditableParameters.ism_metadata[obj].gain; + hIvasDec->st_ivas->hIsmMetaData[obj]->non_diegetic_flag = hIvasEditableParameters.ism_metadata[obj].non_diegetic_flag; + } + } + + if ( fabsf( hIvasEditableParameters.gain_bed - 1.0f ) > OMASA_GAIN_EDIT_THR ) + { + hIvasDec->st_ivas->hMasaIsmData->gain_masa_edited = hIvasEditableParameters.gain_bed; + hIvasDec->st_ivas->hMasaIsmData->masa_gain_is_edited = 1u; + } + else + { + hIvasDec->st_ivas->hMasaIsmData->masa_gain_is_edited = 0u; + } + } +#endif + + return error; +} + +/*---------------------------------------------------------------------* + * IVAS_DEC_PrepareRenderer( ) + * + * Main function to decode to PCM data + *---------------------------------------------------------------------*/ +ivas_error IVAS_DEC_PrepareRenderer( + IVAS_DEC_HANDLE hIvasDec ) +{ + ivas_error error; + + error = IVAS_ERR_OK; + + if ( hIvasDec == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + ivas_jbm_dec_prepare_renderer( hIvasDec->st_ivas ); + hIvasDec->hasBeenPreparedRendering = true; + return error; +} + + +#endif + /*---------------------------------------------------------------------* * IVAS_DEC_GetSamples( ) * @@ -887,21 +1427,38 @@ ivas_error IVAS_DEC_GetSamples( ) { ivas_error error; +#ifdef OBJ_EDITING_API + int16_t nSamplesToRender; + uint16_t nSamplesRendered, nSamplesRendered_loop; + uint8_t nOutChannels; +#else int16_t nOutSamplesElse, nSamplesToRender; uint16_t nSamplesRendered, nSamplesRendered_loop, l_ts, nTimeScalerOutSamples; uint8_t nTransportChannels, nOutChannels; +#endif nSamplesRendered = 0; nOutChannels = 0; nSamplesRendered_loop = 0; +#ifndef OBJ_EDITING_API l_ts = 0; nTransportChannels = 0; +#endif + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } +#ifdef OBJ_EDITING_API + /* the rendering needs to be prepared at this point */ + if ( hIvasDec->hasBeenPreparedRendering == false ) + { + return IVAS_ERR_UNKNOWN; + } +#endif + if ( hIvasDec->updateOrientation ) { /*----------------------------------------------------------------* @@ -950,6 +1507,7 @@ ivas_error IVAS_DEC_GetSamples( } else { +#ifndef OBJ_EDITING_API /* check if we need to run the setup function */ if ( !hIvasDec->isInitialized || hIvasDec->hasBeenFedFrame ) { @@ -960,8 +1518,10 @@ ivas_error IVAS_DEC_GetSamples( return error; } } +#endif { /* check if we need to run the setup function, tc decoding and feeding the renderer */ +#ifndef OBJ_EDITING_API if ( !hIvasDec->isInitialized || hIvasDec->hasBeenFedFrame ) { int16_t nResidualSamples, nSamplesTcsScaled; @@ -1023,6 +1583,35 @@ ivas_error IVAS_DEC_GetSamples( } hIvasDec->hasBeenFedFrame = false; } +#else + nOutChannels = (uint8_t) hIvasDec->st_ivas->hDecoderConfig->nchan_out; + hIvasDec->hasBeenFedFrame = false; + /* check for possible flushed samples from a rate switch */ + if ( hIvasDec->nSamplesFlushed > 0 ) + { + void *pPcmBuffer; +#ifdef DEBUGGING + assert( hIvasDec->pcmType == pcmType ); +#endif + pPcmBuffer = pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ); + if ( pcmType == IVAS_DEC_PCM_INT16 ) + { + mvs2s( (int16_t *) hIvasDec->flushbuffer, pPcmBuffer, hIvasDec->nSamplesFlushed * nOutChannels ); + } + else if ( pcmType == IVAS_DEC_PCM_FLOAT ) + { + mvr2r( (float *) hIvasDec->flushbuffer, pPcmBuffer, hIvasDec->nSamplesFlushed * nOutChannels ); + } +#ifdef DEBUGGING + else + { + assert( 0 & "wrong PCM type for the flush buffer!" ); + } +#endif + nSamplesRendered += hIvasDec->nSamplesFlushed; + hIvasDec->nSamplesFlushed = 0; + } +#endif /* render IVAS frames directly to the output buffer */ nSamplesToRender = nSamplesAsked - nSamplesRendered; @@ -1094,13 +1683,25 @@ ivas_error IVAS_DEC_GetSplitBinauralBitstream( *needNewFrame = false; hSplitBinRend = st_ivas->hSplitBinRend; +#ifndef OBJ_EDITING_API if ( ( error = isar_set_split_rend_setup( hSplitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, splitRendBits ) ) != IVAS_ERR_OK ) { return error; } +#endif numPoses = hSplitBinRend->splitrend.multiBinPoseData.num_poses; +#ifdef OBJ_EDITING_API + /* init flush buffer for rate switch if not already initizalized */ + if ( hIvasDec->flushbuffer == NULL ) + { + hIvasDec->flushbuffer = (void *) malloc( numPoses * BINAURAL_CHANNELS * hIvasDec->nSamplesFrame / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES * sizeof( float ) ); + hIvasDec->pcmType = IVAS_DEC_PCM_FLOAT; + set_zero( (float *) hIvasDec->flushbuffer, numPoses * BINAURAL_CHANNELS * hIvasDec->nSamplesFrame / IVAS_MAX_PARAM_SPATIAL_SUBFRAMES ); + } +#endif + if ( st_ivas->hDecoderConfig->render_framesize != IVAS_RENDER_FRAMESIZE_20MS && ( hIvasDec->st_ivas->hRenderConfig->split_rend_config.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE || hIvasDec->st_ivas->hRenderConfig->split_rend_config.dof == 0 ) ) @@ -1678,14 +2279,56 @@ ivas_error IVAS_DEC_GetObjectMetadata( } else { - metadata->azimuth = hIsmMeta->azimuth; - metadata->elevation = hIsmMeta->elevation; - metadata->radius = hIsmMeta->radius; - metadata->yaw = hIsmMeta->yaw; - metadata->pitch = hIsmMeta->pitch; - metadata->spread = 0.f; - metadata->gainFactor = 1.f; - metadata->non_diegetic_flag = hIsmMeta->non_diegetic_flag; +#ifdef OBJ_EDITING_API + if ( st_ivas->ism_mode == ISM_MODE_DISC ) + { + metadata->azimuth = hIsmMeta->edited_azimuth; + metadata->elevation = hIsmMeta->edited_elevation; + metadata->radius = hIsmMeta->edited_radius; + + metadata->yaw = hIsmMeta->edited_yaw; + metadata->pitch = hIsmMeta->edited_pitch; + metadata->spread = 0.f; + + metadata->gainFactor = hIsmMeta->edited_gain; + metadata->non_diegetic_flag = hIsmMeta->non_diegetic_flag; + } +#ifdef OBJ_EDITING_API + else if ( st_ivas->ism_mode == ISM_MODE_PARAM ) + { + metadata->azimuth = st_ivas->hParamIsmDec->edited_azimuth_values[objectIdx]; + metadata->elevation = st_ivas->hParamIsmDec->edited_elevation_values[objectIdx]; + metadata->radius = hIsmMeta->radius; + metadata->yaw = hIsmMeta->yaw; + metadata->pitch = hIsmMeta->pitch; + metadata->spread = 0.f; + metadata->gainFactor = 1.f; + metadata->non_diegetic_flag = hIsmMeta->non_diegetic_flag; + } +#endif + else if ( st_ivas->ism_mode == ISM_SBA_MODE_DISC || st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + metadata->azimuth = st_ivas->hIsmMetaData[objectIdx]->edited_azimuth; + metadata->elevation = st_ivas->hIsmMetaData[objectIdx]->edited_elevation; + metadata->radius = st_ivas->hIsmMetaData[objectIdx]->edited_radius; + metadata->yaw = st_ivas->hIsmMetaData[objectIdx]->edited_yaw; + metadata->pitch = st_ivas->hIsmMetaData[objectIdx]->edited_pitch; + metadata->spread = 0.f; + metadata->gainFactor = st_ivas->hIsmMetaData[objectIdx]->edited_gain; + metadata->non_diegetic_flag = st_ivas->hIsmMetaData[objectIdx]->non_diegetic_flag; + } + else +#endif + { + metadata->azimuth = hIsmMeta->azimuth; + metadata->elevation = hIsmMeta->elevation; + metadata->radius = hIsmMeta->radius; + metadata->yaw = hIsmMeta->yaw; + metadata->pitch = hIsmMeta->pitch; + metadata->spread = 0.f; + metadata->gainFactor = 1.f; + metadata->non_diegetic_flag = hIsmMeta->non_diegetic_flag; + } } return IVAS_ERR_OK; @@ -2559,6 +3202,11 @@ ivas_error IVAS_DEC_VoIP_GetSamples( JbmTraceFileWriterFn jbmWriterFn, void *jbmWriter #endif +#ifdef OBJ_EDITING_API + , + uint16_t *nSamplesRendered, + bool *parametersAvailableForEditing +#endif ) { Decoder_Struct *st_ivas; @@ -2569,14 +3217,21 @@ ivas_error IVAS_DEC_VoIP_GetSamples( uint16_t extBufferedSamples; int16_t result; ivas_error error; +#ifndef OBJ_EDITING_API int16_t nSamplesRendered; +#endif uint8_t nOutChannels; st_ivas = hIvasDec->st_ivas; hDecoderConfig = st_ivas->hDecoderConfig; hVoIP = hIvasDec->hVoIP; nOutChannels = (uint8_t) st_ivas->hDecoderConfig->nchan_out; +#ifndef OBJ_EDITING_API nSamplesRendered = 0; +#endif +#ifdef OBJ_EDITING_API + *parametersAvailableForEditing = false; +#endif if ( nSamplesPerChannel == 0 ) { @@ -2592,7 +3247,11 @@ ivas_error IVAS_DEC_VoIP_GetSamples( #endif /* make sure that the FIFO after decoder/scaler contains at least one sound card frame (i.e. 20ms) */ +#ifdef OBJ_EDITING_API + while ( *nSamplesRendered < nSamplesPerChannel ) +#else while ( nSamplesRendered < nSamplesPerChannel ) +#endif { if ( hIvasDec->nSamplesAvailableNext == 0 ) { @@ -2658,7 +3317,13 @@ ivas_error IVAS_DEC_VoIP_GetSamples( bsCompactToSerial( dataUnit->data, hIvasDec->hVoIP->bs_conversion_buf, dataUnit->dataSize ); - if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, hIvasDec->hVoIP->bs_conversion_buf, dataUnit->dataSize, 0 ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, hIvasDec->hVoIP->bs_conversion_buf, dataUnit->dataSize, 0 +#if defined( OBJ_EDITING_API ) + , + 0, + 0 +#endif + ) ) != IVAS_ERR_OK ) { return error; } @@ -2666,7 +3331,13 @@ ivas_error IVAS_DEC_VoIP_GetSamples( else if ( hIvasDec->hasDecodedFirstGoodFrame ) { /* Decoder has been initialized with first good frame - do PLC */ - if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, hIvasDec->hVoIP->bs_conversion_buf, 0, 1 ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, hIvasDec->hVoIP->bs_conversion_buf, 0, 1 +#if defined( OBJ_EDITING_API ) + , + 0, + 0 +#endif + ) ) != IVAS_ERR_OK ) { return error; } @@ -2705,6 +3376,14 @@ ivas_error IVAS_DEC_VoIP_GetSamples( hIvasDec->nSamplesAvailableNext = hIvasDec->nSamplesFrame; hIvasDec->nSamplesRendered = 0; } +#ifdef OBJ_EDITING_API + /* :TODO: only return here if we really have editing initialized */ + if ( hIvasDec->hasBeenFedFirstGoodFrame && hIvasDec->hasEditableParameters == true && hIvasDec->enableParameterEditing == true ) + { + *parametersAvailableForEditing = true; + return IVAS_ERR_OK; + } +#endif } /* decode */ @@ -2713,8 +3392,14 @@ ivas_error IVAS_DEC_VoIP_GetSamples( /* codec mode to use not known yet - simply output silence */ /* directly set output zero */ int16_t nSamplesToZero = min( nSamplesPerChannel, hIvasDec->nSamplesAvailableNext ); +#ifdef OBJ_EDITING_API + set_pcm_buffer_to_zero( pcm_buffer_offset( pcmBuf, pcmType, ( *nSamplesRendered ) * nOutChannels ), pcmType, nSamplesToZero * nOutChannels ); + *nSamplesRendered += nSamplesToZero; +#else set_pcm_buffer_to_zero( pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ), pcmType, nSamplesToZero * nOutChannels ); nSamplesRendered += nSamplesToZero; +#endif + hIvasDec->nSamplesRendered += nSamplesToZero; hIvasDec->nSamplesAvailableNext -= nSamplesToZero; update_voip_rendered20ms( hIvasDec, nSamplesToZero ); @@ -2723,15 +3408,39 @@ ivas_error IVAS_DEC_VoIP_GetSamples( { int16_t nSamplesToRender, nSamplesRendered_loop; bool tmp; +#ifdef OBJ_EDITING_API + nSamplesToRender = nSamplesPerChannel - *nSamplesRendered; +#else nSamplesToRender = nSamplesPerChannel - nSamplesRendered; +#endif + +#ifdef OBJ_EDITING_API + /* check if we still need to prepare the renderer */ + if ( hIvasDec->hasBeenPreparedRendering == false ) + { + + if ( ( error = IVAS_DEC_PrepareRenderer( hIvasDec ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif /* render IVAS frames directly to the output buffer */ +#ifdef OBJ_EDITING_API + if ( ( error = IVAS_DEC_GetSamples( hIvasDec, nSamplesToRender, pcmType, pcm_buffer_offset( pcmBuf, pcmType, *nSamplesRendered * nOutChannels ), &nSamplesRendered_loop, &tmp ) ) != IVAS_ERR_OK ) +#else if ( ( error = IVAS_DEC_GetSamples( hIvasDec, nSamplesToRender, pcmType, pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ), &nSamplesRendered_loop, &tmp ) ) != IVAS_ERR_OK ) +#endif { return error; } +#ifdef OBJ_EDITING_API + *nSamplesRendered += nSamplesRendered_loop; +#else nSamplesRendered += nSamplesRendered_loop; +#endif update_voip_rendered20ms( hIvasDec, nSamplesRendered_loop ); } } diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index 8390671276fac806adacf6dadb74e1d0cfd35250..dafae6518ed95256ce27d9c077a1dc2996b11647 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -150,8 +150,29 @@ ivas_error IVAS_DEC_FeedFrame_Serial( uint16_t *serial, /* i : buffer containing serial input bitstream. Each bit should be stored as a single uint16_t value */ const uint16_t num_bits, /* i : number of bits in input bitstream */ int16_t bfi /* i : bad frame indicator flag */ +#if defined( OBJ_EDITING_API ) + , + int16_t isSplitRend, + ISAR_SPLIT_REND_BITS_DATA *splitRendBits +#endif +); + +#ifdef OBJ_EDITING_API +ivas_error IVAS_DEC_GetEditableParameters( + IVAS_DEC_HANDLE hIvasDec, + IVAS_EDITABLE_PARAMETERS *hIvasEditableParameters +); + +ivas_error IVAS_DEC_SetEditableParameters( + IVAS_DEC_HANDLE hIvasDec, + IVAS_EDITABLE_PARAMETERS hIvasEditableParameters ); +ivas_error IVAS_DEC_PrepareRenderer( + IVAS_DEC_HANDLE hIvasDec +); +#endif + /*! r: decoder error code */ ivas_error IVAS_DEC_GetSamples( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ @@ -283,6 +304,11 @@ ivas_error IVAS_DEC_VoIP_GetSamples( , JbmTraceFileWriterFn jbmWriterFn, void* jbmWriter #endif +#ifdef OBJ_EDITING_API + , + uint16_t *nSamplesRendered, + bool *parametersAvailableForEditing +#endif ); ivas_error IVAS_DEC_Flush( diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index 55e5cf2dc9bcd5d4ae0d131aac38b65c01ccbddb..19f6c7e5540bc60f457cdc551b387749f86b5e43 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -723,10 +723,21 @@ static void ivas_dirac_dec_binaural_internal( ivas_sba_prototype_renderer( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe ); } +#ifdef OMASA_OBJECT_EDITING +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( ( st_ivas->ivas_format == MASA_ISM_FORMAT && ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) ) ) ) || st_ivas->ivas_format == ISM_FORMAT ) +#else + if ( st_ivas->ivas_format == MASA_ISM_FORMAT && ( st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ || ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) ) ) ) +#endif + { + ivas_omasa_preProcessStereoTransportsForEditedObjects( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, subframe ); + } +#else if ( st_ivas->ivas_format == MASA_ISM_FORMAT && nchan_transport == 2 && st_ivas->ism_mode != ISM_MASA_MODE_DISC && st_ivas->ism_mode != ISM_MASA_MODE_MASA_ONE_OBJ ) { ivas_omasa_preProcessStereoTransportsForMovedObjects( st_ivas, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, nBins, subframe ); } +#endif if ( hCombinedOrientationData ) { @@ -1203,11 +1214,19 @@ static void ivas_dirac_dec_binaural_formulate_target_covariance_matrices( uint16_t ismDirIndex; ismDirIndex = dirIndex - hSpatParamRendCom->numParametricDirections; assert( hMasaIsmData != NULL && "hMasaIsmData should not be NULL if we use it" ); +#ifdef OMASA_OBJECT_EDITING + if ( hMasaIsmData->ism_dir_is_edited[ismDirIndex] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[ismDirIndex]; + eleDeg = hMasaIsmData->elevation_ism_edited[ismDirIndex]; + } +#else if ( hMasaIsmData->ism_is_edited[ismDirIndex] ) { aziDeg = hMasaIsmData->azimuth_ism_edited[ismDirIndex]; eleDeg = hMasaIsmData->elevation_ism_edited[ismDirIndex]; } +#endif else { aziDeg = hMasaIsmData->azimuth_ism[ismDirIndex][dirac_read_idx]; @@ -1641,13 +1660,39 @@ static void ivas_dirac_dec_binaural_determine_processing_matrices( { if ( ism_mode == ISM_MASA_MODE_DISC ) { +#ifdef OMASA_OBJECT_EDITING + if ( hMasaIsmData->ism_dir_is_edited[chB] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[chB]; + eleDeg = hMasaIsmData->elevation_ism_edited[chB]; + } + else + { + aziDeg = hMasaIsmData->azimuth_ism[chB][dirac_read_idx]; + eleDeg = hMasaIsmData->elevation_ism[chB][dirac_read_idx]; + } +#else aziDeg = hMasaIsmData->azimuth_ism[chB][dirac_read_idx]; eleDeg = hMasaIsmData->elevation_ism[chB][dirac_read_idx]; +#endif } else { +#ifdef OMASA_OBJECT_EDITING + if ( ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ && hMasaIsmData->ism_dir_is_edited[hMasaIsmData->idx_separated_ism] ) + { + aziDeg = hMasaIsmData->azimuth_ism_edited[hMasaIsmData->idx_separated_ism]; + eleDeg = hMasaIsmData->elevation_ism_edited[hMasaIsmData->idx_separated_ism]; + } + else + { + aziDeg = hMasaIsmData->azimuth_separated_ism[dirac_read_idx]; + eleDeg = hMasaIsmData->elevation_separated_ism[dirac_read_idx]; + } +#else aziDeg = hMasaIsmData->azimuth_separated_ism[dirac_read_idx]; eleDeg = hMasaIsmData->elevation_separated_ism[dirac_read_idx]; +#endif instantChange = 1; } } @@ -2422,6 +2467,53 @@ static void getDirectPartGains( if ( renderStereoOutputInsteadOfBinaural ) /* In stereo (i.e. non-binaural) rendering mode */ { +#ifdef OMASA_OBJECT_EDITING + *lImagp = 0.0f; + *rImagp = 0.0f; + if ( aziDeg == gainCache->azi && eleDeg == gainCache->ele ) + { + *lRealp = gainCache->shVec[0]; /* Reused memory */ + *rRealp = gainCache->shVec[1]; /* Reused memory */ + } + else + { + /* Convert azi and ele to an azi value of the cone of confusion */ + aziRad = (float) aziDeg * PI_OVER_180; + eleRad = (float) eleDeg * PI_OVER_180; + y = ( sinf( aziRad ) * cosf( eleRad ) ); + mappedX = sqrtf( max( 0.0f, 1.0f - ( y * y ) ) ); + aziRadMapped = atan2f( y, mappedX ); + + /* Determine the real valued amplitude panning gains */ + if ( aziRadMapped >= LsAngleRad ) + { /* Left side */ + *lRealp = 1.0f; + *rRealp = 0.0f; + } + else if ( aziRadMapped <= -LsAngleRad ) + { /* Right side */ + *lRealp = 0.0f; + *rRealp = 1.0f; + } + else /* Tangent panning law */ + { + A = tanf( aziRadMapped ) / tanf( LsAngleRad ); + A2 = ( A - 1.0f ) / max( 0.001f, A + 1.0f ); + A3 = 1.0f / ( A2 * A2 + 1.0f ); + *lRealp = sqrtf( A3 ); + *rRealp = sqrtf( 1.0f - A3 ); + } + /* Scaling to have the same expected gain as for the HRTF rendering */ + *lRealp *= SQRT2; + *rRealp *= SQRT2; + + /* Store to gain cache */ + gainCache->azi = aziDeg; + gainCache->ele = eleDeg; + gainCache->shVec[0] = *lRealp; /* Reuse memory */ + gainCache->shVec[1] = *rRealp; /* Reuse memory */ + } +#else /* Convert azi and ele to an azi value of the cone of confusion */ aziRad = (float) aziDeg * PI_OVER_180; eleRad = (float) eleDeg * PI_OVER_180; @@ -2454,6 +2546,7 @@ static void getDirectPartGains( /* Scaling to have the same expected gain as for the HRTF rendering */ *lRealp *= SQRT2; *rRealp *= SQRT2; +#endif } else /* In regular binaural rendering mode */ { @@ -2607,6 +2700,685 @@ float configure_reqularization_factor( } +#ifdef OMASA_OBJECT_EDITING +/*-------------------------------------------------------------------* + * ivas_omasa_preProcessStereoTransportsForEditedObjects() + * + * + *-------------------------------------------------------------------*/ + +void ivas_omasa_preProcessStereoTransportsForEditedObjects( + Decoder_Struct *st_ivas, + float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + float inIm[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX], + const int16_t nBins, + const int16_t subframe ) +{ + int16_t bin, ch, inCh, outCh, ismDirIndex, slot, band_idx, bin_lo, bin_hi, max_band, n_ism; + SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; + MASA_ISM_DATA_HANDLE hMasaIsmData; +#ifdef OBJ_EDITING_PARAMISM_BIN + PARAM_ISM_DEC_HANDLE hParamIsmDec; +#endif + uint8_t enableCentering; + int16_t dirac_read_idx; + int16_t nSlots; + float panGainsOut[4][2]; + float panGainsIn[4][2]; + float panEnesOut[4][2]; + float panEnesIn[4][2]; + uint8_t ismGainEdited[4]; + uint8_t ismDirEdited[4]; + uint8_t masaGainEdited; +#ifdef OBJ_EDITING_PARAMISM_BIN + uint8_t masaIsmMode; +#endif + + hSpatParamRendCom = st_ivas->hSpatParamRendCom; + hMasaIsmData = st_ivas->hMasaIsmData; +#ifdef OBJ_EDITING_PARAMISM_BIN + hParamIsmDec = st_ivas->hParamIsmDec; + + if ( st_ivas->ivas_format == MASA_ISM_FORMAT ) + { + masaIsmMode = 1u; + } + else + { + masaIsmMode = 0u; + } +#endif + + if ( st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_FOA || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_HOA2 || st_ivas->hDecoderConfig->output_config == IVAS_AUDIO_CONFIG_HOA3 ) + { + enableCentering = 0; + } + else + { + enableCentering = 1; + } + +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC || st_ivas->ivas_format == ISM_FORMAT ) +#else + if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC ) +#endif + { + n_ism = st_ivas->nchan_ism; + } + else + { + n_ism = hSpatParamRendCom->numIsmDirections; + } + + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + ismGainEdited[ismDirIndex] = hMasaIsmData->ism_gain_is_edited[ismDirIndex]; + ismDirEdited[ismDirIndex] = hMasaIsmData->ism_dir_is_edited[ismDirIndex]; + } + masaGainEdited = hMasaIsmData->masa_gain_is_edited; + + /* Bypass processing until first object is moved or gained */ + if ( hMasaIsmData->objectsEdited == 0 ) + { + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + if ( ismDirEdited[ismDirIndex] ) + { + hMasaIsmData->objectsEdited = 1; + } + + if ( ismGainEdited[ismDirIndex] ) + { + hMasaIsmData->objectsEdited = 1; + } + } + + if ( masaGainEdited ) + { + hMasaIsmData->objectsEdited = 1; + } + + if ( hMasaIsmData->objectsEdited == 0 ) + { + /* No objects have moved so far */ + return; + } + } + + /* OMASA gaining for discrete OMASA mode with stereo_param/bin_room_param renderer */ + if ( ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) && st_ivas->ism_mode == ISM_MASA_MODE_DISC ) + { + /* ISM gaining */ + for ( ch = 0; ch < n_ism; ch++ ) + { + if ( st_ivas->hMasaIsmData->ism_gain_is_edited[ch] ) + { + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + float gain; + gain = st_ivas->hMasaIsmData->gain_ism_edited[ch]; + v_multc( inRe[ch + 2][slot], gain, inRe[ch + 2][slot], nBins ); + v_multc( inIm[ch + 2][slot], gain, inIm[ch + 2][slot], nBins ); + } + } + } + + /* MASA gaining */ + for ( ch = 0; ch < 2; ch++ ) + { + if ( masaGainEdited ) + { + for ( slot = 0; slot < hSpatParamRendCom->subframe_nbslots[subframe]; slot++ ) + { + float gain_masa; + gain_masa = st_ivas->hMasaIsmData->gain_masa_edited; + v_multc( inRe[ch][slot], gain_masa, inRe[ch][slot], nBins ); + v_multc( inIm[ch][slot], gain_masa, inIm[ch][slot], nBins ); + } + } + } + } +#ifdef OBJ_EDITING_PARAMISM_BIN + else if ( st_ivas->ivas_format == ISM_FORMAT && st_ivas->hISMDTX.dtx_flag ) /* If dtx on, perform just the smoothing of the processing gains */ + { + float totalTargetEne; + float ismPreprocMtxNew[2][2]; + float ismPreprocMtxIncrement[2][2]; + float nSlotDiv; + float outSlotRe[2][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float outSlotIm[2][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float eqVal; + + nSlots = hSpatParamRendCom->subframe_nbslots[subframe]; + nSlotDiv = 1.0f / ( (float) nSlots ); + + /* Use diagonal mixing matrix as the instant mixing matrix, to slowly fade away the editing during dtx */ + for ( ch = 0; ch < 2; ch++ ) + { + ismPreprocMtxNew[ch][ch] = 1.0f; + ismPreprocMtxNew[1 - ch][ch] = 0.0f; + } + + /* Determine the highest band */ + max_band = 0; + while ( max_band < MAX_PARAM_ISM_NBANDS && hParamIsmDec->hParamIsm->band_grouping[max_band] < nBins ) + { + max_band++; + } + + /* Init out array */ + for ( int k = 0; k < nSlots; k++ ) + { + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + set_zero( outSlotRe[ch][k], CLDFB_NO_CHANNELS_MAX ); + set_zero( outSlotIm[ch][k], CLDFB_NO_CHANNELS_MAX ); + } + } + + /* Perform the processing in frequency bands */ + for ( band_idx = 0; band_idx < max_band; band_idx++ ) + { + bin_lo = hParamIsmDec->hParamIsm->band_grouping[band_idx]; + bin_hi = min( hParamIsmDec->hParamIsm->band_grouping[band_idx + 1], nBins ); + + /* Determine transport energies */ + totalTargetEne = 0.0f; + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + totalTargetEne += inRe[ch][slot][bin] * inRe[ch][slot][bin]; + totalTargetEne += inIm[ch][slot][bin] * inIm[ch][slot][bin]; + } + } + } + + /* Get increment value for temporal interpolation */ + for ( inCh = 0; inCh < 2; inCh++ ) + { + for ( outCh = 0; outCh < 2; outCh++ ) + { + ismPreprocMtxIncrement[outCh][inCh] = ( ismPreprocMtxNew[outCh][inCh] - hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx] ) * nSlotDiv; + } + } + + /* Mix signals */ + hMasaIsmData->preprocEneTarget[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->preprocEneRealized[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->preprocEneTarget[band_idx] += totalTargetEne; + + for ( outCh = 0; outCh < 2; outCh++ ) + { + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( inCh = 0; inCh < 2; inCh++ ) + { + hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx] += ismPreprocMtxIncrement[outCh][inCh]; + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + outSlotRe[outCh][slot][bin] += inRe[inCh][slot][bin] * hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx]; + outSlotIm[outCh][slot][bin] += inIm[inCh][slot][bin] * hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx]; + } + } + + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + hMasaIsmData->preprocEneRealized[band_idx] += ( outSlotRe[outCh][slot][bin] * outSlotRe[outCh][slot][bin] ) + ( outSlotIm[outCh][slot][bin] * outSlotIm[outCh][slot][bin] ); + } + } + } + + eqVal = fminf( 4.0f, sqrtf( hMasaIsmData->preprocEneTarget[band_idx] / fmaxf( 1e-12f, hMasaIsmData->preprocEneRealized[band_idx] ) ) ); + + for ( ch = 0; ch < 2; ch++ ) + { + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[ch][slot][bin] = outSlotRe[ch][slot][bin] * eqVal; + inIm[ch][slot][bin] = outSlotIm[ch][slot][bin] * eqVal; + } + } + } + } + } +#endif + else /* Other processing modes */ + { + float subframeEne; + float subframeEneCh[2]; + float normEnes[2]; + float ratioAccOrig; + float ratioAccNew; + float ratio; + float ismEneThis; + float ismTargetEneThis; + float ismTargetEneThisCh[2]; + float totalTargetEneCh[2]; + float totalTargetEne; + float masaTargetEneThisCh[2]; + float ismPreprocMtxNew[2][2]; + float ismPreprocMtxIncrement[2][2]; + float eneMove[2]; + float enePreserve[2]; + float ismRatioAcc; + float remainderNormEne; + float centeringFactor; + float eneMoveThis; + float enePreserveThis; + float normVal; + float eqVal; + float outSlotRe[2][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float outSlotIm[2][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX]; + float newRatios[6]; + float gainIsmThis; + float gainMasaPow2; + float nSlotDiv; + float tempDivisor; + float masaEneThisCh[2]; + float ratioAccNewDivisor; + + + gainMasaPow2 = 1.0f; + if ( masaGainEdited ) + { + gainMasaPow2 = hMasaIsmData->gain_masa_edited; + gainMasaPow2 *= gainMasaPow2; + } + + nSlots = hSpatParamRendCom->subframe_nbslots[subframe]; + dirac_read_idx = hSpatParamRendCom->render_to_md_map[subframe]; + nSlotDiv = 1.0f / ( (float) nSlots ); + + /* Determine panning gains and energies for each object */ + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + /* Get input and output panning gains */ + ivas_get_stereo_panning_gains( hMasaIsmData->azimuth_ism[ismDirIndex][dirac_read_idx], + hMasaIsmData->elevation_ism[ismDirIndex][dirac_read_idx], + panGainsIn[ismDirIndex] ); + + if ( ismDirEdited[ismDirIndex] ) + { + ivas_get_stereo_panning_gains( hMasaIsmData->azimuth_ism_edited[ismDirIndex], + hMasaIsmData->elevation_ism_edited[ismDirIndex], + panGainsOut[ismDirIndex] ); + } + else + { + /* When not edited, input and output pan gains are the same */ + for ( ch = 0; ch < 2; ch++ ) + { + panGainsOut[ismDirIndex][ch] = panGainsIn[ismDirIndex][ch]; + } + } + /* Determine pan enes */ + for ( ch = 0; ch < 2; ch++ ) + { + panEnesOut[ismDirIndex][ch] = panGainsOut[ismDirIndex][ch] * panGainsOut[ismDirIndex][ch]; + panEnesIn[ismDirIndex][ch] = panGainsIn[ismDirIndex][ch] * panGainsIn[ismDirIndex][ch]; + } + } + + /* Determine the highest band */ + max_band = 0; +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( masaIsmMode ) + { +#endif + while ( max_band < MASA_FREQUENCY_BANDS && MASA_band_grouping_24[max_band] < nBins ) + { + max_band++; + } +#ifdef OBJ_EDITING_PARAMISM_BIN + } + else + { + while ( max_band < MAX_PARAM_ISM_NBANDS && hParamIsmDec->hParamIsm->band_grouping[max_band] < nBins ) + { + max_band++; + } + } +#endif + + /* Init out array */ + for ( int k = 0; k < nSlots; k++ ) + { + for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ ) + { + set_zero( outSlotRe[ch][k], CLDFB_NO_CHANNELS_MAX ); + set_zero( outSlotIm[ch][k], CLDFB_NO_CHANNELS_MAX ); + } + } + + /* Perform the processing in frequency bands */ + for ( band_idx = 0; band_idx < max_band; band_idx++ ) + { + ratioAccOrig = 0.0f; + ratioAccNew = 0.0f; + ismRatioAcc = 0.0f; + + set_zero( subframeEneCh, 2 ); + set_zero( ismPreprocMtxNew[0], 2 ); + set_zero( ismPreprocMtxNew[1], 2 ); + set_zero( eneMove, 2 ); + set_zero( enePreserve, 2 ); +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( masaIsmMode ) + { +#endif + bin_lo = MASA_band_grouping_24[band_idx]; + bin_hi = min( MASA_band_grouping_24[band_idx + 1], nBins ); +#ifdef OBJ_EDITING_PARAMISM_BIN + } + else + { + bin_lo = hParamIsmDec->hParamIsm->band_grouping[band_idx]; + bin_hi = min( hParamIsmDec->hParamIsm->band_grouping[band_idx + 1], nBins ); + } +#endif + + /* Determine transport normalized energies and subframe energy */ + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( ch = 0; ch < 2; ch++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + subframeEneCh[ch] += inRe[ch][slot][bin] * inRe[ch][slot][bin]; + subframeEneCh[ch] += inIm[ch][slot][bin] * inIm[ch][slot][bin]; + } + } + } + subframeEne = subframeEneCh[0] + subframeEneCh[1]; + totalTargetEneCh[0] = subframeEneCh[0]; + totalTargetEneCh[1] = subframeEneCh[1]; + masaEneThisCh[0] = subframeEneCh[0]; + masaEneThisCh[1] = subframeEneCh[1]; + + /* Gain editing */ + /* For each object, estimate new target energy per channel based on the applied gain */ + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + ratio = hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin_lo]; + ratioAccOrig += ratio; + + /* Calculate MASA energy as a residual of original channel energies subtracted with ISM energies */ + for ( ch = 0; ch < 2; ch++ ) + { + masaEneThisCh[ch] -= panEnesIn[ismDirIndex][ch] * ratio * subframeEne; + } + + /* Calculate target energy, gained ratio for accumulation, and transports gains, if ism gain is edited */ + if ( ismGainEdited[ismDirIndex] ) + { + gainIsmThis = hMasaIsmData->gain_ism_edited[ismDirIndex]; + /* ISM original energy */ + ismEneThis = ratio * subframeEne; + + /* ISM target energy */ + ismTargetEneThis = ( gainIsmThis * gainIsmThis ) * ismEneThis; + + ratio *= gainIsmThis * gainIsmThis; + + /* Determine panning energies and channel target energies */ + for ( ch = 0; ch < 2; ch++ ) + { + ismTargetEneThisCh[ch] = panEnesIn[ismDirIndex][ch] * ismTargetEneThis; /* Ism target energy per channel */ + totalTargetEneCh[ch] -= panEnesIn[ismDirIndex][ch] * ismEneThis; /* Reduce original ism energy */ + totalTargetEneCh[ch] += ismTargetEneThisCh[ch]; /* Add ism target energy per channel */ + } + + /* If separated ism edited, apply gain directly to the separated ism */ + if ( ismDirIndex == hMasaIsmData->idx_separated_ism ) + { + /* Separated object gaining is done elsewhere with DIRAC renderer */ + if ( st_ivas->renderer_type != RENDERER_DIRAC ) + { + /* Gain transport channel of separated ism */ + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[2][slot][bin] = gainIsmThis * inRe[2][slot][bin]; + inIm[2][slot][bin] = gainIsmThis * inIm[2][slot][bin]; + } + } + } + } + } + ratioAccNew += ratio; + } + +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( masaIsmMode ) + { +#endif + /* MASA original ratios */ + ratio = hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin_lo] + hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin_lo]; + ratioAccOrig += ratio; + + /* Estimate MASA original energies and determine adjusted target energies and gained ratio for accumulation, if a gain is applied to MASA */ + if ( masaGainEdited ) + { + /* Gained MASA ratio */ + ratio *= gainMasaPow2; + + /* Calculate MASA target energies and add to total target energy estimation */ + for ( ch = 0; ch < 2; ch++ ) + { + masaEneThisCh[ch] = fmaxf( masaEneThisCh[ch], 0.0f ); /* MASA original energy per channel */ + masaTargetEneThisCh[ch] = gainMasaPow2 * masaEneThisCh[ch]; /* MASA target energy per channel */ + totalTargetEneCh[ch] -= masaEneThisCh[ch]; /* Reduce original energy per channel */ + totalTargetEneCh[ch] += masaTargetEneThisCh[ch]; /* Add target energy per channel */ + } + } + + ratioAccNew += ratio; +#ifdef OBJ_EDITING_PARAMISM_BIN + } +#endif + + /* Limit target energies to non-negative values */ + for ( ch = 0; ch < 2; ch++ ) + { + totalTargetEneCh[ch] = max( totalTargetEneCh[ch], 0.0f ); + } + + /* Diffuse ratio accumulation based on gaining */ + if ( masaGainEdited ) + { + ratioAccNew += gainMasaPow2 * ( 1 - ratioAccOrig ); + } + else + { + ratioAccNew += ( 1.0f - ratioAccOrig ); + } + + ratioAccNewDivisor = 1.0f / fmaxf( 1e-12f, ratioAccNew ); /* New target total energy ratio divider */ + + /* Determine and process object energy ratios based on gaining */ + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + newRatios[ismDirIndex + 2] = hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin_lo]; /* Determine original object energy ratio */ + if ( ismGainEdited[ismDirIndex] ) + { + gainIsmThis = hMasaIsmData->gain_ism_edited[ismDirIndex]; + newRatios[ismDirIndex + 2] *= ( gainIsmThis * gainIsmThis ); /* Gain original object energy ratio, if edited */ + } + newRatios[ismDirIndex + 2] *= ratioAccNewDivisor; /* Divide with new target total ratio */ + } + + /* Determine and process MASA energy ratios based on gaining */ + newRatios[0] = hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin_lo]; + newRatios[1] = hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin_lo]; + if ( masaGainEdited ) + { + newRatios[0] *= gainMasaPow2; + newRatios[1] *= gainMasaPow2; + } + newRatios[0] *= ratioAccNewDivisor; + newRatios[1] *= ratioAccNewDivisor; + + /* Set adjusted energy ratios */ + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin] = newRatios[ismDirIndex + 2]; + } + hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin] = newRatios[0]; + hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin] = newRatios[1]; + } + + /* Direction editing */ + /* Determine new energetic values after gaining */ + totalTargetEne = totalTargetEneCh[0] + totalTargetEneCh[1]; + tempDivisor = 1.0f / fmaxf( 1e-12f, totalTargetEne ); + normEnes[0] = totalTargetEneCh[0] * tempDivisor; + normEnes[1] = totalTargetEneCh[1] * tempDivisor; + + /* For each ismDir, determine moved and preserve energy ratio per channel */ + for ( ismDirIndex = 0; ismDirIndex < n_ism; ismDirIndex++ ) + { + ratio = hMasaIsmData->energy_ratio_ism[ismDirIndex][dirac_read_idx][bin_lo]; + + ismRatioAcc += ratio; + + if ( enableCentering ) + { + centeringFactor = fmaxf( 0.0f, 2.0f * fabsf( panEnesIn[ismDirIndex][0] - panEnesOut[ismDirIndex][0] ) - 1.0f ); + for ( ch = 0; ch < 2; ch++ ) + { + panEnesOut[ismDirIndex][ch] *= ( 1.0f - centeringFactor ); + panEnesOut[ismDirIndex][ch] += 0.5f * centeringFactor; + } + } + + for ( ch = 0; ch < 2; ch++ ) + { + eneMoveThis = fmaxf( 0.0f, panEnesIn[ismDirIndex][ch] - panEnesOut[ismDirIndex][ch] ); + enePreserveThis = panEnesIn[ismDirIndex][ch] - eneMoveThis; + + eneMove[ch] += ratio * eneMoveThis; + enePreserve[ch] += ratio * enePreserveThis; + + /* Subtract object parts from normEnes */ + normEnes[ch] -= panEnesIn[ismDirIndex][ch] * ratio; + } + } + + /* Any remaining (non-object) energy is set to be preserved at both channels */ + remainderNormEne = fmaxf( 0.0f, ( 1.0f - ismRatioAcc ) - normEnes[0] - normEnes[1] ); + + /* Normalize */ + for ( ch = 0; ch < 2; ch++ ) + { + enePreserve[ch] += fmaxf( 0.0f, normEnes[ch] + remainderNormEne / 2.0f ); + normVal = 1.0f / fmaxf( EPSILON, eneMove[ch] + enePreserve[ch] ); + normVal *= fminf( 10.0f, totalTargetEneCh[ch] / fmaxf( EPSILON, subframeEneCh[ch] ) ); + eneMove[ch] *= normVal; + enePreserve[ch] *= normVal; + } + + /* Temporally average target energy */ + hMasaIsmData->eneOrigIIR[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->eneOrigIIR[band_idx] += totalTargetEne; + + /* Temporally average energy moving and preserving, and generate the transport signal preprocessing matrix for + * gaining objects and moving objects between left and right */ + for ( ch = 0; ch < 2; ch++ ) + { + hMasaIsmData->eneMoveIIR[ch][band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->eneMoveIIR[ch][band_idx] += eneMove[ch] * totalTargetEne; + hMasaIsmData->enePreserveIIR[ch][band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->enePreserveIIR[ch][band_idx] += enePreserve[ch] * totalTargetEne; + normVal = 1.0f / fmaxf( EPSILON, hMasaIsmData->eneOrigIIR[band_idx] ); + ismPreprocMtxNew[ch][ch] = fminf( 4.0f, sqrtf( hMasaIsmData->enePreserveIIR[ch][band_idx] * normVal ) ); + ismPreprocMtxNew[1 - ch][ch] = fminf( 4.0f, sqrtf( hMasaIsmData->eneMoveIIR[ch][band_idx] * normVal ) ); + } + + /* Get increment value for temporal interpolation */ + for ( inCh = 0; inCh < 2; inCh++ ) + { + for ( outCh = 0; outCh < 2; outCh++ ) + { + ismPreprocMtxIncrement[outCh][inCh] = ( ismPreprocMtxNew[outCh][inCh] - hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx] ) * nSlotDiv; + } + } + + /* Mix signals */ + hMasaIsmData->preprocEneTarget[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->preprocEneRealized[band_idx] *= STEREO_PREPROCESS_IIR_FACTOR; + hMasaIsmData->preprocEneTarget[band_idx] += totalTargetEne; + + for ( outCh = 0; outCh < 2; outCh++ ) + { + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( inCh = 0; inCh < 2; inCh++ ) + { + hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx] += ismPreprocMtxIncrement[outCh][inCh]; + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + outSlotRe[outCh][slot][bin] += inRe[inCh][slot][bin] * hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx]; + outSlotIm[outCh][slot][bin] += inIm[inCh][slot][bin] * hMasaIsmData->ismPreprocMatrix[outCh][inCh][band_idx]; + } + } + + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + hMasaIsmData->preprocEneRealized[band_idx] += ( outSlotRe[outCh][slot][bin] * outSlotRe[outCh][slot][bin] ) + ( outSlotIm[outCh][slot][bin] * outSlotIm[outCh][slot][bin] ); + } + } + } + + eqVal = fminf( 4.0f, sqrtf( hMasaIsmData->preprocEneTarget[band_idx] / fmaxf( 1e-12f, hMasaIsmData->preprocEneRealized[band_idx] ) ) ); + + for ( ch = 0; ch < 2; ch++ ) + { + for ( slot = 0; slot < nSlots; slot++ ) + { + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + inRe[ch][slot][bin] = outSlotRe[ch][slot][bin] * eqVal; + inIm[ch][slot][bin] = outSlotIm[ch][slot][bin] * eqVal; + } + } + } + } + +#ifdef OBJ_EDITING_PARAMISM_BIN + if ( !masaIsmMode ) + { + int16_t obj_idx1, obj_idx2; + + for ( band_idx = 0; band_idx < hParamIsmDec->hParamIsm->nbands; band_idx++ ) + { + bin_lo = hParamIsmDec->hParamIsm->band_grouping[band_idx]; + bin_hi = hParamIsmDec->hParamIsm->band_grouping[band_idx + 1]; + + obj_idx1 = hParamIsmDec->hParamIsm->obj_indices[band_idx][0][0]; + obj_idx2 = hParamIsmDec->hParamIsm->obj_indices[band_idx][0][1]; + + for ( bin = bin_lo; bin < bin_hi; bin++ ) + { + hSpatParamRendCom->energy_ratio1[dirac_read_idx][bin] = hMasaIsmData->energy_ratio_ism[obj_idx1][dirac_read_idx][bin]; + hSpatParamRendCom->energy_ratio2[dirac_read_idx][bin] = hMasaIsmData->energy_ratio_ism[obj_idx2][dirac_read_idx][bin]; + } + } + } +#endif + } + + return; +} +#else /*-------------------------------------------------------------------* * ivas_omasa_preProcessStereoTransportsForMovedObjects() * @@ -2831,6 +3603,8 @@ void ivas_omasa_preProcessStereoTransportsForMovedObjects( return; } +#endif + static void ivas_masa_ext_rend_parambin_internal( MASA_EXT_REND_HANDLE hMasaExtRend, diff --git a/lib_rend/ivas_dirac_output_synthesis_dec.c b/lib_rend/ivas_dirac_output_synthesis_dec.c index 54ea30216527a106c7381be3fc52a28c32bb77e3..0bfd43ddd89a8e565575583a7fd51d85010572fe 100644 --- a/lib_rend/ivas_dirac_output_synthesis_dec.c +++ b/lib_rend/ivas_dirac_output_synthesis_dec.c @@ -1700,10 +1700,17 @@ void ivas_dirac_dec_compute_directional_responses( for ( dir = 0; dir < hSpatParamRendCom->numIsmDirections; dir++ ) { +#ifdef OMASA_OBJECT_EDITING + if ( hMasaIsm->ism_dir_is_edited[dir] ) + { + ivas_dirac_dec_get_response( hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], direct_response_temp, hDirACRend->hOutSetup.ambisonics_order ); + } +#else if ( hMasaIsm->ism_is_edited[dir] ) { ivas_dirac_dec_get_response( hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], direct_response_temp, hDirACRend->hOutSetup.ambisonics_order ); } +#endif else { ivas_dirac_dec_get_response( hMasaIsm->azimuth_ism[dir][md_idx], hMasaIsm->elevation_ism[dir][md_idx], direct_response_temp, hDirACRend->hOutSetup.ambisonics_order ); @@ -1810,10 +1817,17 @@ void ivas_dirac_dec_compute_directional_responses( for ( dir = 0; dir < hSpatParamRendCom->numIsmDirections; dir++ ) { +#ifdef OMASA_OBJECT_EDITING + if ( hMasaIsm->ism_dir_is_edited[dir] ) + { + vbap_determine_gains( hVBAPdata, direct_response_temp, hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], 1 ); + } +#else if ( hMasaIsm->ism_is_edited[dir] ) { vbap_determine_gains( hVBAPdata, direct_response_temp, hMasaIsm->azimuth_ism_edited[dir], hMasaIsm->elevation_ism_edited[dir], 1 ); } +#endif else { vbap_determine_gains( hVBAPdata, direct_response_temp, hMasaIsm->azimuth_ism[dir][md_idx], hMasaIsm->elevation_ism[dir][md_idx], 1 ); diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index 30b0a4b3c7e68485d6890076a46652a1aaf2195b..dbe5cc8d0675ff9b442b11d7bfd4bd1e558fe8e6 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -503,6 +503,9 @@ static void TDREND_Clear_Update_flags( for ( i = 0; i < hBinRendererTd->NumOfSrcs; i++ ) { hBinRendererTd->Sources[i]->SrcSpatial_p->Updated = FALSE; +#ifdef OBJ_EDITING_API + hBinRendererTd->Sources[i]->SrcRend_p->SrcGainUpdated = FALSE; +#endif } return; @@ -549,6 +552,12 @@ ivas_error TDREND_Update_object_positions( { return error; } +#ifdef OBJ_EDITING_INTERFACE + if ( ( error = TDREND_MIX_SRC_SetGain( hBinRendererTd, nS, hIsmMetaData[nS]->gain ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif if ( hIsmMetaData[nS]->non_diegetic_flag ) { @@ -774,6 +783,10 @@ ivas_error ivas_td_binaural_renderer_ext( hIsmMetaData[0]->yaw = currentPos->yaw; hIsmMetaData[0]->pitch = currentPos->pitch; hIsmMetaData[0]->radius = currentPos->radius; +#ifdef OBJ_EDITING_INTERFACE + hIsmMetaData[0]->gain = 1.0f; +#endif + hIsmMetaData[0]->non_diegetic_flag = currentPos->non_diegetic_flag; } diff --git a/lib_rend/ivas_objectRenderer_sources.c b/lib_rend/ivas_objectRenderer_sources.c index 0dd1ba482761931ba4dc90351207233b659f1539..4f570e6182897732a508e0767ec18c75affb86d8 100644 --- a/lib_rend/ivas_objectRenderer_sources.c +++ b/lib_rend/ivas_objectRenderer_sources.c @@ -53,6 +53,10 @@ static void TDREND_SRC_SPATIAL_SetDirAtten( TDREND_SRC_SPATIAL_t *SrcSpatial_p, static void TDREND_SRC_SPATIAL_SetDistAtten( TDREND_SRC_SPATIAL_t *SrcSpatial_p, const TDREND_DistAtten_t *DistAtten_p ); +#ifdef OBJ_EDITING_INTERFACE +static void TDREND_SRC_SPATIAL_SetGain( const TDREND_SRC_t *Src_p, const float Gain ); +#endif + static float TDREND_SRC_SPATIAL_GetDirGain( const TDREND_DirAtten_t *DirAtten_p, const float *Front_p, const float *RelPos_p ); static float TDREND_SRC_SPATIAL_GetDistGain( const TDREND_DistAtten_t *DistAtten_p, const float Dist ); @@ -128,6 +132,24 @@ ivas_error TDREND_MIX_SRC_SetDir( return IVAS_ERR_OK; } +#ifdef OBJ_EDITING_INTERFACE +/*-------------------------------------------------------------------* + * TDREND_MIX_SRC_SetSourceGain() + * + * Set source gain + --------------------------------------------------------------------*/ + +ivas_error TDREND_MIX_SRC_SetGain( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ + const int16_t SrcInd, /* i : Source index */ + const float Gain /* i : Gain */ +) +{ + TDREND_SRC_SPATIAL_SetGain( hBinRendererTd->Sources[SrcInd], Gain ); + + return IVAS_ERR_OK; +} +#endif /*-------------------------------------------------------------------* * TDREND_MIX_SRC_SetDirAtten() @@ -251,7 +273,11 @@ static void TDREND_SRC_REND_Init( { SrcRend_p->SrcGainMin_p[nC] = 0.0f; SrcRend_p->SrcGain_p[nC] = 1.0f; +#ifdef OBJ_EDITING_API + SrcRend_p->SrcGainMax_p[nC] = 2.0f; +#else SrcRend_p->SrcGainMax_p[nC] = 1.0f; +#endif } SrcRend_p->SrcGainUpdated = FALSE; @@ -529,6 +555,24 @@ static void TDREND_SRC_SPATIAL_SetDistAtten( return; } + + +#ifdef OBJ_EDITING_INTERFACE +/*-------------------------------------------------------------------* + * TDREND_SRC_SPATIAL_SetGain() + * + * Set the object gain + --------------------------------------------------------------------*/ + +static void TDREND_SRC_SPATIAL_SetGain( + const TDREND_SRC_t *Src_p, /* i : Directional attenuation specification */ + const float Gain /* i : Front-pointing vector */ +) +{ + Src_p->SrcRend_p->SrcGain_p[0] = Gain; +} +#endif + /*-------------------------------------------------------------------* * TDREND_SRC_SPATIAL_GetDirGain() * diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index a42f893c6e7284f62b9050909d37e4f15651e376..8e6980b0548fda0ab1a9c66dc70c00b7c2b01446 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -720,6 +720,14 @@ ivas_error TDREND_MIX_SRC_SetDir( const float *Vec_p /* i : Direction vector */ ); +#ifdef OBJ_EDITING_INTERFACE +ivas_error TDREND_MIX_SRC_SetGain( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ + const int16_t SrcInd, /* i : Source index */ + const float Gain /* i : Gain */ +); +#endif + ivas_error TDREND_MIX_SRC_SetDirAtten( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ const int16_t SrcInd, /* i : Source index */ @@ -736,6 +744,14 @@ ivas_error TDREND_MIX_SRC_SetPlayState( const TDREND_PlayStatus_t PlayStatus /* i : Play state */ ); +#ifdef OBJ_EDITING_API +ivas_error TDREND_MIX_SRC_SetSrcGain( + BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ + const int16_t SrcInd, /* i : Source index */ + const float SrcGain_p /* i : Source gain */ +); +#endif + void TDREND_SRC_REND_UpdateFiltersFromSpatialParams( BINAURAL_TD_OBJECT_RENDERER_HANDLE hBinRendererTd, /* i/o: TD renderer handle */ TDREND_SRC_REND_t *SrcRend_p, /* i/o: Source object */