Merge branch '488-waveform-and-md-desynchronized-in-external-renderer' into 'main' (e5b8527a) · Commits · IVAS Codec Public Collaboration / IVAS Codec

apps/renderer.c

+37 −0

Original line number	Diff line number	Diff line
		@@ -175,6 +175,9 @@ typedef struct
		float lfeConfigElevation;
		bool lfeCustomRoutingEnabled;
		char inLfePanningMatrixFile[RENDERER_MAX_CLI_ARG_LENGTH];
		#ifdef FIX_488_SYNC_DELAY
		float syncMdDelay;
		#endif
		} CmdlnArgs;

		typedef enum
		@@ -203,6 +206,9 @@ typedef enum
		#endif
		CmdLnOptionId_referenceVectorFile,
		CmdLnOptionId_exteriorOrientationFile,
		#ifdef FIX_488_SYNC_DELAY
		CmdLnOptionId_syncMdDelay,
		#endif
		} CmdLnOptionId;

		static const CmdLnParser_Option cliOptions[] = {
		@@ -342,6 +348,14 @@ static const CmdLnParser_Option cliOptions[] = {
		.matchShort = "exof",
		.description = "External orientation trajectory file for simulation of external orientations",
		},
		#ifdef FIX_488_SYNC_DELAY
		{
		.id = CmdLnOptionId_syncMdDelay,
		.match = "sync_md_delay",
		.matchShort = "smd",
		.description = "Metadata Synchronization Delay in ms, Default is 0. Quantized by 5ms subframes for TDRenderer (13ms -> 10ms -> 2subframes)",
		},
		#endif
		};


		@@ -1113,6 +1127,9 @@ int main(
		if ( args.inConfig.numAudioObjects > 0 )
		{
		IVAS_REND_SetTotalNumberOfObjects( hIvasRend, args.inConfig.numAudioObjects );
		#ifdef FIX_488_SYNC_DELAY
		IVAS_REND_SetIsmMetadataDelay( hIvasRend, args.syncMdDelay );
		#endif
		}

		IVAS_REND_LfePanMtx lfePanMatrix;
		@@ -1687,6 +1704,8 @@ int main(
		}
		}
		}


		#ifdef SPLIT_REND_WITH_HEAD_ROT
		for ( i = 0; i < args.inConfig.numBinBuses; ++i )
		{
		@@ -1897,6 +1916,13 @@ int main(
		}
		#endif

		#ifdef FIX_488_SYNC_DELAY
		if ( args.inConfig.numAudioObjects != 0 && ( args.outConfig.audioConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL \|\| args.outConfig.audioConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) )
		{
		fprintf( stdout, "\n\nMetadata delayed %d subframes\n\n", (int16_t) round( args.syncMdDelay / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ) );
		}
		#endif

		if ( !args.quietModeEnabled && args.delayCompensationEnabled )
		{
		fprintf( stdout, "\nRenderer delay: %-5u [samples] - Timescale: %5u\n", delayNumSamples_orig, delayTimeScale );
		@@ -2483,6 +2509,10 @@ static CmdlnArgs defaultArgs(

		args.lfeCustomRoutingEnabled = false;
		clearString( args.inLfePanningMatrixFile );

		#ifdef FIX_488_SYNC_DELAY
		args.syncMdDelay = 0;
		#endif
		return args;
		}

		@@ -2622,6 +2652,13 @@ static void parseOption(
		exit( -1 );
		}
		break;
		#ifdef FIX_488_SYNC_DELAY
		case CmdLnOptionId_syncMdDelay:
		assert( numOptionValues == 1 );
		/* Metadata Delay to sync with audio delay in ms */
		args->syncMdDelay = strtof( optionValues[0], NULL );
		break;
		#endif
		default:
		assert( 0 && "This should be unreachable - all command line options should be explicitly handled." );
		break;

lib_com/options.h

+17 −0

Original line number	Diff line number	Diff line
		@@ -158,9 +158,26 @@
		#define FIX_559_EXTL_IGF_MISMATCH /* VA: issue 559: fix mismatch between st->extl and st->igf observed as crash in PlanarSBA bitrate switching */
		#define FIX_571_REVERB_NOT_ACTIVATED_ISM /* Philips: Issue 571: Reverb not activated for discrete and parametric ISM */
		#define FIX_QMETA_SID_5k2 /* Nokia: Issue 137: enable using full 5.2k bitrate in MASA SID */

		#define FIX_578_PARAMMC_ILD_BS /* FhG: Issue 578: transmitt also center ILD in band 0 when LFE is active in 3TC ParamMC */
		#define FIX_UNCLR_ISSUE /* VoiceAge: issue 574: Fix UNCLR mis-classifications in noisy speech stereo */
		#define FIX_TCX_LOWRATE_LIMITATION /* VA: issue 577: TCX bitrate limitation only when DEBUGGING is active */
		#define FIX_575_LOW_OVERLAP_PLC_RECOVERY /* FhG: Issue 575 fix for PLC and transistion to TCX5*/

		#define FIX_488_SYNC_DELAY /* Eri: Issue 488: Waveform and MD desynchronized in external renderer */

		#define FIX_550_FIRST_FRAME_ACCESS /* Eri: Issue 550: TD Object renderer: first frame accesses wrong transport channel offsets */
		#define FIX_550_FIRST_FRAME_ACCESS_ALT /* Eri: Issue 550: Should be merged with FIX_550_FIRST_FRAME_ACCESS above, or accepted at the same time */
		#define FIX_569_TD_FILTER_LENGTH /* Eri: Issue 569: If an HRTF binary file exceeds the SFX_SPAT_BIN_MAX_FILTER_LENGTH the decoder crashes. This truncates the filter when generated from the model. */

		#define ISM_FB_16k4 /* VA: Issue: 579: change BW from SWB to FB in NxISM conditions to match the EVS codec */

		#define FIX_580_PARAMMC_ENER_BURSTS /* FhG: issue 580: energy bursts due to ILD holding when energy relations change too much */

		#define FIX_550_FIRST_FRAME_ACCESS /* Eri: Issue 550: TD Object renderer: first frame accesses wrong transport channel offsets */
		#define FIX_550_FIRST_FRAME_ACCESS_ALT /* Eri: Issue 550: Should be merged with FIX_550_FIRST_FRAME_ACCESS above, or accepted at the same time */
		#define FIX_569_TD_FILTER_LENGTH /* Eri: Issue 569: If an HRTF binary file exceeds the SFX_SPAT_BIN_MAX_FILTER_LENGTH the decoder crashes. This truncates the filter when generated from the model. */

		#define FIX_595_SHL_NOGLOB /* FhG: Issue 595: compilation with BASOP_NOGLOB disabled */
		#define UPDATE_FASTCONV_SBA_FILTER /* Dlb: Issue 584: Update SBA CLDFB-Domain HRTFs */
		#define FIX_570_SF_EXT_ORIENTATION

lib_rend/ivas_objectRenderer.c

+10 −2

Original line number	Diff line number	Diff line
		@@ -681,6 +681,9 @@ ivas_error ivas_td_binaural_renderer_ext(
		const COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, / i : Combined head and external orientations */
		const IVAS_REND_AudioObjectPosition currentPos, / i : Object position */
		const REVERB_HANDLE hReverb, /* i : Reverberator handle */
		#ifdef FIX_488_SYNC_DELAY
		const int16_t ism_md_subframe_update_ext, /* i: Metadata Delay in subframes to sync with audio delay */
		#endif
		const int16_t output_frame, /* i : output frame length */
		float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */
		)
		@@ -693,7 +696,9 @@ ivas_error ivas_td_binaural_renderer_ext(
		IVAS_REND_AudioConfigType inConfigType;
		AUDIO_CONFIG transport_config;
		ivas_error error;
		#ifndef FIX_488_SYNC_DELAY
		int16_t ism_md_subframe_update_ext;
		#endif
		float *p_output[MAX_OUTPUT_CHANNELS];
		int16_t ch;

		@@ -707,7 +712,10 @@ ivas_error ivas_td_binaural_renderer_ext(
		inConfigType = getAudioConfigType( inConfig );
		lfe_idx = LFE_CHANNEL;
		hIsmMetaData[0] = NULL;

		#ifndef FIX_488_SYNC_DELAY
		ism_md_subframe_update_ext = 0;
		#endif

		if ( inConfigType == IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED )
		{

lib_rend/ivas_prot_rend.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -579,6 +579,9 @@ ivas_error ivas_td_binaural_renderer_ext(
		const COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, / i : Combined head and external orientations */
		const IVAS_REND_AudioObjectPosition currentPos, / i : Object position */
		const REVERB_HANDLE hReverb, /* i : Reverberator handle */
		#ifdef FIX_488_SYNC_DELAY
		const int16_t ism_md_subframe_update_ext, /* i: Metadata Delay in subframes to sync with audio delay */
		#endif
		const int16_t output_frame, /* i : output frame length */
		float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */
		);

lib_rend/lib_rend.c

+78 −5

Original line number	Diff line number	Diff line
		@@ -142,6 +142,9 @@ typedef struct
		float nonDiegeticPanGain;
		OMASA_ANA_HANDLE hOMasa;
		uint16_t total_num_objects;
		#ifdef FIX_488_SYNC_DELAY
		float ism_metadata_delay_ms;
		#endif
		} input_ism;

		typedef struct
		@@ -5585,9 +5588,14 @@ static ivas_error renderIsmToBinaural(
		{
		float tmpTDRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k];
		ivas_error error;

		#ifdef FIX_488_SYNC_DELAY
		int16_t ism_md_subframe_update_ext;
		#endif
		push_wmops( "renderIsmToBinaural" );

		#ifdef FIX_488_SYNC_DELAY
		/* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */
		ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) );
		#endif
		copyBufferTo2dArray( ismInput->base.inputBuffer, tmpTDRendBuffer );

		if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper,
		@@ -5596,6 +5604,9 @@ static ivas_error renderIsmToBinaural(
		ismInput->base.ctx.pCombinedOrientationData,
		&ismInput->currentPos,
		ismInput->hReverb,
		#ifdef FIX_488_SYNC_DELAY
		ism_md_subframe_update_ext,
		#endif
		outAudio.config.numSamplesPerChannel,
		tmpTDRendBuffer ) ) != IVAS_ERR_OK )
		{
		@@ -5761,9 +5772,16 @@ static ivas_error renderIsmToBinauralReverb(
		{
		float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k];
		ivas_error error;
		#ifdef FIX_488_SYNC_DELAY
		int16_t ism_md_subframe_update_ext;
		#endif

		push_wmops( "renderIsmToBinauralRoom" );

		#ifdef FIX_488_SYNC_DELAY
		/* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */
		ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) );
		#endif
		copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer );

		if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper,
		@@ -5772,6 +5790,9 @@ static ivas_error renderIsmToBinauralReverb(
		ismInput->base.ctx.pCombinedOrientationData,
		&ismInput->currentPos,
		ismInput->hReverb,
		#ifdef FIX_488_SYNC_DELAY
		ism_md_subframe_update_ext,
		#endif
		outAudio.config.numSamplesPerChannel,
		tmpRendBuffer ) ) != IVAS_ERR_OK )
		{
		@@ -5915,11 +5936,18 @@ static ivas_error renderIsmToSplitBinaural(
		float tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k];
		int16_t output_frame = ismInput->base.inputBuffer.config.numSamplesPerChannel;
		COMBINED_ORIENTATION_HANDLE pCombinedOrientationData;
		#ifdef FIX_488_SYNC_DELAY
		int16_t ism_md_subframe_update_ext;
		#endif

		push_wmops( "renderIsmToSplitBinaural" );

		pSplitRendWrapper = ismInput->base.ctx.pSplitRendWrapper;
		pMultiBinPoseData = &pSplitRendWrapper->multiBinPoseData;
		#ifdef FIX_488_SYNC_DELAY
		/* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */
		ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) );
		#endif

		/* If not yet allocated, open additional instances of TD renderer */
		for ( i = 0; i < pMultiBinPoseData->num_poses - 1; ++i )
		@@ -5998,6 +6026,9 @@ static ivas_error renderIsmToSplitBinaural(
		ismInput->base.ctx.pCombinedOrientationData,
		&ismInput->currentPos,
		NULL,
		#ifdef FIX_488_SYNC_DELAY
		ism_md_subframe_update_ext,
		#endif
		output_frame,
		tmpProcessing );
		if ( error != IVAS_ERR_OK )
		@@ -6013,6 +6044,9 @@ static ivas_error renderIsmToSplitBinaural(
		ismInput->base.ctx.pCombinedOrientationData,
		&ismInput->currentPos,
		NULL,
		#ifdef FIX_488_SYNC_DELAY
		ism_md_subframe_update_ext,
		#endif
		output_frame,
		tmpProcessing );
		if ( error != IVAS_ERR_OK )
		@@ -6141,6 +6175,7 @@ static ivas_error renderActiveInputsIsm(
		}

		if ( ( error = renderInputIsm( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK )

		{
		return error;
		}
		@@ -6283,7 +6318,11 @@ static ivas_error renderMcToBinaural(
		if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput,
		mcInput->base.ctx.pCombinedOrientationData,
		NULL,
		mcInput->hReverb, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK )
		mcInput->hReverb,
		#ifdef FIX_488_SYNC_DELAY
		0,
		#endif
		mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK )
		{
		return error;
		}
		@@ -6387,7 +6426,11 @@ static ivas_error renderMcToBinauralRoom(

		if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput,
		mcInput->base.ctx.pCombinedOrientationData,
		NULL, mcInput->hReverb, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK )
		NULL, mcInput->hReverb,
		#ifdef FIX_488_SYNC_DELAY
		0,
		#endif
		mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK )
		{
		return error;
		}
		@@ -6699,6 +6742,9 @@ static ivas_error renderMcToSplitBinaural(
		&pCombinedOrientationDataLocal,
		NULL,
		mcInput->hReverb,
		#ifdef FIX_488_SYNC_DELAY
		0, /* Ism Audio Metadata Delay Sync in ms for External Renderer */
		#endif
		mcInput->base.inputBuffer.config.numSamplesPerChannel,
		tmpRendBuffer ) ) != IVAS_ERR_OK )
		{
		@@ -8157,6 +8203,33 @@ ivas_error IVAS_REND_SetTotalNumberOfObjects(
		return IVAS_ERR_OK;
		}

		#ifdef FIX_488_SYNC_DELAY
		/---------------------------------------------------------------------
		* IVAS_REND_SetIsmMetadataDelay( )
		*
		* Set the Metadata Delay in ms in order to sync with audio delay
		---------------------------------------------------------------------/

		ivas_error IVAS_REND_SetIsmMetadataDelay(
		IVAS_REND_HANDLE hIvasRend, /* i/o: IVAS renderer handle */
		const float sync_md_delay /* i: Ism Metadata Delay in ms to sync with audio delay */
		)
		{
		int16_t i;

		if ( hIvasRend == NULL )
		{
		return IVAS_ERR_UNEXPECTED_NULL_POINTER;
		}

		for ( i = 0; i < RENDERER_MAX_ISM_INPUTS; ++i )
		{
		hIvasRend->inputsIsm[i].ism_metadata_delay_ms = sync_md_delay;
		}

		return IVAS_ERR_OK;
		}
		#endif

		/-------------------------------------------------------------------
		* IVAS_REND_GetSamples()
		@@ -8260,6 +8333,7 @@ ivas_error IVAS_REND_GetSamples(
		/* Clear original output buffer */
		set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel );


		#ifdef SPLIT_REND_WITH_HEAD_ROT
		outAudioOrig = outAudio;
		/* Use internal buffer if outputting split rendering bitstream */
		@@ -8284,7 +8358,6 @@ ivas_error IVAS_REND_GetSamples(
		set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel );
		}
		#endif

		if ( ( error = renderActiveInputsIsm( hIvasRend, outAudio ) ) != IVAS_ERR_OK )
		{
		return error;