Fix 0 DOF split rendering [WIP] (6ce810a9) · Commits · IVAS Codec Public Collaboration / IVAS Codec

apps/decoder.c

+40 −1

Original line number	Diff line number	Diff line
		@@ -435,6 +435,45 @@ int main(
		}
		}

		#ifdef API_5MS
		/-------------------------------------------------------------------
		* Load renderer configuration from file
		--------------------------------------------------------------------/

		IVAS_RENDER_CONFIG_DATA renderConfig;
		if ( arg.renderConfigEnabled )
		{
		/* sanity check */
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		if ( arg.outputFormat != IVAS_DEC_OUTPUT_BINAURAL && arg.outputFormat != IVAS_DEC_OUTPUT_BINAURAL_ROOM_IR && arg.outputFormat != IVAS_DEC_OUTPUT_BINAURAL_ROOM_REVERB &&
		arg.outputFormat != IVAS_DEC_OUTPUT_SPLIT_BINAURAL_CODED &&
		arg.outputFormat != IVAS_DEC_OUTPUT_SPLIT_BINAURAL_PCM )
		{
		fprintf( stderr, "\nExternal Renderer Config is supported only when binaural output configurations is used as output OR when Split rendering mode is enabled. Exiting. \n" );
		exit( -1 );
		}
		#else
		if ( arg.outputFormat != IVAS_DEC_OUTPUT_BINAURAL && arg.outputFormat != IVAS_DEC_OUTPUT_BINAURAL_ROOM_IR && arg.outputFormat != IVAS_DEC_OUTPUT_BINAURAL_ROOM_REVERB )
		{
		fprintf( stderr, "\nExternal Renderer Config is supported only for binaural output configurations. Exiting. \n\n" );
		goto cleanup;
		}
		#endif

		if ( ( error = IVAS_DEC_GetDefaultRenderConfig( &renderConfig ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "\nIVAS_DEC_GetDefaultRenderConfig failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) );
		goto cleanup;
		}

		if ( RenderConfigReader_read( renderConfigReader, &renderConfig ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Failed to read renderer configuration from file %s\n\n", arg.renderConfigFilename );
		goto cleanup;
		}
		}
		#endif

		/------------------------------------------------------------------------------------------
		* Configure the decoder
		------------------------------------------------------------------------------------------/
		@@ -443,7 +482,7 @@ int main(
		{
		arg.enableHeadRotation = true;
		#ifdef API_5MS_BASELINE
		arg.enable5ms = false;
		arg.enable5ms = renderConfig.split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE \|\| renderConfig.split_rend_config.dof == 0;
		#endif
		}
		#endif

lib_dec/ivas_init_dec.c

+10 −0

Original line number	Diff line number	Diff line
		@@ -2144,6 +2144,10 @@ void ivas_initialize_handles_dec(
		st_ivas->splitBinRend.hMultiBinCldfbData = NULL;
		st_ivas->splitBinRend.hSplitRendBits = NULL;
		st_ivas->splitBinRend.hCldfbDataOut = NULL;
		#ifdef API_5MS
		st_ivas->splitBinRend.tdDataOut = NULL;
		st_ivas->splitBinRend.numTdSamplesPerChannelCached = 0;
		#endif
		ivas_init_split_rend_handles( &st_ivas->splitBinRend.splitrend );
		#endif
		#ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN
		@@ -2325,6 +2329,12 @@ void ivas_destroy_dec(
		{
		free( st_ivas->splitBinRend.hCldfbDataOut );
		}
		#ifdef API_5MS
		if ( st_ivas->splitBinRend.tdDataOut != NULL )
		{
		free( st_ivas->splitBinRend.tdDataOut );
		}
		#endif
		#endif

		/* Parametric binaural renderer handle */

lib_dec/ivas_stat_dec.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -934,6 +934,10 @@ typedef struct
		IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits; /scratch buffer for frame by frame processing/
		SPLIT_REND_WRAPPER splitrend;
		IVAS_DEC_SPLIT_REND_CLDFB_OUT_DATA_HANDLE hCldfbDataOut; /buffer to store cldfb data before binauralization/
		#ifdef API_5MS
		float* tdDataOut; /buffer to store TD data before binauralization/
		int16_t numTdSamplesPerChannelCached;
		#endif
		} IVAS_DEC_SPLIT_REND_WRAPPER;
		#endif

lib_dec/lib_dec.c

+145 −20

Original line number	Diff line number	Diff line
		@@ -1128,7 +1128,8 @@ ivas_error IVAS_DEC_GetSplitBinauralBitstream(
		int32_t output_Fs;
		float output[BINAURAL_CHANNELS * MAX_HEAD_ROT_POSES][L_FRAME48k];
		int16_t output_int[BINAURAL_CHANNELS * MAX_HEAD_ROT_POSES * L_FRAME48k]; /* TODO(sgi): Need conversion */
		int16_t numSamplesPerChannel;
		int16_t numSamplesPerChannelToDecode;
		int16_t numSamplesPerChannelToSplitEncode;
		int16_t i, j;
		ivas_error error;
		IVAS_DEC_SPLIT_REND_WRAPPER *hSplitBinRend;
		@@ -1141,21 +1142,44 @@ ivas_error IVAS_DEC_GetSplitBinauralBitstream(
		st_ivas = hIvasDec->st_ivas;
		output_config = st_ivas->hDecoderConfig->output_config;
		output_Fs = st_ivas->hDecoderConfig->output_Fs;
		numSamplesPerChannel = (int16_t) ( output_Fs / FRAMES_PER_SEC ); /* TODO(sgi): Accommodate 5ms framing */
		numSamplesPerChannelToDecode = (int16_t) ( output_Fs / FRAMES_PER_SEC );

		*needNewFrame = FALSE;
		hSplitBinRend = &st_ivas->splitBinRend;
		ivas_set_split_rend_setup( hSplitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, hSplitRendBits );
		numPoses = hSplitBinRend->splitrend.multiBinPoseData.num_poses;

		if ( hIvasDec->st_ivas->hRenderConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE \|\|
		hIvasDec->st_ivas->hRenderConfig->split_rend_config.dof == 0 )
		{
		numSamplesPerChannelToSplitEncode = (int16_t) ( output_Fs / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES );

		if ( hSplitBinRend->tdDataOut == NULL )
		{
		/* Allocate enough space to save all decoded samples that will not be split encoded directly after decoding */
		hSplitBinRend->tdDataOut = malloc( ( numSamplesPerChannelToDecode - numSamplesPerChannelToSplitEncode ) * BINAURAL_CHANNELS * numPoses * sizeof( float ) );
		if ( hSplitBinRend->tdDataOut == NULL )
		{
		return IVAS_ERR_FAILED_ALLOC;
		}
		}
		}
		else
		{
		numSamplesPerChannelToSplitEncode = (int16_t) ( output_Fs / FRAMES_PER_SEC );
		}

		if ( output_config != AUDIO_CONFIG_BINAURAL_SPLIT_CODED && output_config != AUDIO_CONFIG_BINAURAL_SPLIT_PCM )
		{
		return IVAS_ERR_WRONG_PARAMS;
		}

		*needNewFrame = FALSE;
		hSplitBinRend = &st_ivas->splitBinRend;
		ivas_set_split_rend_setup( hSplitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, hSplitRendBits );

		if ( numSamplesPerChannelToDecode == numSamplesPerChannelToSplitEncode \|\| hSplitBinRend->numTdSamplesPerChannelCached == 0 )
		{
		/* Decode and render */
		error = IVAS_DEC_GetSamples(
		hIvasDec,
		numSamplesPerChannel,
		numSamplesPerChannelToDecode,
		output_int,
		nOutSamples,
		needNewFrame );
		@@ -1163,12 +1187,44 @@ ivas_error IVAS_DEC_GetSplitBinauralBitstream(
		{
		return error;
		}
		assert( numSamplesPerChannel == *nOutSamples );
		assert( numSamplesPerChannelToDecode == *nOutSamples );
		#ifdef DEBUGGING
		dbgwrite( output_int, sizeof( int16_t ), 960 * 2, 1, "res/output_int.raw" );
		#endif

		numPoses = hSplitBinRend->splitrend.multiBinPoseData.num_poses;
		/* copy to cache if cache is in use */
		if ( hSplitBinRend->tdDataOut != NULL )
		{
		float *writePtr;
		int16_t readPtr, readEnd;
		writePtr = hSplitBinRend->tdDataOut;
		readPtr = output_int + numSamplesPerChannelToSplitEncode * BINAURAL_CHANNELS * numPoses;
		readEnd = output_int + nOutSamples BINAURAL_CHANNELS * numPoses;

		while ( readPtr != readEnd )
		{
		writePtr++ = readPtr++; /* TODO: test */
		}
		hSplitBinRend->numTdSamplesPerChannelCached = *nOutSamples - numSamplesPerChannelToSplitEncode;
		}
		}
		else
		{
		/* copy from cache */
		assert( hSplitBinRend->tdDataOut != NULL );

		int16_t *writePtr;
		float readPtr, readEnd;
		readPtr = hSplitBinRend->tdDataOut + ( numSamplesPerChannelToDecode - hSplitBinRend->numTdSamplesPerChannelCached ) * BINAURAL_CHANNELS * numPoses;
		readEnd = readPtr + numSamplesPerChannelToSplitEncode * BINAURAL_CHANNELS * numPoses;
		writePtr = output_int;

		while ( readPtr != readEnd )
		{
		writePtr++ = readPtr++; /* TODO: test */
		}
		hSplitBinRend->numTdSamplesPerChannelCached -= numSamplesPerChannelToSplitEncode;
		}

		/* [tmp] convert int back to float and change buffer layout */
		for ( i = 0; i < *nOutSamples; ++i )
		@@ -1970,6 +2026,51 @@ ivas_error IVAS_DEC_GetHrtfParamBinHandle(
		return IVAS_ERR_OK;
		}

		#ifdef API_5MS
		static ivas_error copyRendererConfigStruct( RENDER_CONFIG_HANDLE hRCin, IVAS_RENDER_CONFIG_HANDLE hRCout )
		{
		if ( hRCin == NULL \|\| hRCout == NULL )
		{
		return IVAS_ERR_UNEXPECTED_NULL_POINTER;
		}

		#ifdef DEBUGGING
		switch ( hRCin->renderer_type_override )
		{
		case RENDER_TYPE_OVERRIDE_CREND:
		hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_CREND;
		break;
		case RENDER_TYPE_OVERRIDE_FASTCONV:
		hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_FASTCONV;
		break;
		default:
		hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_NONE;
		break;
		}
		#endif
		hRCout->room_acoustics.override = hRCin->roomAcoustics.override;
		hRCout->room_acoustics.nBands = hRCin->roomAcoustics.nBands;
		hRCout->room_acoustics.acousticPreDelay = hRCin->roomAcoustics.acousticPreDelay;
		hRCout->room_acoustics.inputPreDelay = hRCin->roomAcoustics.inputPreDelay;

		mvr2r( hRCin->roomAcoustics.pFc_input, hRCout->room_acoustics.pFc_input, CLDFB_NO_CHANNELS_MAX );
		mvr2r( hRCin->roomAcoustics.pAcoustic_rt60, hRCout->room_acoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX );
		mvr2r( hRCin->roomAcoustics.pAcoustic_dsr, hRCout->room_acoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX );
		mvr2r( hRCin->directivity, hRCout->directivity, 3 );

		#ifdef SPLIT_REND_WITH_HEAD_ROT
		hRCout->split_rend_config.splitRendBitRate = SPLIT_REND_768k;
		hRCout->split_rend_config.dof = 3;
		hRCout->split_rend_config.hq_mode = 0;
		hRCout->split_rend_config.codec_delay_ms = 0;
		hRCout->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT;
		hRCout->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB;
		hRCout->split_rend_config.rendererSelection = hRCin->split_rend_config.rendererSelection;
		#endif

		return IVAS_ERR_OK;
		}
		#endif

		/---------------------------------------------------------------------
		* IVAS_DEC_GetRenderConfig( )
		@@ -1982,13 +2083,18 @@ ivas_error IVAS_DEC_GetRenderConfig(
		const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render configuration handle */
		)
		{
		#ifndef API_5MS
		RENDER_CONFIG_HANDLE hRCin;
		#endif

		if ( hIvasDec == NULL \|\| hIvasDec->st_ivas == NULL \|\| hIvasDec->st_ivas->hRenderConfig == NULL \|\| hRCout == NULL )
		{
		return IVAS_ERR_UNEXPECTED_NULL_POINTER;
		}

		#ifdef API_5MS
		return copyRendererConfigStruct( hIvasDec->st_ivas->hRenderConfig, hRCout );
		#else
		hRCin = hIvasDec->st_ivas->hRenderConfig;
		#ifdef DEBUGGING
		switch ( hRCin->renderer_type_override )
		@@ -2022,11 +2128,30 @@ ivas_error IVAS_DEC_GetRenderConfig(
		hRCout->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT;
		hRCout->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB;
		hRCout->split_rend_config.rendererSelection = hRCin->split_rend_config.rendererSelection;
		#endif
		#endif

		return IVAS_ERR_OK;
		}

		#ifdef API_5MS
		/! r: error code/
		ivas_error IVAS_DEC_GetDefaultRenderConfig(
		IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render config handle */
		)
		{
		RENDER_CONFIG_HANDLE hRCin;
		ivas_error error;

		if ( ( error = ivas_render_config_init_from_rom( &hRCin ) ) != IVAS_ERR_OK )
		{
		return error;
		}

		return copyRendererConfigStruct( hRCin, hRCout );
		}
		#endif


		/---------------------------------------------------------------------
		* IVAS_DEC_FeedRenderConfig( )

lib_dec/lib_dec.h

+7 −0

Original line number	Diff line number	Diff line
		@@ -440,6 +440,13 @@ ivas_error IVAS_DEC_GetRenderConfig(
		const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render config handle */
		);

		#ifdef API_5MS
		/! r: error code/
		ivas_error IVAS_DEC_GetDefaultRenderConfig(
		IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render config handle */
		);
		#endif

		/! r: error code/
		ivas_error IVAS_DEC_FeedRenderConfig(
		IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */