Fix split rendering with 5ms framing (WIP) (2bd8ba59) · Commits · IVAS Codec Public Collaboration / IVAS Codec

apps/decoder.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -2130,7 +2130,7 @@ static ivas_error decodeG192(
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		if ( arg.outputFormat == IVAS_DEC_OUTPUT_SPLIT_BINAURAL_CODED \|\| arg.outputFormat == IVAS_DEC_OUTPUT_SPLIT_BINAURAL_PCM )
		{
		error = IVAS_DEC_GetSplitBinaural( hIvasDec, &splitRendBits, &nSamplesRendered_loop, &needNewFrame );
		error = IVAS_DEC_GetSplitBinauralBitstream( hIvasDec, &splitRendBits, &nSamplesRendered_loop, &needNewFrame );
		nSamplesRendered += nSamplesRendered_loop;
		nSamplesToRender -= nSamplesRendered_loop;
		if ( error != IVAS_ERR_OK )

apps/renderer.c

+60 −8

Original line number	Diff line number	Diff line
		@@ -804,6 +804,9 @@ int main(
		#ifndef API_5MS
		IVAS_VECTOR3 Pos[RENDERER_HEAD_POSITIONS_PER_FRAME];
		#endif
		#ifdef API_5MS
		bool splitBinNeedsNewFrame = true;
		#endif

		#ifdef WMOPS
		reset_wmops();
		@@ -1474,7 +1477,11 @@ int main(

		#ifdef SPLIT_REND_WITH_HEAD_ROT
		numSamplesRead = 0;
		if ( ( hSplitRendFileReadWrite != NULL ) && is_split_post_rend_mode( &args ) )
		if ( ( hSplitRendFileReadWrite != NULL ) && is_split_post_rend_mode( &args )
		#ifdef API_5MS
		&& splitBinNeedsNewFrame
		#endif
		)
		{
		ivas_error error_tmp;
		numSamplesRead = (int16_t) inBufferSize;
		@@ -1506,7 +1513,11 @@ int main(
		}
		#endif

		if ( numSamplesRead == 0 )
		if ( numSamplesRead == 0
		#ifdef API_5MS
		&& splitBinNeedsNewFrame /* TODO(sgi): clean up */
		#endif
		)
		{
		/* end of input data */
		break;
		@@ -1645,7 +1656,11 @@ int main(

		#ifdef SPLIT_REND_WITH_HEAD_ROT
		/* Read from split renderer bfi file if specified */
		if ( splitRendBFIReader != NULL )
		if ( splitRendBFIReader != NULL
		#ifdef API_5MS
		&& splitBinNeedsNewFrame
		#endif
		)
		{
		int16_t bfi;
		SplitRendBFIFileReading( splitRendBFIReader, &bfi );
		@@ -1821,6 +1836,16 @@ int main(
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		for ( i = 0; i < args.inConfig.numBinBuses; ++i )
		{
		#ifdef API_5MS
		if ( splitBinNeedsNewFrame )
		{
		if ( ( error = IVAS_REND_FeedSplitBinauralBitstream( hIvasRend, splitBinIds[i], &bitsBuffer ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) );
		exit( -1 );
		}
		}
		#else
		if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, splitBinIds[i], &numChannels ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) );
		@@ -1833,13 +1858,37 @@ int main(
		fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) );
		exit( -1 );
		}
		#endif
		}
		#endif

		#ifdef API_5MS
		if ( args.inConfig.numBinBuses != 0 )
		{
		if ( ( error = IVAS_REND_GetSplitBinauralSamples( hIvasRend, outBuffer, &splitBinNeedsNewFrame ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) );
		exit( -1 );
		}
		}
		else if ( args.outConfig.audioConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_SPLIT_CODED \|\|
		args.outConfig.audioConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_SPLIT_PCM )
		{
		if ( ( error = IVAS_REND_GetSplitBinauralBitstream( hIvasRend, &bitsBuffer ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) );
		exit( -1 );
		}
		}
		else
		{
		#endif
		if ( ( error = IVAS_REND_GetSamples( hIvasRend, outBuffer
		#ifndef API_5MS
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		,
		&bitsBuffer
		#endif
		#endif
		) ) != IVAS_ERR_OK )
		{
		@@ -1850,6 +1899,9 @@ int main(
		#endif
		exit( -1 );
		}
		#ifdef API_5MS
		}
		#endif

		int16_t num_out_channels;
		num_out_channels = outBuffer.config.numChannels;

lib_dec/lib_dec.c

+56 −55

Original line number	Diff line number	Diff line
		@@ -1116,7 +1116,7 @@ ivas_error IVAS_DEC_GetSamples(
		#endif

		#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT
		ivas_error IVAS_DEC_GetSplitBinaural(
		ivas_error IVAS_DEC_GetSplitBinauralBitstream(
		IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */
		IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits, /* o : bitstream output for split rendering mode*/
		int16_t nOutSamples, / o : number of samples per channel written to output buffer */
		@@ -1131,6 +1131,11 @@ ivas_error IVAS_DEC_GetSplitBinaural(
		int16_t numSamplesPerChannel;
		int16_t i, j;
		ivas_error error;
		IVAS_DEC_SPLIT_REND_WRAPPER *hSplitBinRend;
		int16_t max_band;
		int16_t pcm_out;
		int16_t td_input;
		int16_t numPoses;

		error = IVAS_ERR_OK;
		st_ivas = hIvasDec->st_ivas;
		@@ -1143,20 +1148,8 @@ ivas_error IVAS_DEC_GetSplitBinaural(
		return IVAS_ERR_WRONG_PARAMS;
		}

		*nOutSamples = 0;
		*needNewFrame = FALSE;

		while ( error == IVAS_ERR_OK && !needNewFrame /TODO(jbm): && numOutSamples < numAskedSamples */ )
		{
		int16_t nOutSamplesLocal;
		IVAS_DEC_SPLIT_REND_WRAPPER *hSplitBinRend;
		int16_t max_band;
		int16_t pcm_out;
		int16_t td_input;
		int16_t numPoses;

		hSplitBinRend = &st_ivas->splitBinRend;

		ivas_set_split_rend_setup( hSplitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, hSplitRendBits );

		/* Decode and render */
		@@ -1164,31 +1157,40 @@ ivas_error IVAS_DEC_GetSplitBinaural(
		hIvasDec,
		numSamplesPerChannel,
		output_int,
		&nOutSamplesLocal,
		nOutSamples,
		needNewFrame );
		if ( error != IVAS_ERR_OK )
		{
		return error;
		}

		*nOutSamples += nOutSamplesLocal;
		assert( numSamplesPerChannel == *nOutSamples );
		#ifdef DEBUGGING
		dbgwrite(output_int, sizeof(int16_t), 960 * 2, 1, "res/output_int.raw");
		#endif

		numPoses = hSplitBinRend->splitrend.multiBinPoseData.num_poses;

		/* [tmp] convert int back to float and change buffer layout */
		for ( i = 0; i < nOutSamplesLocal; ++i )
		for ( i = 0; i < *nOutSamples; ++i )
		{
		#ifdef DEBUGGING
		for ( j = 0; j < BINAURAL_CHANNELS * numPoses; ++j )
		{
		output[j][i] = (float) output_int[i * BINAURAL_CHANNELS * numPoses + j];
		output[j][i] = 0;
		}
		#endif
		for ( j = 0; j < BINAURAL_CHANNELS /* * numPoses */; ++j )
		{
		output[j][i] = (float) output_int[i * BINAURAL_CHANNELS /* * numPoses */ + j];
		}

		}
		#ifdef DEBUGGING
		dbgwrite(output[0], sizeof(float), 960, 1, "res/output.raw");
		#endif
		max_band = (int16_t) ( ( BINAURAL_MAXBANDS * output_Fs ) / 48000 );
		pcm_out = ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0;
		td_input = st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC;

		/* TODO(jbm): need to wait 20ms before writing split bitstream in 5ms mode */
		error = ivas_renderMultiBinToSplitBinaural( &hSplitBinRend->splitrend,
		st_ivas->hHeadTrackData->Quaternion,
		st_ivas->hRenderConfig->split_rend_config.splitRendBitRate,
		@@ -1205,7 +1207,6 @@ ivas_error IVAS_DEC_GetSplitBinaural(
		}

		free( st_ivas->splitBinRend.hMultiBinCldfbData );
		}

		return error;
		}

lib_dec/lib_dec.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -201,10 +201,10 @@ ivas_error IVAS_DEC_GetSamples(
		);

		#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT
		ivas_error IVAS_DEC_GetSplitBinaural(
		ivas_error IVAS_DEC_GetSplitBinauralBitstream(
		IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */
		IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits, /* o : bitstream output for split rendering mode*/
		int16_t nOutSamples, / o : number of samples per channel written to output buffer */
		int16_t nOutSamples, / o : number of samples per channel written to output buffer / / TODO(sgi): might not be necessary - revisit */
		bool needNewFrame / indication that the decoder needs a new frame */
		);
		#endif

lib_rend/ivas_prot_rend.h

+14 −1

Original line number	Diff line number	Diff line
		@@ -1536,9 +1536,15 @@ void ivas_rend_CldfbSplitPreRendProcess(
		void ivas_rend_CldfbSplitPostRendProcess(
		BIN_HR_SPLIT_POST_REND_HANDLE hBinHrSplitPostRend,
		MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData,
		#ifdef API_5MS
		const IVAS_QUATERNION QuaternionPost,
		float Cldfb_RealBuffer_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX],
		float Cldfb_ImagBuffer_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX],
		#else
		const IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES],
		float Cldfb_RealBuffer_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX],
		float Cldfb_ImagBuffer_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX],
		#endif
		float output[][L_FRAME48k],
		const int16_t is_cldfb_in );

		@@ -1604,9 +1610,16 @@ void ivas_SplitRenderer_GetRotMd(
		void ivas_SplitRenderer_PostRenderer(
		BIN_HR_SPLIT_POST_REND_HANDLE hBinPostRenderer, /* i/o: binaural renderer handle */
		MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData,
		#ifdef API_5MS
		float Cldfb_RealBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */
		float Cldfb_ImagBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */
		const IVAS_QUATERNION Quaternion_act
		#else
		float Cldfb_RealBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */
		float Cldfb_ImagBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */
		const IVAS_QUATERNION Quaternions_act[MAX_PARAM_SPATIAL_SUBFRAMES]);
		const IVAS_QUATERNION Quaternions_act[MAX_PARAM_SPATIAL_SUBFRAMES]
		#endif
		);

		#endif /* SPLIT_REND_WITH_HEAD_ROT */
		/----------------------------------------------------------------------------------