Implementation of JBM for OMASA. (e6777207) · Commits · IVAS Codec Public Collaboration / IVAS Codec

lib_com/ivas_prot.h

+32 −0

Original line number	Diff line number	Diff line
		@@ -5767,12 +5767,34 @@ ivas_error ivas_omasa_dirac_td_binaural(
		const int16_t output_frame /* i : output frame length per channel */
		);

		#ifdef MASA_AND_OBJECTS
		ivas_error ivas_omasa_dirac_td_binaural_jbm(
		Decoder_Struct st_ivas, / i/o: IVAS decoder handle */
		const uint16_t nSamplesAsked, /* i : number of samples requested */
		uint16_t nSamplesRendered, / o : number of samples rendered */
		uint16_t nSamplesAvailable, / o : number of samples still to render */
		const int16_t nchan_transport, /* i : number of transport channels */
		float output_f[] / o : rendered time signal */
		);
		#endif

		void ivas_omasa_dirac_rend(
		Decoder_Struct st_ivas, / i/o: IVAS decoder structure */
		float output[][L_FRAME48k], /* o : output synthesis signal */
		const int16_t output_frame /* i : output frame length per channel */
		);

		#ifdef MASA_AND_OBJECTS
		void ivas_omasa_dirac_rend_jbm(
		Decoder_Struct st_ivas, / i/o: IVAS decoder handle */
		const uint16_t nSamplesAsked, /* i : number of samples requested */
		uint16_t nSamplesRendered, / o : number of samples rendered */
		uint16_t nSamplesAvailable, / o : number of samples still to render */
		const int16_t nchan_transport, /* i : number of transport channels */
		float output_f[] / o : rendered time signal */
		);
		#endif

		void ivas_omasa_preProcessStereoTransportsForMovedObjects(
		Decoder_Struct *st_ivas,
		float inRe[][CLDFB_SLOTS_PER_SUBFRAME][CLDFB_NO_CHANNELS_MAX],
		@@ -5796,6 +5818,16 @@ void ivas_omasa_separate_object_render(
		const int16_t output_frame /* i : output frame length per channel */
		);

		#ifdef MASA_AND_OBJECTS
		void ivas_omasa_separate_object_render_jbm(
		Decoder_Struct st_ivas, / i/o: IVAS decoder structure */
		const uint16_t nSamplesRendered, /* i : number of samples rendered */
		float output_f[], / o : rendered time signal */
		const int16_t subframes_rendered, /* i : number of subframes rendered */
		const int16_t slots_rendered /* i : number of CLDFB slots rendered */
		);
		#endif

		void ivas_omasa_set_edited_objects(
		Decoder_Struct st_ivas / i/o: IVAS decoder structure */
		);

lib_dec/ivas_ism_renderer.c

+158 −2

Original line number	Diff line number	Diff line
		@@ -365,7 +365,6 @@ ivas_error ivas_omasa_separate_object_renderer_open(
		set_f( st_ivas->hIsmRendererData->prev_gains[i], 0.0f, MAX_OUTPUT_CHANNELS );
		}

		// Todo OMASA JBM: This needs touches for VOIP path at least. Current version is mostly an adapted copy from ivas_ism_renderer_open()
		if ( st_ivas->hDecoderConfig->voip_active )
		{
		init_interpolator_length = NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_CLDFB_TIMESLOTS * CLDFB_SLOT_NS );
		@@ -382,6 +381,7 @@ ivas_error ivas_omasa_separate_object_renderer_open(
		{
		st_ivas->hIsmRendererData->interpolator[i] = (float) i / ( (float) interpolator_length );
		}
		st_ivas->hIsmRendererData->interpolator_length = interpolator_length;

		st_ivas->hMasaIsmData->delayBuffer_size = (int16_t) ( ( st_ivas->hDecoderConfig->output_Fs / 50 ) / MAX_PARAM_SPATIAL_SUBFRAMES );

		@@ -464,7 +464,6 @@ void ivas_omasa_separate_object_renderer_close(
		* Rendering separated objects and mixing them to the parametrically rendered signals
		-------------------------------------------------------------------------/

		// Todo OMASA JBM: This might need adjustments
		void ivas_omasa_separate_object_render(
		Decoder_Struct st_ivas, / i/o: IVAS decoder structure */
		float input_f[][L_FRAME48k], /* i : separated object signal */
		@@ -569,4 +568,161 @@ void ivas_omasa_separate_object_render(

		return;
		}

		/-------------------------------------------------------------------------
		* ivas_omasa_separate_object_render_jbm()
		*
		* Rendering separated objects and mixing them to the parametrically rendered signals for JBM
		-------------------------------------------------------------------------/

		void ivas_omasa_separate_object_render_jbm(
		Decoder_Struct st_ivas, / i/o: IVAS decoder structure */
		const uint16_t nSamplesRendered, /* i : number of samples rendered */
		float output_f[], / o : rendered time signal */
		const int16_t subframes_rendered, /* i : number of subframes rendered */
		const int16_t slots_rendered /* i : number of CLDFB slots rendered */
		)
		{
		VBAP_HANDLE hVBAPdata;
		DIRAC_REND_HANDLE hDirACRend;
		SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom;
		int16_t nchan_out_woLFE, num_lfe;
		ISM_RENDERER_HANDLE hRendererData;
		int16_t j, k, j2;
		int16_t obj;
		float gains[MAX_OUTPUT_CHANNELS];
		float g1, g2;
		int16_t lfe_index;
		int16_t azimuth, elevation;
		int16_t num_objects;
		uint8_t single_separated;
		float *input_f[MAX_TRANSPORT_CHANNELS];
		float *output_f_local[MAX_OUTPUT_CHANNELS];
		int16_t offsetSamples;
		int16_t n_samples_sf, md_idx;
		int16_t slots_to_render, first_sf, last_sf, subframe_idx;

		hVBAPdata = st_ivas->hVBAPdata;
		hDirACRend = st_ivas->hDirACRend;
		hSpatParamRendCom = st_ivas->hSpatParamRendCom;
		nchan_out_woLFE = st_ivas->hIntSetup.nchan_out_woLFE;
		num_lfe = st_ivas->hIntSetup.num_lfe;
		hRendererData = st_ivas->hIsmRendererData;
		lfe_index = hDirACRend->hOutSetup.index_lfe[0];

		if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ \|\| st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ )
		{
		single_separated = 1;
		num_objects = 1;
		}
		else
		{
		single_separated = 0;
		num_objects = st_ivas->nchan_ism;
		}

		offsetSamples = slots_rendered * hSpatParamRendCom->slot_size;

		for ( j = 0; j < nchan_out_woLFE + num_lfe; j++ )
		{
		output_f_local[j] = output_f[j];
		}

		for ( obj = 0; obj < num_objects; obj++ )
		{
		input_f[obj] = &st_ivas->hTcBuffer->tc[obj + 2][offsetSamples];
		}

		slots_to_render = nSamplesRendered / hSpatParamRendCom->slot_size;
		first_sf = subframes_rendered;
		last_sf = first_sf;

		while ( slots_to_render > 0 )
		{
		slots_to_render -= hSpatParamRendCom->subframe_nbslots[last_sf];
		last_sf++;
		}

		for ( obj = 0; obj < num_objects; obj++ )
		{
		/* Delay the signal to match CLDFB delay. Delay the whole buffer with the first rendering call of the stretched buffer. */
		if ( slots_rendered == 0 )
		{
		int16_t tcBufferSize;

		tcBufferSize = hSpatParamRendCom->num_slots * hSpatParamRendCom->slot_size;
		delay_signal( input_f[obj], tcBufferSize, st_ivas->hMasaIsmData->delayBuffer[obj], st_ivas->hMasaIsmData->delayBuffer_size );
		}

		offsetSamples = 0;

		for ( subframe_idx = first_sf; subframe_idx < last_sf; subframe_idx++ )
		{
		n_samples_sf = hSpatParamRendCom->subframe_nbslots[subframe_idx] * hSpatParamRendCom->slot_size;
		if ( n_samples_sf != hRendererData->interpolator_length )
		{
		for ( k = 0; k < n_samples_sf; k++ )
		{
		hRendererData->interpolator[k] = (float) k / ( (float) n_samples_sf );
		}
		hRendererData->interpolator_length = n_samples_sf;
		}

		md_idx = hSpatParamRendCom->render_to_md_map[subframe_idx];

		if ( single_separated )
		{
		azimuth = st_ivas->hMasaIsmData->azimuth_separated_ism[md_idx];
		elevation = st_ivas->hMasaIsmData->elevation_separated_ism[md_idx];
		}
		else
		{
		azimuth = st_ivas->hMasaIsmData->azimuth_ism[obj][md_idx];
		elevation = st_ivas->hMasaIsmData->elevation_ism[obj][md_idx];
		}

		if ( st_ivas->hOutSetup.is_planar_setup )
		{
		/* If no elevation support in output format, then rendering should be done with zero elevation */
		elevation = 0;
		}

		if ( hVBAPdata != NULL )
		{
		vbap_determine_gains( hVBAPdata, gains, azimuth, elevation, 1 );
		}
		else
		{
		ivas_dirac_dec_get_response( azimuth, elevation, gains, hDirACRend->hOutSetup.ambisonics_order );
		}

		for ( j = 0; j < nchan_out_woLFE; j++ )
		{
		if ( hDirACRend->hOutSetup.num_lfe > 0 )
		{
		j2 = j + ( j >= lfe_index );
		}
		else
		{
		j2 = j;
		}

		if ( fabsf( gains[j] ) > 0.0f \|\| fabsf( hRendererData->prev_gains[obj][j] ) > 0.0f )
		{
		for ( k = 0; k < n_samples_sf; k++ )
		{
		g1 = hRendererData->interpolator[k];
		g2 = 1.0f - g1;
		output_f_local[j2][k + offsetSamples] += ( g1 * gains[j] + g2 * hRendererData->prev_gains[obj][j] ) * input_f[obj][k + offsetSamples];
		}
		}
		hRendererData->prev_gains[obj][j] = gains[j];
		}

		offsetSamples += n_samples_sf;
		}
		}

		return;
		}
		#endif

lib_dec/ivas_jbm_dec.c

+134 −0

Original line number	Diff line number	Diff line
		@@ -77,7 +77,11 @@ ivas_error ivas_jbm_dec_tc(
		float output[MAX_TRANSPORT_CHANNELS][L_FRAME48k]; /* 'float' buffer for transport channels, MAX_TRANSPORT_CHANNELS channels */
		int16_t nchan_remapped;
		float output_lfe_ch[L_FRAME48k];
		#ifdef MASA_AND_OBJECTS
		int16_t nb_bits_metadata[MAX_SCE + 1];
		#else
		int16_t nb_bits_metadata[MAX_SCE];
		#endif
		int32_t output_Fs, ivas_total_brate;
		AUDIO_CONFIG output_config;
		ivas_error error;
		@@ -85,6 +89,9 @@ ivas_error ivas_jbm_dec_tc(
		#ifdef VLBR_20MS_MD
		int16_t num_md_sub_frames;
		#endif
		#ifdef MASA_AND_OBJECTS
		int32_t ism_total_brate;
		#endif

		error = IVAS_ERR_OK;

		@@ -331,6 +338,80 @@ ivas_error ivas_jbm_dec_tc(
		#endif
		}
		}
		#ifdef MASA_AND_OBJECTS
		else if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
		{
		int16_t nchan_ism, nchan_transport_ism;
		int16_t dirac_bs_md_write_idx;

		st = st_ivas->hCPE[0]->hCoreCoder[0];
		set_s( nb_bits_metadata, 0, MAX_SCE + 1 );

		/* Set the number of objects for the parametric rendering */
		dirac_bs_md_write_idx = 0;
		if ( st_ivas->hDirAC != NULL )
		{
		st_ivas->hSpatParamRendCom->numIsmDirections = 0;
		if ( st_ivas->ism_mode != ISM_MASA_MODE_DISC && st_ivas->ism_mode != ISM_MASA_MODE_MASA_ONE_OBJ )
		{
		st_ivas->hSpatParamRendCom->numIsmDirections = st_ivas->nchan_ism;
		}

		dirac_bs_md_write_idx = st_ivas->hSpatParamRendCom->dirac_bs_md_write_idx; /* Store the write-index for this frame */
		}

		/* MASA metadata decoding */
		if ( ( error = ivas_masa_decode( st_ivas, st, &nb_bits_metadata[0] ) ) != IVAS_ERR_OK )
		{
		return error;
		}

		/* Configuration of combined-format bit-budget distribution */
		ivas_set_surplus_brate_dec( st_ivas, &ism_total_brate );

		st->bit_stream = &( st_ivas->bit_stream[( ism_total_brate / FRAMES_PER_SEC )] );

		/* set ISM parameters and decode ISM metadata in OMASA format */
		if ( ( error = ivas_omasa_ism_metadata_dec( st_ivas, ism_total_brate, &nchan_ism, &nchan_transport_ism, dirac_bs_md_write_idx, &nb_bits_metadata[1] ) ) != IVAS_ERR_OK )
		{
		return error;
		}

		/* decode ISM channels */
		for ( n = 0; n < nchan_transport_ism; n++ )
		{
		if ( ( error = ivas_sce_dec( st_ivas, n, &output[st_ivas->nchan_transport + n], output_frame, nb_bits_metadata[1] ) ) != IVAS_ERR_OK )
		{
		return error;
		}
		}

		/* decode MASA channels */
		if ( ( error = ivas_cpe_dec( st_ivas, 0, output, output_frame, nb_bits_metadata[0] ) ) != IVAS_ERR_OK )
		{
		return error;
		}

		if ( st_ivas->hCPE[0]->nchan_out == 1 )
		{
		mvr2r( output[0], output[1], output_frame ); /* Copy mono signal to stereo output channels */
		}

		/* HP filtering */
		for ( n = 0; n < getNumChanSynthesis( st_ivas ); n++ )
		{
		hp20( output[n], output_frame, st_ivas->mem_hp20_out[n], output_Fs );
		}

		if ( st_ivas->renderer_type == RENDERER_MONO_DOWNMIX )
		{
		ivas_mono_downmix_render_passive( st_ivas, output, output_frame );
		}

		/* Set edited object positions, if editing enabled */
		ivas_omasa_set_edited_objects( st_ivas );
		}
		#endif
		else if ( st_ivas->ivas_format == MC_FORMAT )
		{
		st = ( st_ivas->nSCE > 0 ) ? st_ivas->hSCE[0]->hCoreCoder[0] : st_ivas->hCPE[0]->hCoreCoder[0];
		@@ -642,6 +723,21 @@ ivas_error ivas_jbm_dec_feed_tc_to_renderer(
		{
		ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available );
		}
		#ifdef MASA_AND_OBJECTS
		else if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
		{
		if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC && st_ivas->ism_mode == ISM_MASA_MODE_DISC )
		{
		n_render_timeslots *= (st_ivas->hTcBuffer->n_samples_granularity / st_ivas->hSpatParamRendCom->slot_size);
		}
		ivas_sba_dec_digest_tc( st_ivas, n_render_timeslots, st_ivas->hTcBuffer->n_samples_available );

		if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC )
		{
		ivas_ism_dec_digest_tc( st_ivas );
		}
		}
		#endif
		else if ( st_ivas->ivas_format == MC_FORMAT )
		{
		if ( st_ivas->mc_mode == MC_MODE_MCT )
		@@ -878,6 +974,31 @@ ivas_error ivas_jbm_dec_render(
		ivas_sba_dec_render( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, p_output );
		}
		}
		#ifdef MASA_AND_OBJECTS
		else if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
		{
		nchan_remapped = st_ivas->nchan_transport;

		if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC \|\| st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM \|\| st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC )
		{
		if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC && st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC )
		{
		if ( ( ivas_omasa_dirac_td_binaural_jbm( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output ) ) != IVAS_ERR_OK )
		{
		return error;
		}
		}
		else
		{
		ivas_dirac_dec_binaural_render( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output );
		}
		}
		else if ( st_ivas->renderer_type == RENDERER_DIRAC )
		{
		ivas_omasa_dirac_rend_jbm( st_ivas, nSamplesAskedLocal, nSamplesRendered, nSamplesAvailableNext, nchan_remapped, p_output );
		}
		}
		#endif
		else if ( st_ivas->ivas_format == MC_FORMAT )
		{
		if ( st_ivas->mc_mode == MC_MODE_MCT )
		@@ -1512,6 +1633,19 @@ int16_t ivas_jbm_dec_get_num_tc_channels(
		}
		}
		}
		#ifdef MASA_AND_OBJECTS
		else if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
		{
		if ( st_ivas->ism_mode == ISM_MASA_MODE_MASA_ONE_OBJ \|\| st_ivas->ism_mode == ISM_MASA_MODE_PARAM_ONE_OBJ )
		{
		num_tc++;
		}
		else if ( st_ivas->ism_mode == ISM_MASA_MODE_DISC )
		{
		num_tc += st_ivas->nchan_ism;
		}
		}
		#endif
		else if ( st_ivas->ivas_format == MC_FORMAT )
		{
		if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_MONO )

lib_dec/ivas_masa_dec.c

+13 −0

Original line number	Diff line number	Diff line
		@@ -674,8 +674,21 @@ ivas_error ivas_masa_dec_open(
		{
		buffer_mode = TC_BUFFER_MODE_BUFFER;
		}
		#ifdef MASA_AND_OBJECTS
		else if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_MONO_DOWNMIX )
		{
		buffer_mode = TC_BUFFER_MODE_BUFFER;
		}
		#endif

		nchan_to_allocate = ivas_jbm_dec_get_num_tc_channels( st_ivas );
		#ifdef MASA_AND_OBJECTS
		if ( st_ivas->ivas_format == MASA_ISM_FORMAT && st_ivas->renderer_type == RENDERER_MONO_DOWNMIX )
		{
		nchan_to_allocate = 1;
		}
		#endif

		if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, buffer_mode, nchan_to_allocate, nchan_to_allocate, nchan_to_allocate, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK )
		{
		return error;

lib_dec/ivas_objectRenderer_internal.c

+43 −0

Original line number	Diff line number	Diff line
		@@ -155,6 +155,23 @@ ivas_error ivas_td_binaural_renderer_sf(

		int16_t ism_md_subframe_update_jbm;
		int16_t c_indx, nS;
		#ifdef MASA_AND_OBJECTS
		int16_t nchan_ism_internal, nchan_ism, ch_offset;

		/* Set the number of ISMs */
		if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
		{
		nchan_ism_internal = st_ivas->nchan_ism;
		nchan_ism = st_ivas->nchan_ism;
		ch_offset = 2;
		}
		else
		{
		nchan_ism_internal = st_ivas->hTcBuffer->nchan_transport_internal;
		nchan_ism = st_ivas->nchan_transport;
		ch_offset = 0;
		}
		#endif

		/* Number of subframes to delay metadata to sync with audio */
		if ( st_ivas->hDecoderConfig->Opt_delay_comp )
		@@ -166,15 +183,29 @@ ivas_error ivas_td_binaural_renderer_sf(
		ism_md_subframe_update_jbm = st_ivas->hTcBuffer->nb_subframes - 2;
		}

		#ifdef MASA_AND_OBJECTS
		if ( st_ivas->ivas_format == MASA_ISM_FORMAT )
		{
		ism_md_subframe_update_jbm = max( 0, st_ivas->hTcBuffer->nb_subframes - 4 ); /* Todo Nokia: Update this value to match the value of nonJBM rendering. */
		}
		#endif

		for ( ch = 0; ch < BINAURAL_CHANNELS; ch++ )
		{
		p_reverb_signal[ch] = reverb_signal[ch];
		}

		#ifdef MASA_AND_OBJECTS
		for ( ch = 0; ch < nchan_ism_internal; ch++ )
		{
		tc_local[ch] = st_ivas->hTcBuffer->tc[ch + ch_offset] + st_ivas->hTcBuffer->n_samples_rendered;
		}
		#else
		for ( ch = 0; ch < st_ivas->hTcBuffer->nchan_transport_internal; ch++ )
		{
		tc_local[ch] = st_ivas->hTcBuffer->tc[ch] + st_ivas->hTcBuffer->n_samples_rendered;
		}
		#endif

		for ( ch = 0; ch < st_ivas->hDecoderConfig->nchan_out; ch++ )
		{
		@@ -202,7 +233,11 @@ ivas_error ivas_td_binaural_renderer_sf(
		/* Update object position(s) */
		c_indx = 0;

		#ifdef MASA_AND_OBJECTS
		for ( nS = 0; nS < nchan_ism; nS++ )
		#else
		for ( nS = 0; nS < st_ivas->nchan_transport; nS++ )
		#endif
		{
		if ( !( st_ivas->ivas_format == MC_FORMAT && nS == LFE_CHANNEL ) ) /* Skip LFE for MC */
		{
		@@ -213,7 +248,11 @@ ivas_error ivas_td_binaural_renderer_sf(
		}
		if ( subframe_idx == ism_md_subframe_update_jbm )
		{
		#ifdef MASA_AND_OBJECTS
		TDREND_Update_object_positions( st_ivas->hBinRendererTd, nchan_ism, st_ivas->ivas_format, st_ivas->hIsmMetaData );
		#else
		TDREND_Update_object_positions( st_ivas->hBinRendererTd, st_ivas->nchan_transport, st_ivas->ivas_format, st_ivas->hIsmMetaData );
		#endif
		}

		/* Update the listener's location/orientation */
		@@ -244,7 +283,11 @@ ivas_error ivas_td_binaural_renderer_sf(
		}


		#ifdef MASA_AND_OBJECTS
		for ( ch = 0; ch < nchan_ism_internal; ch++ )
		#else
		for ( ch = 0; ch < st_ivas->hTcBuffer->nchan_transport_internal; ch++ )
		#endif
		{
		tc_local[ch] += output_frame;
		}