Adding pre-rotation to the encoder (cc43c7e1) · Commits · IVAS Codec Public Collaboration / IVAS Codec

apps/encoder.c

+43 −0

Original line number	Diff line number	Diff line
		@@ -38,6 +38,7 @@
		#include "ism_file_reader.h"
		#include "jbm_file_reader.h"
		#include "masa_file_reader.h"
		#include "rotation_file_reader.h"
		#ifdef DEBUGGING
		#include "debug.h"
		#endif
		@@ -146,6 +147,7 @@ typedef struct
		#endif
		bool pca;
		bool ism_extended_metadata;
		char *headRotationFilePath;

		} EncArguments;

		@@ -190,6 +192,7 @@ int main(
		MasaFileReader *masaReader = NULL;
		IsmFileReader *ismReaders[IVAS_MAX_NUM_OBJECTS] = { NULL, NULL, NULL, NULL };
		int16_t *pcmBuf = NULL;
		RotFileReader *headRotReader = NULL;
		#ifdef DEBUGGING
		FILE *f_forcedModeProfile = NULL;
		#ifdef DEBUG_SBA
		@@ -339,6 +342,17 @@ int main(
		}
		}

		convert_backslash( arg.headRotationFilePath );

		if ( !isEmptyString( arg.headRotationFilePath ) )
		{
		if ( RotationFileReader_open( arg.headRotationFilePath, &headRotReader ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error opening file: %s\n", arg.headRotationFilePath );
		exit( -1 );
		}
		}

		/------------------------------------------------------------------------------------------
		* Handle Channel-aware mode
		------------------------------------------------------------------------------------------/
		@@ -756,6 +770,25 @@ int main(
		}
		}

		/* Read from head rotation trajectory file if specified */
		if ( headRotReader != NULL )
		{
		IVAS_QUATERNION headRot;
		IVAS_VECTOR3 Pos;

		if ( ( error = HeadRotationFileReading( headRotReader, &headRot, &Pos ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error in Head Rotation File Reading: %s\n", ivas_error_to_string( error ) );
		exit( -1 );
		}

		if ( ( error = IVAS_ENC_SetHeadRotation( hIvasEnc, headRot, Pos ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "Error setting Head Rotation: %s\n", ivas_error_to_string( error ) );
		exit( -1 );
		}
		}

		/* * Encode one frame * */
		if ( ( error = IVAS_ENC_EncodeFrameToSerial( hIvasEnc, pcmBuf, pcmBufSize, bitStream, &numBits ) ) != IVAS_ERR_OK )
		{
		@@ -840,6 +873,8 @@ cleanup:
		fclose( f_bitrateProfile );
		}

		RotationFileReader_close( &headRotReader );

		IVAS_ENC_Close( &hIvasEnc );

		#ifdef WMOPS
		@@ -1695,6 +1730,13 @@ static bool parseCmdlIVAS_enc(
		arg->pca = 1;
		i++;
		}
		else if ( strcmp( argv_to_upper, "-TRAJECTORY_FILE" ) == 0 )
		{
		fprintf( stderr, "Error: Head Roration supplied\n\n" );
		i++;
		arg->headRotationFilePath = argv[i];
		i++;
		}

		/-----------------------------------------------------------------
		* Option not recognized
		@@ -1913,6 +1955,7 @@ static void usage_enc( void )
		#endif
		fprintf( stdout, "-q : Quiet mode, no frame counters\n" );
		fprintf( stdout, " default is deactivated\n" );
		fprintf( stdout, "-trajectory_file F : Head rotation trajectory file for simulation of head tracking (only for SBA inputs)\n" );
		fprintf( stdout, "\n" );

		return;

lib_enc/ivas_enc.c

+400 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@
		*******************************************************************************************************/

		#include <stdint.h>
		#include <math.h>
		#include "options.h"
		#include "cnst.h"
		#include "ivas_cnst.h"
		@@ -43,6 +44,370 @@
		#endif
		#include "wmc_auto.h"

		typedef struct
		{
		int16_t numSamplesPerChannel;
		int16_t numChannels;
		} IVAS_ENC_AudioBufferConfig;

		typedef struct
		{
		IVAS_ENC_AudioBufferConfig config;
		float *data;
		} IVAS_ENC_AudioBuffer;

		static float *getSmplPtr(
		IVAS_ENC_AudioBuffer buffer,
		const uint32_t chnlIdx,
		const uint32_t smplIdx )
		{
		return buffer.data + chnlIdx * buffer.config.numSamplesPerChannel + smplIdx;
		}

		static ivas_error getAmbisonicsOrder(
		AUDIO_CONFIG config,
		int16_t *order )
		{
		switch ( config )
		{
		case IVAS_AUDIO_CONFIG_FOA:
		*order = 1;
		break;
		case IVAS_AUDIO_CONFIG_HOA2:
		*order = 2;
		break;
		case IVAS_AUDIO_CONFIG_HOA3:
		*order = 3;
		break;
		default:
		return IVAS_ERROR( IVAS_ERR_INTERNAL_FATAL, "Unsupported audio config" );
		}

		return IVAS_ERR_OK;
		}

		/*-------------------------------------------------------------------------
		* Helper functions used by SHrotmatgen,
		* an implementation of the algorithm in
		* Ivanic, J. & Ruedenberg, K., J. Phys. Chem. 100, 6342 (1996)
		------------------------------------------------------------------------/

		static float SHrot_p(
		const int16_t i,
		const int16_t l,
		const int16_t a,
		const int16_t b,
		float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM],
		float R_lm1[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM] )
		{
		float ri1 = 0.0f, rim1 = 0.0f, ri0 = 0.0f, p = 0.0f, R_lm1_1 = 0.0f, R_lm1_2 = 0.0f;

		ri1 = SHrotmat[i + 1 + 1][1 + 1 + 1];
		rim1 = SHrotmat[i + 1 + 1][-1 + 1 + 1];
		ri0 = SHrotmat[i + 1 + 1][0 + 1 + 1];

		if ( b == -l )
		{
		R_lm1_1 = R_lm1[a + l - 1][0];
		R_lm1_2 = R_lm1[a + l - 1][2 * l - 2];
		p = ri1 * R_lm1_1 + rim1 * R_lm1_2;
		}
		else
		{
		if ( b == l )
		{
		R_lm1_1 = R_lm1[a + l - 1][2 * l - 2];
		R_lm1_2 = R_lm1[a + l - 1][0];
		p = ri1 * R_lm1_1 - rim1 * R_lm1_2;
		}
		else
		{
		R_lm1_1 = R_lm1[a + l - 1][b + l - 1];
		p = ri0 * R_lm1_1;
		}
		}

		return p;
		}

		static float SHrot_u(
		const int16_t l,
		const int16_t m,
		const int16_t n,
		float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM],
		float R_lm1[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM] )
		{
		return SHrot_p( 0, l, m, n, SHrotmat, R_lm1 );
		}

		static float SHrot_v(
		const int16_t l,
		const int16_t m,
		const int16_t n,
		float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM],
		float R_lm1[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM] )
		{

		float result, d, p0, p1;

		if ( m == 0 )
		{
		p0 = SHrot_p( 1, l, 1, n, SHrotmat, R_lm1 );
		p1 = SHrot_p( -1, l, -1, n, SHrotmat, R_lm1 );
		result = p0 + p1;
		}
		else
		{
		if ( m > 0 )
		{
		d = ( m == 1 ) ? 1.0f : 0.0f;
		p0 = SHrot_p( 1, l, m - 1, n, SHrotmat, R_lm1 );
		p1 = SHrot_p( -1, l, -m + 1, n, SHrotmat, R_lm1 );
		result = p0 * sqrtf( 1.0f + d ) - p1 * ( 1.0f - d );
		}
		else
		{
		d = ( m == -1 ) ? 1.0f : 0.0f;
		p0 = SHrot_p( 1, l, m + 1, n, SHrotmat, R_lm1 );
		p1 = SHrot_p( -1, l, -m - 1, n, SHrotmat, R_lm1 );
		result = p0 * ( 1.0f - d ) + p1 * sqrtf( 1.0f + d );
		}
		}

		return result;
		}

		static float SHrot_w(
		const int16_t l,
		const int16_t m,
		const int16_t n,
		float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM],
		float R_lm1[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM] )
		{
		float result, p0, p1;

		if ( m == 0 )
		{
		#ifdef SPLIT_REND_WITH_HEAD_ROT
		assert( 0 && "ERROR should not be called\n" );
		#else
		printf( "ERROR should not be called\n" );
		#endif
		return 0.0f;
		}
		else
		{
		if ( m > 0 )
		{
		p0 = SHrot_p( 1, l, m + 1, n, SHrotmat, R_lm1 );
		p1 = SHrot_p( -1, l, -m - 1, n, SHrotmat, R_lm1 );
		result = p0 + p1;
		}
		else
		{
		p0 = SHrot_p( 1, l, m - 1, n, SHrotmat, R_lm1 );
		p1 = SHrot_p( -1, l, -m + 1, n, SHrotmat, R_lm1 );
		result = p0 - p1;
		}
		}

		return result;
		}

		static void SHrotmatgen(
		float SHrotmat[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM], /* o : rotation matrix in SHD */
		float Rmat[3][3], /* i : real-space rotation matrix */
		const int16_t order /* i : ambisonics order */
		)
		{
		int16_t d = 0;
		int16_t band_idx = 0;
		int16_t i, j;
		int16_t l, m, n;
		int16_t absm;
		float sqdenom = 0.0f, sql2mm2 = 0.0f, sqdabsm = 0.0f, sqlabsm = 0.0f;
		float u = 0.0f, v = 0.0f, w = 0.0f;
		float R_lm1[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM];
		float R_l[HEADROT_SHMAT_DIM][HEADROT_SHMAT_DIM];

		SHrotmat[0][0] = 1.0f;

		SHrotmat[1][1] = Rmat[1][1];
		SHrotmat[1][2] = Rmat[1][2];
		SHrotmat[1][3] = Rmat[1][0];

		SHrotmat[2][1] = Rmat[2][1];
		SHrotmat[2][2] = Rmat[2][2];
		SHrotmat[2][3] = Rmat[2][0];

		SHrotmat[3][1] = Rmat[0][1];
		SHrotmat[3][2] = Rmat[0][2];
		SHrotmat[3][3] = Rmat[0][0];

		for ( i = 0; i < 2 * 1 + 1; i++ )
		{
		for ( j = 0; j < 2 * 1 + 1; j++ )
		{
		R_lm1[i][j] = SHrotmat[i + 1][j + 1];
		}
		}

		band_idx = 4;
		for ( l = 2; l <= order; l++ )
		{
		set_zero( &R_l[0][0], HEADROT_SHMAT_DIM2 );

		for ( m = -l; m <= l; m++ )
		{
		d = ( m == 0 ) ? 1 : 0;
		absm = (int16_t) abs( m );
		sql2mm2 = sqrtf( (float) ( l * l - m * m ) );
		sqdabsm = sqrtf( (float) ( ( 1 + d ) * ( l + absm - 1 ) * ( l + absm ) ) );
		sqlabsm = sqrtf( (float) ( ( l - absm - 1 ) * ( l - absm ) ) );

		for ( n = -l; n <= l; n++ )
		{
		if ( abs( n ) == l )
		{
		sqdenom = sqrtf( (float) ( ( 2 * l ) * ( 2 * l - 1 ) ) );
		}
		else
		{
		sqdenom = sqrtf( (float) ( l * l - n * n ) );
		}

		u = sql2mm2 / sqdenom;
		v = sqdabsm / sqdenom * ( 1 - 2 * d ) * 0.5f;
		w = sqlabsm / sqdenom * ( 1 - d ) * ( -0.5f );

		if ( u != 0 )
		{
		u = u * SHrot_u( l, m, n, SHrotmat, R_lm1 );
		}
		if ( v != 0 )
		{
		v = v * SHrot_v( l, m, n, SHrotmat, R_lm1 );
		}
		if ( w != 0 )
		{
		w = w * SHrot_w( l, m, n, SHrotmat, R_lm1 );
		}
		R_l[m + l][n + l] = u + v + w;
		}
		}

		for ( i = 0; i < 2 * l + 1; i++ )
		{
		for ( j = 0; j < 2 * l + 1; j++ )
		{
		SHrotmat[band_idx + i][band_idx + j] = R_l[i][j];
		}
		}

		for ( i = 0; i < 2 * l + 1; i++ )
		{
		for ( j = 0; j < 2 * l + 1; j++ )
		{
		R_lm1[i][j] = R_l[i][j];
		}
		}

		band_idx += 2 * l + 1;
		}

		return;
		}

		static ivas_error rotateFrameSba(
		IVAS_ENC_AudioBuffer inAudio, /* i : Input Audio buffer */
		int16_t sba_order, /* i : Input Audio config */
		const ENC_HeadRotData headRotData, / i : Head rotation data */
		enc_rotation_gains gains_prev, /* i/o: Previous frame rotation gains */
		IVAS_ENC_AudioBuffer outAudio /* o : Output Audio buffer */
		)
		{
		int16_t i, l, n, m;
		int16_t m1, m2;
		int16_t shd_rot_max_order;
		const float *crossfade;

		float *writePtr;
		enc_rotation_matrix Rmat;
		float tmpRot[2 * HEADROT_ORDER + 1];
		enc_rotation_gains gains;
		ivas_error error;
		int16_t idx;
		float val, cf, oneminuscf;

		push_wmops( "rotateFrameSba" );

		crossfade = headRotData->crossfade;

		shd_rot_max_order = sba_order;

		/* initialize rotation matrices with zeros */
		for ( i = 0; i < HEADROT_SHMAT_DIM; i++ )
		{
		set_zero( gains[i], HEADROT_SHMAT_DIM );
		}

		for ( i = 0; i < 3; i++ )
		{
		/* Set to identity */
		set_zero( Rmat[i], 3 );
		Rmat[i][i] = 1.0f;
		}

		/* calculate ambisonics rotation matrices for the previous and current frames */
		SHrotmatgen( gains, Rmat, shd_rot_max_order );

		for ( i = 0; i < inAudio.config.numSamplesPerChannel; i++ )
		{
		idx = i;
		cf = crossfade[i];
		oneminuscf = 1 - cf;
		/* As the rotation matrix becomes block diagonal in a SH basis, we can*/
		/* apply each angular-momentum block individually to save complexity. */

		/* loop over l blocks */
		m1 = 1;
		m2 = 4;
		for ( l = 1; l <= shd_rot_max_order; l++ )
		{
		/* compute mtx-vector product for this l */
		for ( n = m1; n < m2; n++ )
		{
		tmpRot[n - m1] = 0.f;

		for ( m = m1; m < m2; m++ )
		{
		val = inAudio.data[m * inAudio.config.numSamplesPerChannel + idx];
		/* crossfade with previous rotation gains */
		tmpRot[n - m1] += ( cf * gains[n][m] * val + oneminuscf * gains_prev[n][m] * val );
		}
		}
		/* write back the result */
		for ( n = m1; n < m2; n++ )
		{
		writePtr = getSmplPtr( outAudio, n, idx );
		( *writePtr ) = tmpRot[n - m1];
		}
		m1 = m2;
		m2 += 2 * ( l + 1 ) + 1;
		}
		}

		/* move SHrotmat to SHrotmat_prev */
		for ( i = 0; i < HEADROT_SHMAT_DIM; i++ )
		{
		mvr2r( gains[i], gains_prev[i], HEADROT_SHMAT_DIM );
		}

		pop_wmops();

		return IVAS_ERR_OK;
		}

		/-------------------------------------------------------------------
		* ivas_enc()
		*
		@@ -238,6 +603,41 @@ ivas_error ivas_enc(
		}
		else if ( ivas_format == SBA_FORMAT )
		{
		/* Rotate the SBA Scene */
		if ( st_ivas->headRotData.headRotEnabled )
		{
		int16_t ch;
		IVAS_ENC_AudioBuffer data_in;
		data_in.config.numChannels = nchan_inp;
		data_in.config.numSamplesPerChannel = n_samples_chan;
		data_in.data = malloc( nchan_inp * n_samples_chan * sizeof( float ) );

		/* Copy data to audio buffer (needs to be cleaned up) */
		for ( ch = 0; ch < nchan_inp; ++ch )
		{
		mvr2r( data_f[ch], &data_in.data[ch * n_samples_chan], n_samples_chan );
		}

		IVAS_ENC_AudioBuffer data_out;
		data_out.config.numChannels = nchan_inp;
		data_out.config.numSamplesPerChannel = n_samples_chan;
		data_out.data = malloc( nchan_inp * n_samples_chan * sizeof( float ) );

		if ( ( error = rotateFrameSba( data_in, st_ivas->hEncoderConfig->sba_order, &st_ivas->headRotData, &st_ivas->headRotData.rot_gains_prev[0], data_out ) ) != IVAS_ERR_OK )
		{
		return error;
		}

		/* Copy data to audio buffer (needs to be cleaned up) */
		for ( ch = 0; ch < nchan_inp; ++ch )
		{
		mvr2r( &data_out.data[ch * n_samples_chan], data_f[ch], n_samples_chan );
		}

		free( data_in.data );
		free( data_out.data );
		}

		if ( ( error = ivas_spar_enc( st_ivas, data_f, input_frame, nb_bits_metadata, hMetaData ) ) != IVAS_ERR_OK )
		{
		return error;

lib_enc/ivas_stat_enc.h

+15 −0

Original line number	Diff line number	Diff line
		@@ -39,6 +39,7 @@
		#include "ivas_cnst.h"
		#include "stat_enc.h"
		#include "ivas_stat_com.h"
		#include "common_api_types.h"

		/----------------------------------------------------------------------------------
		* DFT Stereo encoder structures
		@@ -1211,6 +1212,18 @@ typedef struct encoder_config_structure

		} ENCODER_CONFIG, *ENCODER_CONFIG_HANDLE;

		typedef float enc_rotation_gains[MAX_INPUT_CHANNELS][MAX_INPUT_CHANNELS];
		typedef float enc_rotation_matrix[3][3];

		typedef struct
		{
		int8_t headRotEnabled;
		IVAS_QUATERNION headPositions;
		IVAS_VECTOR3 Pos;
		float crossfade[L_FRAME48k];
		enc_rotation_gains rot_gains_prev;

		} ENC_HeadRotData;

		/----------------------------------------------------------------------------------
		*
		@@ -1265,6 +1278,8 @@ typedef struct
		/* Stereo downmix for EVS module */
		STEREO_DMX_EVS_ENC_HANDLE hStereoDmxEVS; /* Stereo downmix for EVS encoder handle */

		ENC_HeadRotData headRotData;

		} Encoder_Struct;

		/* clang-format on */

lib_enc/lib_enc.c

+44 −0

Original line number	Diff line number	Diff line
		@@ -468,6 +468,50 @@ ivas_error IVAS_ENC_FeedObjectMetadata(
		return error;
		}

		/-------------------------------------------------------------------
		* IVAS_ENC_SetHeadRotation()
		*
		*
		-------------------------------------------------------------------/

		ivas_error IVAS_ENC_SetHeadRotation(
		IVAS_ENC_HANDLE hIvasEnc, /* i/o: Renderer handle */
		const IVAS_QUATERNION headRot, /* i : head orientations for next rendering call */
		const IVAS_VECTOR3 Pos /* i : listener positions for next rendering call */
		)
		{
		IVAS_QUATERNION rotQuat;

		/* Validate function arguments */
		if ( hIvasEnc == NULL )
		{
		return IVAS_ERR_UNEXPECTED_NULL_POINTER;
		}

		if ( ( hIvasEnc->st_ivas->hEncoderConfig->ivas_format != SBA_FORMAT ) )
		{
		/* Head rotation can be set only with binaural output */
		return IVAS_ERR_INVALID_INPUT_FORMAT;
		}

		hIvasEnc->st_ivas->headRotData.headRotEnabled = 1;

		/* check for Euler angle signaling */
		if ( headRot.w == -3.0f )
		{
		Euler2Quat( deg2rad( headRot.x ), deg2rad( headRot.y ), deg2rad( headRot.z ), &rotQuat );
		}
		else
		{
		rotQuat = headRot;
		}

		hIvasEnc->st_ivas->headRotData.headPositions = rotQuat;

		hIvasEnc->st_ivas->headRotData.Pos = Pos;

		return IVAS_ERR_OK;
		}

		/---------------------------------------------------------------------
		* IVAS_ENC_ConfigureForAmbisonics()

lib_enc/lib_enc.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -280,6 +280,12 @@ ivas_error IVAS_ENC_FeedObjectMetadata(
		const IVAS_ISM_METADATA metadata /* i : object metadata handle for current frame */
		);

		ivas_error IVAS_ENC_SetHeadRotation(
		IVAS_ENC_HANDLE hIvasEnc, /* i/o: Renderer handle */
		const IVAS_QUATERNION headRot, /* i : head orientations for next rendering call */
		const IVAS_VECTOR3 Pos /* i : listener positions for next rendering call */
		);

		/! r: error code /
		ivas_error IVAS_ENC_FeedMasaMetadata(
		IVAS_ENC_HANDLE hIvasEnc, /* i/o: IVAS encoder handle */