From 5060f7584dbe736e734c23a3945e3aa29e499a00 Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Wed, 19 Oct 2022 12:58:32 +0200 Subject: [PATCH 01/20] Extend framework to include MASA rendering --- CMakeLists.txt | 2 +- apps/renderer.c | 95 ++++- lib_com/options.h | 1 + lib_rend/lib_rend.c | 655 ++++++++++++++++++++++++++++++++- lib_rend/lib_rend.h | 3 +- scripts/tests/constants.py | 4 +- scripts/tests/test_renderer.py | 27 +- 7 files changed, 751 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 806f1f3e2a..210eee1cd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,7 +165,7 @@ if(WIN32) endif() add_executable(IVAS_rend apps/renderer.c) -target_link_libraries(IVAS_rend lib_rend lib_util) +target_link_libraries(IVAS_rend lib_rend lib_util lib_dec) # Todo refactor: dependency to lib_dec should be removed if(COPY_EXECUTABLES_TO_ROOT) # Optionally copy executables to root directory after build diff --git a/apps/renderer.c b/apps/renderer.c index 08dd6f4e0d..a851c1fa2e 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -323,7 +323,7 @@ static void parseSceneDescriptionFile( char *audioFilePath, InputConfig *inConfig, IsmPositionProvider *positionProvider, - MasaFileReader** masaReaders ); + MasaFileReader **masaReaders ); static ivas_error parseCustomLayoutFile( const char *filePath, @@ -390,7 +390,7 @@ static void parseMetadata( char *inDir, InputConfig *inConfig, IsmPositionProvider *positionProvider, - MasaFileReader** masaReaders ); + MasaFileReader **masaReaders ); static void convert_backslash( char *str ); @@ -431,7 +431,12 @@ static IVAS_REND_ReadOnlyAudioBuffer getReadOnlySubBuffer( IVAS_REND_AudioBuffer static int32_t getTotalNumInChannels( IVAS_REND_HANDLE hIvasRend, IVAS_REND_InputId mcIds[RENDERER_MAX_MC_INPUTS], IVAS_REND_InputId ismIds[RENDERER_MAX_ISM_INPUTS], - IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] ) + IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + , + IVAS_REND_InputId masaIds[RENDERER_MAX_MASA_INPUTS] +#endif +) { int32_t totalNumInChannels = 0; ivas_error error; @@ -487,6 +492,25 @@ static int32_t getTotalNumInChannels( IVAS_REND_HANDLE hIvasRend, totalNumInChannels += numInputChannels; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( int32_t i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + if ( masaIds[i] == 0 ) + { + /* Skip inactive inputs */ + continue; + } + + int32_t numInputChannels; + if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, masaIds[i], &numInputChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + totalNumInChannels += numInputChannels; + } +#endif + return totalNumInChannels; } @@ -651,6 +675,9 @@ int32_t main( int32_t argc, char **argv ) IVAS_REND_InputId mcIds[RENDERER_MAX_MC_INPUTS] = { 0 }; IVAS_REND_InputId ismIds[RENDERER_MAX_ISM_INPUTS] = { 0 }; IVAS_REND_InputId sbaIds[RENDERER_MAX_SBA_INPUTS] = { 0 }; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + IVAS_REND_InputId masaIds[RENDERER_MAX_MASA_INPUTS] = { 0 }; +#endif if ( ( error = IVAS_REND_Open( &hIvasRend, args.sampleRate, args.outConfig.audioConfig ) ) != IVAS_ERR_OK ) { @@ -723,7 +750,24 @@ int32_t main( int32_t argc, char **argv ) } } - const int32_t totalNumInChannels = getTotalNumInChannels( hIvasRend, mcIds, ismIds, sbaIds ); +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( int32_t i = 0; i < args.inConfig.numMasaBuses; ++i ) + { + if ( ( error = IVAS_REND_AddInput( hIvasRend, args.inConfig.masaBuses[i].audioConfig, &masaIds[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( ( error = IVAS_REND_SetInputGain( hIvasRend, masaIds[i], args.inputGainGlobal * dBToLin( args.inConfig.masaBuses[i].gain_dB ) ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } +#endif + + const int32_t totalNumInChannels = getTotalNumInChannels( hIvasRend, mcIds, ismIds, sbaIds, masaIds ); if ( AudioFileReader_getNumChannels( audioReader ) != 0 /* If input file is raw PCM, audio reader has no info about number of channels */ && totalNumInChannels != AudioFileReader_getNumChannels( audioReader ) ) { @@ -801,6 +845,7 @@ int32_t main( int32_t argc, char **argv ) /* Convert from int to float and from interleaved to packed */ convertInputBuffer( inpInt16Buffer, numSamplesRead, frameSize_smpls, num_in_channels, inFloatBuffer ); +#ifndef NOKIA_MASA_EXTERNAL_RENDERER for ( int32_t i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) { if ( masaReaders[i] != NULL ) @@ -811,6 +856,7 @@ int32_t main( int32_t argc, char **argv ) (void) hMasaMetadata; } } +#endif ObjectPositionBuffer mtdBuffer; IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer ); @@ -878,6 +924,37 @@ int32_t main( int32_t argc, char **argv ) } } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( int32_t i = 0; i < args.inConfig.numMasaBuses; ++i ) + { + int32_t numChannels; + if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, masaIds[i], &numChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, args.inConfig.masaBuses[i].inputChannelIndex, numChannels ); + + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, masaIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( masaReaders[i] != NULL ) + { + /* This will update data in hMasaMetadata[i] */ + MasaFileReader_readNextFrame( masaReaders[i] ); + + if ( ( error = IVAS_REND_FeedInputMasaMetadata( hIvasRend, masaIds[i], hMasaMetadata[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } +#endif + IVAS_REND_GetSamples( hIvasRend, outBuffer ); int32_t num_out_channels; @@ -1263,7 +1340,7 @@ static IVAS_REND_AudioConfig parseAudioConfig( const char *configString ) return IVAS_REND_AUDIO_CONFIG_UNKNOWN; } -static const CmdLnParser_Option* findOptionById( int32_t id ) +static const CmdLnParser_Option *findOptionById( int32_t id ) { for ( int32_t i = 0; i < numCliOptions; ++i ) { @@ -1278,7 +1355,7 @@ static const CmdLnParser_Option* findOptionById( int32_t id ) static bool checkRequiredArgs( CmdlnArgs args ) { - const CmdLnParser_Option* tmpOption; + const CmdLnParser_Option *tmpOption; /* Check required arguments */ bool missingRequiredArg = false; @@ -1972,7 +2049,7 @@ static void parseMetadata( char *inDir, InputConfig *inConfig, IsmPositionProvider *positionProvider, - MasaFileReader** masaReaders ) + MasaFileReader **masaReaders ) { char line[RENDERER_MAX_METADATA_LINE_LENGTH]; char *delimiter; @@ -2056,7 +2133,7 @@ static void parseMetadata( } parseMasa( line, inDir, inConfig, masaReaders, counterMasaInputs - 1 ); } - else if ( isEmptyString(line) ) + else if ( isEmptyString( line ) ) { fprintf( stderr, "Metadata string too short - expected %d inputs, found %d.\n", totalNumberOfAudioObjects, num_parsed_inputs ); exit( -1 ); @@ -2085,7 +2162,7 @@ static void parseMetadata( } } -static void parseSceneDescriptionFile( char *path, char *audioFilePath, InputConfig *inConfig, IsmPositionProvider *positionProvider, MasaFileReader** masaReaders ) +static void parseSceneDescriptionFile( char *path, char *audioFilePath, InputConfig *inConfig, IsmPositionProvider *positionProvider, MasaFileReader **masaReaders ) { uint32_t inAudioFilePathLen; char inAudioFilePath[FILENAME_MAX]; diff --git a/lib_com/options.h b/lib_com/options.h index 4ea53d4013..f91fae9b8e 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -156,6 +156,7 @@ #define EXT_RENDERER /* FhG: external renderer library and standalone application */ #define FIX_EFAP_MATH /* fix for EFAP: remove angle quantization and a bug in polygon lookup causing incorrect gains. minor tweak for ALLRAD. non-BE for modes using EFAP */ +#define NOKIA_MASA_EXTERNAL_RENDERER /* Nokia: MASA support for external renderer */ /* ################## End DEVELOPMENT switches ######################### */ /* clang-format on */ #endif diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index b31ffafac5..83a0bc1252 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -135,6 +135,19 @@ typedef struct rotation_gains rot_gains_prev; } input_sba; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +/* Due to API of some rendering methods, the renderer has to use the decoder struct. + Only struct members relevant for rendering will be initialized, therefore typedef as "dummy" decoder struct */ +typedef Decoder_Struct DecoderDummy; + +typedef struct +{ + input_base base; + DecoderDummy *decDummy; + MASA_METADATA_FRAME masaMetadata; +} input_masa; +#endif + struct IVAS_REND { int32_t sampleRateOut; @@ -147,6 +160,9 @@ struct IVAS_REND input_ism inputsIsm[RENDERER_MAX_ISM_INPUTS]; input_mc inputsMc[RENDERER_MAX_MC_INPUTS]; input_sba inputsSba[RENDERER_MAX_SBA_INPUTS]; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + input_masa inputsMasa[RENDERER_MAX_MASA_INPUTS]; +#endif IVAS_REND_AudioConfig outputConfig; EFAP_WRAPPER efapOutWrapper; @@ -358,11 +374,13 @@ ivas_error getAudioConfigNumChannels( IVAS_REND_AudioConfig config, int32_t *num { case IVAS_REND_AUDIO_CONFIG_MONO: case IVAS_REND_AUDIO_CONFIG_OBJECT: + case IVAS_REND_AUDIO_CONFIG_MASA1: *numChannels = 1; break; case IVAS_REND_AUDIO_CONFIG_STEREO: case IVAS_REND_AUDIO_CONFIG_BINAURAL: case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: + case IVAS_REND_AUDIO_CONFIG_MASA2: *numChannels = 2; break; case IVAS_REND_AUDIO_CONFIG_FOA: @@ -1786,6 +1804,366 @@ static void clearInputSba( input_sba *inputSba ) } } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +static ivas_error initMasaDummyDecForMcOut( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + int16_t numCldfbAnalyses; + int16_t numCldfbSyntheses; + int16_t i; + AUDIO_CONFIG output_config; + DecoderDummy *decDummy; + decDummy = inputMasa->decDummy; + + output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->hDecoderConfig->output_config = output_config; + + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + ivas_output_init( &( decDummy->hOutSetup ), output_config ); + ivas_output_init( &( decDummy->hIntSetup ), output_config ); + + decDummy->renderer_type = RENDERER_DIRAC; + if ( output_config == AUDIO_CONFIG_STEREO ) + { + decDummy->renderer_type = RENDERER_STEREO_PARAMETRIC; + } + + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = AUDIO_CONFIG_INVALID; + + /* Todo refactor: Access to qmetadata is not required by the algorithm. */ + if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hQMetaData->coherence_flag = 1; + + if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + numCldfbAnalyses = decDummy->nchan_transport; + numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; + + for ( i = 0; i < numCldfbAnalyses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + } + + for ( i = 0; i < numCldfbSyntheses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + +static ivas_error initMasaDummyDecForSbaOut( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + int16_t numCldfbAnalyses; + int16_t numCldfbSyntheses; + int16_t i; + AUDIO_CONFIG output_config; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + + output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->hDecoderConfig->output_config = output_config; + + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + ivas_output_init( &( decDummy->hOutSetup ), output_config ); + ivas_output_init( &( decDummy->hIntSetup ), output_config ); + decDummy->renderer_type = RENDERER_DIRAC; + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = AUDIO_CONFIG_INVALID; + + /* Todo refactor: Access to qmetadata is not required by the algorithm. */ + if ( ( error = ivas_qmetadata_open( &( decDummy->hQMetaData ) ) ) != IVAS_ERR_OK ) + { + return error; + } + decDummy->hQMetaData->coherence_flag = 1; + + if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + + numCldfbAnalyses = decDummy->nchan_transport; + numCldfbSyntheses = decDummy->hDecoderConfig->nchan_out; + + for ( i = 0; i < numCldfbAnalyses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + } + + for ( i = 0; i < numCldfbSyntheses; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_OUTPUT_CHANNELS; i++ ) + { + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + +static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + + int16_t i; + AUDIO_CONFIG output_config; + DecoderDummy *decDummy; + + decDummy = inputMasa->decDummy; + + output_config = rendAudioConfigToIvasAudioConfig( outConfig ); + decDummy->hDecoderConfig->output_config = output_config; + + output_config = decDummy->hDecoderConfig->output_config; + + decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + ivas_output_init( &( decDummy->hOutSetup ), output_config ); + if ( output_config == AUDIO_CONFIG_BINAURAL ) + { + decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC; + } + else + { + decDummy->renderer_type = RENDERER_BINAURAL_PARAMETRIC_ROOM; + } + decDummy->ivas_format = MASA_FORMAT; + decDummy->transport_config = AUDIO_CONFIG_INVALID; + decDummy->hDecoderConfig->forceSubframeBinauralization = 0; + + if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = ivas_dirac_dec_init_binaural_data( decDummy ) ) != IVAS_ERR_OK ) + { + return error; + } + + for ( i = 0; i < BINAURAL_CHANNELS; i++ ) + { + if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( ( error = openCldfb( &( decDummy->cldfbSynDec[i] ), CLDFB_SYNTHESIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) + { + return error; + } + } + for ( ; i < MAX_INTERN_CHANNELS; i++ ) + { + decDummy->cldfbAnaDec[i] = NULL; + decDummy->cldfbSynDec[i] = NULL; + } + + return IVAS_ERR_OK; +} + +static ivas_error updateMasaDummyDec( input_masa *inputMasa, IVAS_REND_AudioConfig outConfig ) +{ + ivas_error error; + + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + error = initMasaDummyDecForMcOut( inputMasa, outConfig ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + error = initMasaDummyDecForSbaOut( inputMasa, outConfig ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + error = initMasaDummyDecForBinauralOut( inputMasa, outConfig ); + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + /* Check error here to keep switch statement more compact */ + if ( error != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; +} + +static DecoderDummy *initDecoderDummy( int32_t sampleRate, int16_t numTransChannels, IVAS_REND_AudioConfig outConfig, const uint8_t enableRenderConfig ) +{ + ivas_error error; + int16_t i; + int32_t numOutChannels; + DecoderDummy *decDummy; + + if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) + { + /* Checking error with assert is enough, this function is only temporary anyway */ + assert(error == IVAS_ERR_OK); + } + + decDummy = count_malloc( sizeof( DecoderDummy ) ); + decDummy->hDecoderConfig = count_malloc( sizeof( DECODER_CONFIG ) ); + decDummy->hDecoderConfig->output_Fs = sampleRate; + decDummy->hDecoderConfig->nchan_out = (int16_t)numOutChannels; + decDummy->hDecoderConfig->Opt_Headrotation = 0; + + decDummy->hBinRenderer = NULL; + decDummy->hEFAPdata = NULL; + decDummy->hHrtf = NULL; + decDummy->hHrtfTD = NULL; + decDummy->hSpar = NULL; + decDummy->hoa_dec_mtx = NULL; + decDummy->hVBAPdata = NULL; + decDummy->hMasa = NULL; + decDummy->hDiracDecBin = NULL; + decDummy->hQMetaData = NULL; + decDummy->hDecoderConfig->output_config = rendAudioConfigToIvasAudioConfig(outConfig); + decDummy->nchan_transport = numTransChannels; + + decDummy->hHeadTrackData = count_malloc( sizeof( HEAD_TRACK_DATA ) ); + /* Initialise Rmat_prev to I, Rmat will be computed later */ + for ( i = 0; i < 3; i++ ) + { + set_zero( decDummy->hHeadTrackData->Rmat_prev[i], 3 ); + decDummy->hHeadTrackData->Rmat_prev[i][i] = 1.0f; + } + + decDummy->hHeadTrackData->num_quaternions = 0; + decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; + decDummy->hHeadTrackData->lrSwitchedCurrent = 0; + decDummy->hHeadTrackData->lrSwitchedNext = 0; + + if ( enableRenderConfig ) + { + ivas_render_config_open( &decDummy->hRenderConfig ); + decDummy->hRenderConfig->roomAcoustics.late_reverb_on = 0; + decDummy->hRenderConfig->roomAcoustics.use_brir = 0; + } + else + { + decDummy->hRenderConfig = NULL; + } + + decDummy->renderer_type = RENDERER_DISABLE; + + return decDummy; +} + +static MASA_METADATA_FRAME defaultMasaMetadata( void ) +{ + MASA_METADATA_FRAME mf; + + /* TODO @ Nokia: Is there a sensible default here that can be used in case no metadata is fed? */ + + return mf; +} + +static ivas_error setRendInputActiveMasa( + void *input, + IVAS_REND_AudioConfig inConfig, + IVAS_REND_InputId id ) +{ + ivas_error error; + rendering_context rendCtx; + IVAS_REND_AudioConfig outConfig; + input_masa *inputMasa; + int32_t numInChannels; + + inputMasa = (input_masa *) input; + rendCtx = inputMasa->base.ctx; + outConfig = *rendCtx.pOutConfig; + + initRendInputBase( &inputMasa->base, inConfig, id, rendCtx ); + + if ( ( error = getAudioConfigNumChannels( inConfig, &numInChannels ) ) != IVAS_ERR_OK ) + { + return error; + } + /* TODO(sgi): Don't hard-code enableRenderConfig here (last argument) */ + inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, (int16_t)numInChannels, outConfig, 0 ); + inputMasa->masaMetadata = defaultMasaMetadata(); + + if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; +} + +static void freeDecoderDummy( DecoderDummy **ppDecDummy ) +{ + DecoderDummy *pDecDummy; + + if ( ppDecDummy == NULL || *ppDecDummy == NULL ) + { + return; + } + pDecDummy = *ppDecDummy; + + if ( pDecDummy->hDecoderConfig != NULL ) + { + free( pDecDummy->hDecoderConfig ); + } + if ( pDecDummy->hHeadTrackData != NULL ) + { + free( pDecDummy->hHeadTrackData ); + } + ivas_render_config_close( &pDecDummy->hRenderConfig ); +} + +static void clearInputMasa( input_masa *inputMasa ) +{ + rendering_context rendCtx; + + rendCtx = inputMasa->base.ctx; + + initRendInputBase( &inputMasa->base, IVAS_REND_AUDIO_CONFIG_UNKNOWN, 0, rendCtx ); + freeDecoderDummy( &inputMasa->decDummy ); +} +#endif + ivas_error IVAS_REND_Open( IVAS_REND_HANDLE *phIvasRend, int32_t outputSampleRate, @@ -1871,6 +2249,16 @@ ivas_error IVAS_REND_Open( getRendCtx( hIvasRend ) ); hIvasRend->inputsSba[i].crendWrapper.hCrend = NULL; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + initRendInputBase( &hIvasRend->inputsMasa[i].base, + IVAS_REND_AUDIO_CONFIG_UNKNOWN, + 0, + getRendCtx( hIvasRend ) ); + hIvasRend->inputsMasa[i].decDummy = NULL; + } +#endif return IVAS_ERR_OK; } @@ -1963,13 +2351,13 @@ ivas_error IVAS_REND_ConfigureCustomOutputLoudspeakerLayout( return error; } initLimiter( &hIvasRend->hLimiter, - numOutChannels, - hIvasRend->sampleRateOut ); + numOutChannels, + hIvasRend->sampleRateOut ); /* Re-initialize EFAP - output layout has changed or has been fully defined for the first time */ initEfap( &hIvasRend->efapOutWrapper, - hIvasRend->outputConfig, - &hIvasRend->customLsOut ); + hIvasRend->outputConfig, + &hIvasRend->customLsOut ); /* Re-initialize panning gains for each active MC input, This includes re-initializing * LFE handling for the new output layout, which means custom LFE handling is overwritten, @@ -2090,6 +2478,15 @@ static ivas_error getInputById( IVAS_REND_HANDLE hIvasRend, IVAS_REND_InputId in } pInputBase = &hIvasRend->inputsSba[inputIndex].base; break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + break; +#endif default: return IVAS_ERR_INVALID_INPUT_ID; } @@ -2145,6 +2542,15 @@ static ivas_error getConstInputById( IVAS_REND_CONST_HANDLE hIvasRend, IVAS_REND } pInputBase = &hIvasRend->inputsSba[inputIndex].base; break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + if ( inputIndex > RENDERER_MAX_MASA_INPUTS ) + { + return IVAS_ERR_INVALID_INPUT_ID; + } + pInputBase = &hIvasRend->inputsMasa[inputIndex].base; + break; +#endif default: return IVAS_ERR_INVALID_INPUT_ID; } @@ -2168,7 +2574,7 @@ static ivas_error findFreeInputSlot( int32_t *inputIndex ) { /* Using a void pointer and a separately provided size is a hack for this function - to be reusable for arrays of any input type (input_ism, input_mc, input_sba). + to be reusable for arrays of any input type (input_ism, input_mc, input_sba, input_masa). Assumptions: - input_base is always the first member in the input struct - provided size is correct @@ -2243,15 +2649,23 @@ ivas_error IVAS_REND_AddInput( inputStructSize = sizeof( *hIvasRend->inputsSba ); activateInput = setRendInputActiveSba; break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + maxNumInputsOfType = RENDERER_MAX_MASA_INPUTS; + inputsArray = hIvasRend->inputsMasa; + inputStructSize = sizeof( *hIvasRend->inputsMasa ); + activateInput = setRendInputActiveMasa; + break; +#endif default: return IVAS_ERR_INVALID_INPUT_FORMAT; } /* Find first free input in array corresponding to input type */ if ( ( error = findFreeInputSlot( inputsArray, - inputStructSize, - maxNumInputsOfType, - &inputIndex ) ) != IVAS_ERR_OK ) + inputStructSize, + maxNumInputsOfType, + &inputIndex ) ) != IVAS_ERR_OK ) { return error; } @@ -2444,6 +2858,11 @@ ivas_error IVAS_REND_RemoveInput( case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: clearInputSba( (input_sba *) inputBase ); break; +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + case IVAS_REND_AUDIO_CONFIG_TYPE_MASA: + clearInputMasa( (input_masa *) inputBase ); + break; +#endif default: return IVAS_ERR_INVALID_INPUT_FORMAT; } @@ -2534,6 +2953,17 @@ ivas_error IVAS_REND_GetDelay( } } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; i++ ) + { + if ( hIvasRend->inputsMasa[i].base.inConfig != IVAS_REND_AUDIO_CONFIG_UNKNOWN ) + { + latency_ns = NS2SA( hIvasRend->sampleRateOut, (int32_t) ( (float) IVAS_FB_DEC_DELAY_NS + 0.5f ) ); + *nSamples = max( *nSamples, NS2SA( *timeScale, latency_ns ) ); + } + } +#endif + return IVAS_ERR_OK; } @@ -2627,6 +3057,41 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( return IVAS_ERR_OK; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +ivas_error IVAS_REND_FeedInputMasaMetadata( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_InputId inputId, + IVAS_MASA_METADATA_HANDLE masaMetadata ) +{ + ivas_error error; + input_base *inputBase; + input_masa *inputMasa; + + /*-----------------------------------------------------------------* + * Validate function arguments + *-----------------------------------------------------------------*/ + + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + { + return error; + } + if ( getAudioConfigType( inputBase->inConfig ) != IVAS_REND_AUDIO_CONFIG_TYPE_MASA ) + { + /* MASA metadata should only be fed for MASA inputs */ + return IVAS_ERR_METADATA_NOT_EXPECTED; + } + + inputMasa = (input_masa *) inputBase; + inputMasa->masaMetadata = *masaMetadata; + + return IVAS_ERR_OK; +} +#endif + ivas_error IVAS_REND_SetHeadRotation( IVAS_REND_HANDLE hIvasRend, const IVAS_QUATERNION headRot[RENDERER_HEAD_POSITIONS_PER_FRAME] ) @@ -3885,6 +4350,174 @@ static ivas_error renderActiveInputsSba( return IVAS_ERR_OK; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER +static void copyMasaMetadataToDiracRenderer( MASA_METADATA_FRAME *meta, DIRAC_DEC_HANDLE hDirAC ) +{ + int16_t band, sf, bin; + + hDirAC->numSimultaneousDirections = meta->descriptive_meta.numberOfDirections + 1; + + for ( sf = 0; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) + { + for ( band = 0; band < MASA_MAXIMUM_CODING_SUBBANDS; band++ ) + { + for ( bin = MASA_band_grouping_24[band]; bin < MASA_band_grouping_24[band + 1]; bin++ ) + { + hDirAC->azimuth[sf][bin] = (int16_t) meta->directional_meta[0].azimuth[sf][band]; + hDirAC->elevation[sf][bin] = (int16_t) meta->directional_meta[0].elevation[sf][band]; + hDirAC->energy_ratio1[sf][bin] = meta->directional_meta[0].energy_ratio[sf][band]; + hDirAC->diffuseness_vector[sf][bin] = 1.0f - meta->directional_meta[0].energy_ratio[sf][band]; + hDirAC->spreadCoherence[sf][bin] = meta->directional_meta[0].spread_coherence[sf][band]; + hDirAC->surroundingCoherence[sf][bin] = meta->common_meta.surround_coherence[sf][band]; + + if ( hDirAC->numSimultaneousDirections == 2 ) + { + hDirAC->azimuth2[sf][bin] = (int16_t) meta->directional_meta[1].azimuth[sf][band]; + hDirAC->elevation2[sf][bin] = (int16_t) meta->directional_meta[1].elevation[sf][band]; + hDirAC->energy_ratio2[sf][bin] = meta->directional_meta[1].energy_ratio[sf][band]; + hDirAC->diffuseness_vector[sf][bin] -= meta->directional_meta[1].energy_ratio[sf][band]; + hDirAC->spreadCoherence2[sf][bin] = meta->directional_meta[1].spread_coherence[sf][band]; + } + } + } + } +} + +static ivas_error renderMasaToMc( input_masa *masaInput, IVAS_REND_AudioBuffer outAudio ) +{ + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); + + /* TODO(sgi): Remove code duplication w.r.t. MASA rendering to other output configs */ + if ( masaInput->decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) + { + ivas_dirac_dec_binaural( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); + } + else + { + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, NULL, NULL, -1 ); + } + + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); + + return IVAS_ERR_OK; +} + +static ivas_error renderMasaToSba( input_masa *masaInput, IVAS_REND_AudioBuffer outAudio ) +{ + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); + + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, NULL, NULL, -1 ); + + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); + + return IVAS_ERR_OK; +} + +static ivas_error renderMasaToBinaural( input_masa *masaInput, IVAS_REND_AudioBuffer outAudio ) +{ + float tmpBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; + + copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); + copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); + + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, NULL, NULL, -1 ); + + accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); + + return IVAS_ERR_OK; +} + +static ivas_error renderInputMasa( + input_masa *masaInput, + IVAS_REND_AudioConfig outConfig, + IVAS_REND_AudioBuffer outAudio ) +{ + ivas_error error; + IVAS_REND_AudioBuffer inAudio; + + inAudio = masaInput->base.inputBuffer; + + if ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) + { + /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ + return IVAS_ERR_INVALID_BUFFER_SIZE; + } + masaInput->base.numNewSamplesPerChannel = 0; + + /* Apply input gain to new audio */ + v_multc( inAudio.data, + masaInput->base.gain, + inAudio.data, + inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + + /* TODO(sgi): */ + switch ( getAudioConfigType( outConfig ) ) + { + case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: + error = renderMasaToMc( masaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_AMBISONICS: + error = renderMasaToSba( masaInput, outAudio ); + break; + case IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL: + switch ( outConfig ) + { + case IVAS_REND_AUDIO_CONFIG_BINAURAL: + error = renderMasaToBinaural( masaInput, outAudio ); + break; + /* ToDo */ + // case IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM: + // error = renderMasaToBinauralRoom( masaInput, outConfig, outAudio ); + // break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + break; + default: + return IVAS_ERR_INVALID_OUTPUT_FORMAT; + } + /* Check error here to keep switch statement more compact */ + if ( error != IVAS_ERR_OK ) + { + return error; + } + + return IVAS_ERR_OK; +} + +static ivas_error renderActiveInputsMasa( + IVAS_REND_HANDLE hIvasRend, + IVAS_REND_AudioBuffer outAudio ) +{ + int32_t i; + input_masa *pCurrentInput; + ivas_error error; + + for ( i = 0, pCurrentInput = hIvasRend->inputsMasa; i < RENDERER_MAX_MASA_INPUTS; ++i, ++pCurrentInput ) + { + if ( pCurrentInput->base.inConfig == IVAS_REND_AUDIO_CONFIG_UNKNOWN ) + { + /* Skip inactive inputs */ + continue; + } + if ( ( error = renderInputMasa( pCurrentInput, + hIvasRend->outputConfig, + outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } + } + + return IVAS_ERR_OK; +} +#endif + ivas_error IVAS_REND_GetSamples( IVAS_REND_HANDLE hIvasRend, IVAS_REND_AudioBuffer outAudio ) @@ -3938,6 +4571,12 @@ ivas_error IVAS_REND_GetSamples( { return error; } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + if ( ( error = renderActiveInputsMasa( hIvasRend, outAudio ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif #ifdef DEBUGGING hIvasRend->numClipping += diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index 3be9ad9964..4abc0b543e 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -227,11 +227,10 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( IVAS_REND_AudioObjectPosition objectPosition /* i : object position struct */ ); -/* Support for MASA input will be added in the future. */ ivas_error IVAS_REND_FeedInputMasaMetadata( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ IVAS_REND_InputId inputId, /* i : ID of the input */ - void* TODO + IVAS_MASA_METADATA_HANDLE masaMetadata /* i : MASA metadata frame */ ); ivas_error IVAS_REND_SetHeadRotation( diff --git a/scripts/tests/constants.py b/scripts/tests/constants.py index dcf8bbe3a9..670efaa40d 100644 --- a/scripts/tests/constants.py +++ b/scripts/tests/constants.py @@ -137,8 +137,8 @@ FORMAT_TO_METADATA_FILES = { str(TEST_VECTOR_DIR.joinpath("stvISM3.csv")), str(TEST_VECTOR_DIR.joinpath("stvISM4.csv")), ], - "MASA1": [str(TEST_VECTOR_DIR.joinpath("stv_IVASMASAQ_1dir1TC.met"))], - "MASA2": [str(TEST_VECTOR_DIR.joinpath("stv_IVASMASAQ_2dir2TC.met"))], + "MASA1": [str(TEST_VECTOR_DIR.joinpath("stv_IVASMASA_1dir1TC.met"))], + "MASA2": [str(TEST_VECTOR_DIR.joinpath("stv_IVASMASA_2dir2TC.met"))], } """ Input formats """ diff --git a/scripts/tests/test_renderer.py b/scripts/tests/test_renderer.py index ef304e9461..bf417371f9 100644 --- a/scripts/tests/test_renderer.py +++ b/scripts/tests/test_renderer.py @@ -325,22 +325,21 @@ def test_ism(test_info, in_fmt, out_fmt): check_BE(test_info, ref, ref_fs, cut, cut_fs) -# MASA inputs not supported yet -# @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -# @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -# def test_masa(test_info, in_fmt, out_fmt): -# # TODO: implement MASA in Python, compare BE -# # ref, ref_fs = run_pyscripts( -# # in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] -# # ) +@pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) +def test_masa(test_info, in_fmt, out_fmt): + # TODO: implement MASA in Python, compare BE + # ref, ref_fs = run_pyscripts( + # in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] + # ) -# cut, cut_fs = run_renderer( -# in_fmt, -# out_fmt, -# in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], -# ) + cut, cut_fs = run_renderer( + in_fmt, + out_fmt, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + ) -# # check_BE(test_info, ref, ref_fs, cut, cut_fs) + # check_BE(test_info, ref, ref_fs, cut, cut_fs) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) -- GitLab From 0e656b557da162c696ddaf109134bd40ac690bcc Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Wed, 19 Oct 2022 13:58:24 +0200 Subject: [PATCH 02/20] Try to fix instrumented build --- lib_rend/lib_rend.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 83a0bc1252..5c4098d2a6 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -1971,7 +1971,9 @@ static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_RE } decDummy->ivas_format = MASA_FORMAT; decDummy->transport_config = AUDIO_CONFIG_INVALID; +#ifdef DEBUGGING decDummy->hDecoderConfig->forceSubframeBinauralization = 0; +#endif if ( ( error = ivas_dirac_dec_open( decDummy ) ) != IVAS_ERR_OK ) { -- GitLab From e9657705aaeb5c0457163415e9d2619e582b15e4 Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Wed, 19 Oct 2022 14:12:03 +0200 Subject: [PATCH 03/20] Fix Make build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9514e9a582..c72afb6fb3 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ $(CLI_APIDEC): $(OBJS_CLI_APIDEC) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(L $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APIDEC) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(CLI_APIDEC) $(CLI_APIREND): $(OBJS_CLI_APPREND) $(LIB_LIBREND) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) - $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasutil -livasdebug -livascom $(LDLIBS) -o $(CLI_APIREND) + $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasutil -livasdebug -livascom -livasdec $(LDLIBS) -o $(CLI_APIREND) $(CLI_UTESTS_CREND): $(OBJS_CLI_UTESTS_CREND) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_UTESTS_CREND) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(UTESTS_CREND_DIR)/$(CLI_UTESTS_CREND) -- GitLab From 9e416500a35039345597564bfb816fe525e1054f Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Wed, 19 Oct 2022 14:18:02 +0200 Subject: [PATCH 04/20] Change linking order so that it also works on Linux --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c72afb6fb3..0a023a17d9 100644 --- a/Makefile +++ b/Makefile @@ -185,7 +185,7 @@ $(CLI_APIDEC): $(OBJS_CLI_APIDEC) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(L $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APIDEC) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(CLI_APIDEC) $(CLI_APIREND): $(OBJS_CLI_APPREND) $(LIB_LIBREND) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) - $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasutil -livasdebug -livascom -livasdec $(LDLIBS) -o $(CLI_APIREND) + $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasdec -livasutil -livasdebug -livascom $(LDLIBS) -o $(CLI_APIREND) $(CLI_UTESTS_CREND): $(OBJS_CLI_UTESTS_CREND) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_UTESTS_CREND) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(UTESTS_CREND_DIR)/$(CLI_UTESTS_CREND) -- GitLab From 55ef9075a6b5e18c32fcd5dadfe3b6b4b5ea0475 Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Wed, 19 Oct 2022 14:35:36 +0200 Subject: [PATCH 05/20] Another attempt to fix Make --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0a023a17d9..141a607b07 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ $(CLI_APIENC): $(OBJS_CLI_APIENC) $(LIB_LIBENC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(L $(CLI_APIDEC): $(OBJS_CLI_APIDEC) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APIDEC) -L. -livasdec -livascom -livasutil -livasdebug $(LDLIBS) -o $(CLI_APIDEC) -$(CLI_APIREND): $(OBJS_CLI_APPREND) $(LIB_LIBREND) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) +$(CLI_APIREND): $(OBJS_CLI_APPREND) $(LIB_LIBREND) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) $(LIB_LIBDEC) $(QUIET_LINK)$(CC) $(LDFLAGS) $(OBJS_CLI_APPREND) -L. -livasrend -livasdec -livasutil -livasdebug -livascom $(LDLIBS) -o $(CLI_APIREND) $(CLI_UTESTS_CREND): $(OBJS_CLI_UTESTS_CREND) $(LIB_LIBDEC) $(LIB_LIBCOM) $(LIB_LIBUTIL) $(LIB_LIBDEBUG) -- GitLab From 81f8749d9531f514e9d558832a739da36034dc7f Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 28 Oct 2022 13:47:33 +0300 Subject: [PATCH 06/20] Add fix to SBA mode being uninitialized. --- lib_rend/lib_rend.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 5c4098d2a6..4d5bc9ddf2 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -1819,6 +1819,8 @@ static ivas_error initMasaDummyDecForMcOut( input_masa *inputMasa, IVAS_REND_Aud decDummy->hDecoderConfig->output_config = output_config; decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + ivas_output_init( &( decDummy->hOutSetup ), output_config ); ivas_output_init( &( decDummy->hIntSetup ), output_config ); @@ -1896,6 +1898,8 @@ static ivas_error initMasaDummyDecForSbaOut( input_masa *inputMasa, IVAS_REND_Au decDummy->hDecoderConfig->output_config = output_config; decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + ivas_output_init( &( decDummy->hOutSetup ), output_config ); ivas_output_init( &( decDummy->hIntSetup ), output_config ); decDummy->renderer_type = RENDERER_DIRAC; @@ -1960,6 +1964,8 @@ static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_RE output_config = decDummy->hDecoderConfig->output_config; decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + ivas_output_init( &( decDummy->hOutSetup ), output_config ); if ( output_config == AUDIO_CONFIG_BINAURAL ) { -- GitLab From 110e4616245df651ba2a7696a8c241c6fc76a8e4 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 28 Oct 2022 15:29:58 +0300 Subject: [PATCH 07/20] Adds early exit for MASA1 until there is possibility to fix it. --- apps/renderer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/renderer.c b/apps/renderer.c index a851c1fa2e..38f2737b07 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1321,6 +1321,8 @@ static IVAS_REND_AudioConfig parseAudioConfig( const char *configString ) switch ( charBuf[4] ) { case '1': + fprintf(stderr, "1TC MASA support is not functional and is pending on DirAC renderer refactoring.\n"); + exit(EXIT_FAILURE); return IVAS_REND_AUDIO_CONFIG_MASA1; case '2': return IVAS_REND_AUDIO_CONFIG_MASA2; -- GitLab From 0a453214da5489d5d38bfbf1acc0591296b64db7 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Fri, 28 Oct 2022 15:46:33 +0300 Subject: [PATCH 08/20] Disable reserveration of head track data for decoder dummy when not rendering to binaural. --- lib_rend/lib_rend.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 4d5bc9ddf2..2524f4baa6 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2069,18 +2069,25 @@ static DecoderDummy *initDecoderDummy( int32_t sampleRate, int16_t numTransChann decDummy->hDecoderConfig->output_config = rendAudioConfigToIvasAudioConfig(outConfig); decDummy->nchan_transport = numTransChannels; - decDummy->hHeadTrackData = count_malloc( sizeof( HEAD_TRACK_DATA ) ); - /* Initialise Rmat_prev to I, Rmat will be computed later */ - for ( i = 0; i < 3; i++ ) + if ( outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM || outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM ) { - set_zero( decDummy->hHeadTrackData->Rmat_prev[i], 3 ); - decDummy->hHeadTrackData->Rmat_prev[i][i] = 1.0f; - } + decDummy->hHeadTrackData = count_malloc( sizeof( HEAD_TRACK_DATA ) ); + /* Initialise Rmat_prev to I, Rmat will be computed later */ + for ( i = 0; i < 3; i++ ) + { + set_zero( decDummy->hHeadTrackData->Rmat_prev[i], 3 ); + decDummy->hHeadTrackData->Rmat_prev[i][i] = 1.0f; + } - decDummy->hHeadTrackData->num_quaternions = 0; - decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; - decDummy->hHeadTrackData->lrSwitchedCurrent = 0; - decDummy->hHeadTrackData->lrSwitchedNext = 0; + decDummy->hHeadTrackData->num_quaternions = 0; + decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; + decDummy->hHeadTrackData->lrSwitchedCurrent = 0; + decDummy->hHeadTrackData->lrSwitchedNext = 0; + } + else + { + decDummy->hHeadTrackData = NULL; + } if ( enableRenderConfig ) { -- GitLab From 6fa665ec59ea5a9a59bfb1c8adf3638bc96ae019 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 31 Oct 2022 09:19:28 +0200 Subject: [PATCH 09/20] Fix decoder dummy init for MASA renderer for MC mode and subframe mode. --- lib_rend/lib_rend.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 2524f4baa6..18ae23f815 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -1820,6 +1820,7 @@ static ivas_error initMasaDummyDecForMcOut( input_masa *inputMasa, IVAS_REND_Aud decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ ivas_output_init( &( decDummy->hOutSetup ), output_config ); ivas_output_init( &( decDummy->hIntSetup ), output_config ); @@ -1899,6 +1900,7 @@ static ivas_error initMasaDummyDecForSbaOut( input_masa *inputMasa, IVAS_REND_Au decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ ivas_output_init( &( decDummy->hOutSetup ), output_config ); ivas_output_init( &( decDummy->hIntSetup ), output_config ); @@ -1965,6 +1967,7 @@ static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_RE decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ ivas_output_init( &( decDummy->hOutSetup ), output_config ); if ( output_config == AUDIO_CONFIG_BINAURAL ) @@ -1990,6 +1993,8 @@ static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_RE return error; } + decDummy->hDiracDecBin->useSubframeMode = 0; /* Todo Nokia: This will disappear in later work but needs to be this now. */ + for ( i = 0; i < BINAURAL_CHANNELS; i++ ) { if ( ( error = openCldfb( &( decDummy->cldfbAnaDec[i] ), CLDFB_ANALYSIS, decDummy->hDecoderConfig->output_Fs, CLDFB_PROTOTYPE_5_00MS ) ) != IVAS_ERR_OK ) -- GitLab From 1ad74401603a17256b5aab65893555dc1da2e404 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 31 Oct 2022 09:31:57 +0200 Subject: [PATCH 10/20] Removes default MASA metadata functionality from external renderer. --- lib_rend/lib_rend.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 18ae23f815..c42c712e61 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2110,15 +2110,6 @@ static DecoderDummy *initDecoderDummy( int32_t sampleRate, int16_t numTransChann return decDummy; } -static MASA_METADATA_FRAME defaultMasaMetadata( void ) -{ - MASA_METADATA_FRAME mf; - - /* TODO @ Nokia: Is there a sensible default here that can be used in case no metadata is fed? */ - - return mf; -} - static ivas_error setRendInputActiveMasa( void *input, IVAS_REND_AudioConfig inConfig, @@ -2142,7 +2133,6 @@ static ivas_error setRendInputActiveMasa( } /* TODO(sgi): Don't hard-code enableRenderConfig here (last argument) */ inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, (int16_t)numInChannels, outConfig, 0 ); - inputMasa->masaMetadata = defaultMasaMetadata(); if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { -- GitLab From 64c9d8ec4026d86034a899ecd0eaeb4c7d19fe28 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 31 Oct 2022 09:48:28 +0200 Subject: [PATCH 11/20] Temporarily disables MASA1 from external renderer tests until it is possible to implement the feature. --- scripts/tests/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/constants.py b/scripts/tests/constants.py index 670efaa40d..36c747c21d 100644 --- a/scripts/tests/constants.py +++ b/scripts/tests/constants.py @@ -145,7 +145,7 @@ FORMAT_TO_METADATA_FILES = { INPUT_FORMATS_AMBI = ["FOA", "HOA2", "HOA3"] INPUT_FORMATS_MC = ["MONO", "STEREO", "5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"] INPUT_FORMATS_ISM = ["ISM1", "ISM2", "ISM3", "ISM4"] -INPUT_FORMATS_MASA = ["MASA1", "MASA2"] +INPUT_FORMATS_MASA = ["MASA2"] #["MASA1", "MASA2"] # Disable MASA1 tests until MASA1 can be implemented properly """ Non binaural / parametric output formats """ OUTPUT_FORMATS = [ -- GitLab From 1385334b3fc71b93f2951a038d561303ae30f053 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 31 Oct 2022 09:54:09 +0200 Subject: [PATCH 12/20] Fixes dummy decoder subframe mode also for stereo output. --- lib_rend/lib_rend.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index c42c712e61..de37d2e6da 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -1852,6 +1852,7 @@ static ivas_error initMasaDummyDecForMcOut( input_masa *inputMasa, IVAS_REND_Aud { return error; } + decDummy->hDiracDecBin->useSubframeMode = 0; /* Todo Nokia: This will disappear in later work but needs to be this now. */ } numCldfbAnalyses = decDummy->nchan_transport; -- GitLab From 85b78c9b6b42b69dbcc3924ba017bd62dc7a554c Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 31 Oct 2022 14:31:19 +0200 Subject: [PATCH 13/20] Fixes releases of MASA renderer memories and correct binaural renderer function for MASA. --- lib_rend/lib_rend.c | 60 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index de37d2e6da..566bbcfbdc 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2145,6 +2145,7 @@ static ivas_error setRendInputActiveMasa( static void freeDecoderDummy( DecoderDummy **ppDecDummy ) { + int16_t i; DecoderDummy *pDecDummy; if ( ppDecDummy == NULL || *ppDecDummy == NULL ) @@ -2155,13 +2156,60 @@ static void freeDecoderDummy( DecoderDummy **ppDecDummy ) if ( pDecDummy->hDecoderConfig != NULL ) { - free( pDecDummy->hDecoderConfig ); + count_free( pDecDummy->hDecoderConfig ); } if ( pDecDummy->hHeadTrackData != NULL ) { - free( pDecDummy->hHeadTrackData ); + count_free( pDecDummy->hHeadTrackData ); } ivas_render_config_close( &pDecDummy->hRenderConfig ); + +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + /* CLDFB handles */ + for ( i = 0; i < MAX_INTERN_CHANNELS; i++ ) + { + if ( pDecDummy->cldfbAnaDec[i] != NULL ) + { + deleteCldfb( &( pDecDummy->cldfbAnaDec[i] ) ); + pDecDummy->cldfbAnaDec[i] = NULL; + } + } + + for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) + { + if ( pDecDummy->cldfbSynDec[i] != NULL ) + { + deleteCldfb( &( pDecDummy->cldfbSynDec[i] ) ); + pDecDummy->cldfbSynDec[i] = NULL; + } + } + + /* DirAC handle */ + if ( pDecDummy->hDirAC != NULL ) + { + ivas_dirac_dec_close( pDecDummy->hDirAC ); + pDecDummy->hDirAC = NULL; + } + + /* Qmetadata handle */ + ivas_qmetadata_close( &pDecDummy->hQMetaData ); + + /* VBAP handle */ + vbap_free_data( &( pDecDummy->hVBAPdata ) ); + + /* HOA decoder matrix */ + if ( pDecDummy->hoa_dec_mtx != NULL ) + { + count_free( pDecDummy->hoa_dec_mtx ); + pDecDummy->hoa_dec_mtx = NULL; + } + + /* Parametric binaural renderer handle */ + ivas_dirac_dec_close_binaural_data( &pDecDummy->hDiracDecBin ); +#endif + + count_free( pDecDummy ); + pDecDummy = NULL; } static void clearInputMasa( input_masa *inputMasa ) @@ -4437,7 +4485,7 @@ static ivas_error renderMasaToBinaural( input_masa *masaInput, IVAS_REND_AudioBu copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer ); copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hDirAC ); - ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels, NULL, NULL, -1 ); + ivas_dirac_dec_binaural( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); accumulate2dArrayToBuffer( tmpBuffer, &outAudio ); @@ -4630,6 +4678,12 @@ void IVAS_REND_Close( IVAS_REND_HANDLE *phIvasRend ) { clearInputSba( &hIvasRend->inputsSba[i] ); } +#ifdef NOKIA_MASA_EXTERNAL_RENDERER + for ( i = 0; i < RENDERER_MAX_MASA_INPUTS; ++i ) + { + clearInputMasa( &hIvasRend->inputsMasa[i] ); + } +#endif ivas_limiter_close( &hIvasRend->hLimiter ); -- GitLab From 16f40abe6ccd7db06a3ba8db26ff8f12f38f0987 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Wed, 2 Nov 2022 15:37:24 +0200 Subject: [PATCH 14/20] Fixes build when NOKIA_MASA_EXTERNAL_RENDERER is disabled. --- apps/renderer.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/renderer.c b/apps/renderer.c index f862ff2e72..4e6a0e2988 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -769,7 +769,12 @@ int32_t main( int32_t argc, char **argv ) } #endif +#ifdef NOKIA_MASA_EXTERNAL_RENDERER const int32_t totalNumInChannels = getTotalNumInChannels( hIvasRend, mcIds, ismIds, sbaIds, masaIds ); +#else + const int32_t totalNumInChannels = getTotalNumInChannels( hIvasRend, mcIds, ismIds, sbaIds ); +#endif + if ( AudioFileReader_getNumChannels( audioReader ) != 0 /* If input file is raw PCM, audio reader has no info about number of channels */ && totalNumInChannels != AudioFileReader_getNumChannels( audioReader ) ) { -- GitLab From 55e5bf076fe4fd78a2988a56f0fc1efa38a13d49 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Wed, 2 Nov 2022 15:50:07 +0200 Subject: [PATCH 15/20] Adjust linking of lib_dec to lib_rend to fix CMake build with GCC. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 721abd980c..24c9b136a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,7 +144,7 @@ target_link_libraries(lib_enc lib_com lib_debug) file(GLOB libRendSrcs "lib_rend/*.c") file(GLOB libRendHeaders "lib_rend/*.h") add_library(lib_rend ${libRendSrcs} ${libRendHeaders}) -target_link_libraries(lib_rend lib_com lib_debug) +target_link_libraries(lib_rend lib_dec lib_com lib_debug) # Todo refactor: This dependency on lib_dec should be removed. file(GLOB libDecSrcs "lib_dec/*.c") file(GLOB libDecHeaders "lib_dec/*.h") @@ -176,7 +176,7 @@ if(WIN32) endif() add_executable(IVAS_rend apps/renderer.c) -target_link_libraries(IVAS_rend lib_rend lib_util lib_dec) # Todo refactor: dependency to lib_dec should be removed +target_link_libraries(IVAS_rend lib_rend lib_util) if(COPY_EXECUTABLES_FROM_BUILD_DIR) # Optionally copy executables to the same place where Make puts them (useful for tests that expect executables in specific places) -- GitLab From 086e3e7d0bb9bf861ae407e4aa98880753e3fe74 Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Fri, 4 Nov 2022 11:32:49 +0100 Subject: [PATCH 16/20] Add check for valid MASA metadata before rendering --- lib_com/ivas_error.h | 1 + lib_rend/lib_rend.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib_com/ivas_error.h b/lib_com/ivas_error.h index 8168a6c6d6..ea08aa9e8b 100644 --- a/lib_com/ivas_error.h +++ b/lib_com/ivas_error.h @@ -92,6 +92,7 @@ typedef enum IVAS_ERR_INVALID_INPUT_ID, IVAS_ERR_WRONG_NUM_CHANNELS, IVAS_ERR_INVALID_BUFFER_SIZE, + IVAS_ERR_MISSING_METADATA, #endif /*----------------------------------------* diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index f50642d433..d060734462 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -145,6 +145,7 @@ typedef struct input_base base; DecoderDummy *decDummy; MASA_METADATA_FRAME masaMetadata; + bool metadataHasBeenFed; } input_masa; #endif @@ -2130,8 +2131,8 @@ static ivas_error setRendInputActiveMasa( { return error; } - /* TODO(sgi): Don't hard-code enableRenderConfig here (last argument) */ inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, (int16_t)numInChannels, outConfig, 0 ); + inputMasa->metadataHasBeenFed = false; if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) { @@ -2314,6 +2315,7 @@ ivas_error IVAS_REND_Open( 0, getRendCtx( hIvasRend ) ); hIvasRend->inputsMasa[i].decDummy = NULL; + hIvasRend->inputsMasa[i].metadataHasBeenFed = false; } #endif @@ -3144,6 +3146,7 @@ ivas_error IVAS_REND_FeedInputMasaMetadata( inputMasa = (input_masa *) inputBase; inputMasa->masaMetadata = *masaMetadata; + inputMasa->metadataHasBeenFed = true; return IVAS_ERR_OK; } @@ -4560,6 +4563,11 @@ static ivas_error renderInputMasa( ivas_error error; IVAS_REND_AudioBuffer inAudio; + if ( !masaInput->metadataHasBeenFed ) + { + return IVAS_ERR_MISSING_METADATA; + } + inAudio = masaInput->base.inputBuffer; if ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) @@ -4575,7 +4583,6 @@ static ivas_error renderInputMasa( inAudio.data, inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); - /* TODO(sgi): */ switch ( getAudioConfigType( outConfig ) ) { case IVAS_REND_AUDIO_CONFIG_TYPE_CHANNEL_BASED: -- GitLab From 2cb96db276e4e5e8fc33460c14352724ab8a7cbd Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Thu, 10 Nov 2022 14:24:21 +0200 Subject: [PATCH 17/20] Fix build warnings. --- apps/renderer.c | 1 - lib_rend/lib_rend.c | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/renderer.c b/apps/renderer.c index 4442301466..940e6be024 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -493,7 +493,6 @@ static int16_t getTotalNumInChannels( continue; } - int32_t numInputChannels; if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, masaIds[i], &numInputChannels ) ) != IVAS_ERR_OK ) { fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index be6e672ce7..76d1e7f45d 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -2190,6 +2190,7 @@ static ivas_error setRendInputActiveMasa( inputMasa = (input_masa *) input; rendCtx = inputMasa->base.ctx; outConfig = *rendCtx.pOutConfig; + (void) hRendCfg; /* Suppress warning */ initRendInputBase( &inputMasa->base, inConfig, id, rendCtx ); -- GitLab From 403f71c75116afa9fd1154cb1d06638e52e3d830 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Thu, 10 Nov 2022 14:25:58 +0200 Subject: [PATCH 18/20] Apply clang format. --- apps/renderer.c | 4 ++-- lib_rend/lib_rend.c | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/renderer.c b/apps/renderer.c index 940e6be024..0d46dd05f4 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -1388,8 +1388,8 @@ static IVAS_REND_AudioConfig parseAudioConfig( switch ( charBuf[4] ) { case '1': - fprintf(stderr, "1TC MASA support is not functional and is pending on DirAC renderer refactoring.\n"); - exit(EXIT_FAILURE); + fprintf( stderr, "1TC MASA support is not functional and is pending on DirAC renderer refactoring.\n" ); + exit( EXIT_FAILURE ); return IVAS_REND_AUDIO_CONFIG_MASA1; case '2': return IVAS_REND_AUDIO_CONFIG_MASA2; diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 76d1e7f45d..63ea1c3c1a 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -1883,8 +1883,8 @@ static ivas_error initMasaDummyDecForMcOut( input_masa *inputMasa, IVAS_REND_Aud decDummy->hDecoderConfig->output_config = output_config; decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ - decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ - decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ ivas_output_init( &( decDummy->hOutSetup ), output_config ); ivas_output_init( &( decDummy->hIntSetup ), output_config ); @@ -1964,8 +1964,8 @@ static ivas_error initMasaDummyDecForSbaOut( input_masa *inputMasa, IVAS_REND_Au decDummy->hDecoderConfig->output_config = output_config; decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ - decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ - decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ ivas_output_init( &( decDummy->hOutSetup ), output_config ); ivas_output_init( &( decDummy->hIntSetup ), output_config ); @@ -2031,8 +2031,8 @@ static ivas_error initMasaDummyDecForBinauralOut( input_masa *inputMasa, IVAS_RE output_config = decDummy->hDecoderConfig->output_config; decDummy->hDecoderConfig->ivas_total_brate = IVAS_512k; /* Todo Nokia: This is preventing initialization of 2TC as 1TC, should be fixed properly in ivas_dirac_dec_config() */ - decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ - decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ + decDummy->sba_mode = SBA_MODE_NONE; /* Todo Nokia: This is done to prevent ivas_dirac_dec_config() to not use uninitialized value. It could be considered if this should not be even accessed when not in SBA. */ + decDummy->mc_mode = MC_MODE_NONE; /* Todo Nokia: This should be also refactored in such way that it is not checked if not in MC mode */ ivas_output_init( &( decDummy->hOutSetup ), output_config ); if ( output_config == AUDIO_CONFIG_BINAURAL ) @@ -2117,13 +2117,13 @@ static DecoderDummy *initDecoderDummy( int32_t sampleRate, int16_t numTransChann if ( ( error = getAudioConfigNumChannels( outConfig, &numOutChannels ) ) != IVAS_ERR_OK ) { /* Checking error with assert is enough, this function is only temporary anyway */ - assert(error == IVAS_ERR_OK); + assert( error == IVAS_ERR_OK ); } decDummy = count_malloc( sizeof( DecoderDummy ) ); decDummy->hDecoderConfig = count_malloc( sizeof( DECODER_CONFIG ) ); decDummy->hDecoderConfig->output_Fs = sampleRate; - decDummy->hDecoderConfig->nchan_out = (int16_t)numOutChannels; + decDummy->hDecoderConfig->nchan_out = (int16_t) numOutChannels; decDummy->hDecoderConfig->Opt_Headrotation = 0; decDummy->hBinRenderer = NULL; @@ -2136,7 +2136,7 @@ static DecoderDummy *initDecoderDummy( int32_t sampleRate, int16_t numTransChann decDummy->hMasa = NULL; decDummy->hDiracDecBin = NULL; decDummy->hQMetaData = NULL; - decDummy->hDecoderConfig->output_config = rendAudioConfigToIvasAudioConfig(outConfig); + decDummy->hDecoderConfig->output_config = rendAudioConfigToIvasAudioConfig( outConfig ); decDummy->nchan_transport = numTransChannels; if ( outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM || outConfig == IVAS_REND_AUDIO_CONFIG_BINAURAL_ROOM ) @@ -2198,7 +2198,7 @@ static ivas_error setRendInputActiveMasa( { return error; } - inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, (int16_t)numInChannels, outConfig, 0 ); + inputMasa->decDummy = initDecoderDummy( *rendCtx.pOutSampleRate, (int16_t) numInChannels, outConfig, 0 ); inputMasa->metadataHasBeenFed = false; if ( ( error = updateMasaDummyDec( inputMasa, outConfig ) ) != IVAS_ERR_OK ) -- GitLab From 8db5bc2502b5717109485d19383bed0d7934097a Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Thu, 10 Nov 2022 14:41:58 +0200 Subject: [PATCH 19/20] Correct unintended const change. --- lib_rend/lib_rend.c | 2 +- lib_rend/lib_rend.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 63ea1c3c1a..caff4d52c0 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -3172,7 +3172,7 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( #ifdef NOKIA_MASA_EXTERNAL_RENDERER ivas_error IVAS_REND_FeedInputMasaMetadata( IVAS_REND_HANDLE hIvasRend, - IVAS_REND_InputId inputId, + const IVAS_REND_InputId inputId, IVAS_MASA_METADATA_HANDLE masaMetadata ) { ivas_error error; diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index 225e9121cd..158a734f46 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -232,14 +232,14 @@ ivas_error IVAS_REND_FeedInputObjectMetadata( #ifdef NOKIA_MASA_EXTERNAL_RENDERER ivas_error IVAS_REND_FeedInputMasaMetadata( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ IVAS_MASA_METADATA_HANDLE masaMetadata /* i : MASA metadata frame */ ); #else /* Support for MASA input will be added in the future. */ ivas_error IVAS_REND_FeedInputMasaMetadata( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_REND_InputId inputId, /* i : ID of the input */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ void* TODO ); #endif -- GitLab From 8d4acdfbeff46dbb99388d4acc84a19aec8acfcf Mon Sep 17 00:00:00 2001 From: Archit Tamarapu Date: Thu, 10 Nov 2022 21:55:48 +0100 Subject: [PATCH 20/20] [tests] revert change to constants.py --- tests/renderer/constants.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 68b40a9454..da7302bdc2 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -188,6 +188,40 @@ FORMAT_TO_METADATA_FILES = { "MASA2": [str(TESTV_DIR.joinpath("stv_IVASMASA_2dir2TC.met"))], } +FORMAT_TO_IVAS = { + "MONO": ["", ""], + "STEREO": ["-stereo", ""], + "FOA": ["-sba", "1"], + "HOA2": ["-sba", "2"], + "HOA3": ["-sba", "3"], + "5_1": ["-mc", "5_1"], + "7_1": ["-mc", "7_1"], + "5_1_2": ["-mc", "5_1_2"], + "5_1_4": ["-mc", "5_1_4"], + "7_1_4": ["-mc", "7_1_4"], + "ISM1": ["-ism", "1"], + "ISM2": ["-ism", "2"], + "ISM3": ["-ism", "3"], + "ISM4": ["-ism", "4"], +} + +FORMAT_TO_IVAS_BR = { + "MONO": "128000", + "STEREO": "256000", + "FOA": "512000", + "HOA2": "512000", + "HOA3": "512000", + "5_1": "512000", + "7_1": "512000", + "5_1_2": "512000", + "5_1_4": "512000", + "7_1_4": "512000", + "ISM1": "256000", + "ISM2": "256000", + "ISM3": "256000", + "ISM4": "256000", +} + FORMAT_TO_CREND_FORMAT = { "MONO": "0", "STEREO": "1", -- GitLab