diff --git a/.gitignore b/.gitignore index 9de6ea6bb4c54a975fb5c7da0db0815389669ac0..2024f865f12beaeeb9f2e46dfaa14a5ae851c790 100644 --- a/.gitignore +++ b/.gitignore @@ -52,9 +52,9 @@ scripts/testv/stvOMASA_*.csv scripts/testv/stvOMASA_2ISM_1MASA1TC48c.wav scripts/testv/stvOMASA_3ISM_1MASA1TC48c.wav # default reference binary name -IVAS_cod_ref -IVAS_dec_ref -IVAS_rend_ref +IVAS_cod_ref* +IVAS_dec_ref* +IVAS_rend_ref* # Python files that pop up when running scripts __pycache__/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a69b946bd918dd93431f2e3f0f5ccd821c9074f..31fb259ac0be07813bbb5726c7e6c8f78012149c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,12 +68,12 @@ stages: # needed when depth is lower than the number of commits in the branch - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -.mr-get-target-commit: &mr-get-target-commit - # compare to last target branch commit before pipeline was created +.mr-get-target-commit: + &mr-get-target-commit # compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) -.check-for-testvectors: &check-for-testvectors - # check if the testvector files specified in scripts/config/ci_linux*.json are present +.check-for-testvectors: + &check-for-testvectors # check if the testvector files specified in scripts/config/ci_linux*.json are present - python3 -m pytest ci/test_vectors_available.py .merge-request-comparison-setup-codec: @@ -141,7 +141,7 @@ stages: when: never - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) when: never - - if: $CI_PIPELINE_SOURCE == 'trigger' # Don't run triggered pipeline by default + - if: $CI_PIPELINE_SOURCE == 'trigger' # Don't run triggered pipeline by default when: never - when: on_success @@ -162,7 +162,6 @@ stages: rules: - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - # templates to define stages and platforms .test-job-linux: tags: @@ -202,7 +201,6 @@ stages: exit_codes: - 123 - # --------------------------------------------------------------- # .pre jobs for setting up things # --------------------------------------------------------------- @@ -225,8 +223,6 @@ uninterruptible: tags: - ivas-linux - - # --------------------------------------------------------------- # Validation jobs # --------------------------------------------------------------- @@ -245,7 +241,6 @@ check-if-branch-is-up-to-date-with-main: - echo $commits_behind_count - if [ $commits_behind_count -eq 0 ]; then exit 0; else exit 1; fi; - # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- @@ -448,7 +443,7 @@ renderer-asan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py - + artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -471,7 +466,7 @@ renderer-msan: - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py - + artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -526,8 +521,7 @@ renderer-pytest-on-merge-request: # write to temporary file as workaround for failures observed with piping echo - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true - # TODO: needs splitting the test between reference and cut generation - #- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true # store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) @@ -545,6 +539,9 @@ renderer-pytest-on-merge-request: - make clean - make -j IVAS_rend + ### Run test using scripts and input from main + - if [ $ref_using_main == 1 ]; then git checkout $target_commit; fi + # run test - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? @@ -603,10 +600,9 @@ split-rendering-pytest-on-merge-request: # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true - # TODO: needs splitting the test between reference and cut generation - #- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true # store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) @@ -629,6 +625,9 @@ split-rendering-pytest-on-merge-request: - *enable-split-rendering - make -j INCLUDE_SPLIT=1 + ### Run test using scripts and input from main + - if [ $ref_using_main == 1 ]; then git checkout $target_commit; fi + # run test - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py || exit_code=$? @@ -665,7 +664,7 @@ ivas-pytest-on-merge-request: # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true @@ -714,10 +713,10 @@ evs-pytest-on-merge-request: script: - *print-common-info - *merge-request-comparison-setup-codec - + # some helper variables - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo - - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[evs[ -]*non[ -]*be\]" tmp.txt) || true - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true @@ -787,7 +786,7 @@ clang-format-check: - place the patch file in the root directory of your local IVAS repo\n - run: git apply $PATCH_FILE_NAME\n - commit new changes" - + - scripts/check-format.sh -af -p 8 || format_problems=$? - if [ $format_problems == 0 ] ; then exit 0; fi @@ -807,7 +806,7 @@ clang-format-check: - tmp-formatting-fix/ when: on_failure name: "$ARTIFACT_BASE_NAME" - expose_as: 'formatting patch' + expose_as: "formatting patch" # check for crashes if first received frame on decoder side is an SID check-first-frame-is-sid: @@ -896,7 +895,6 @@ lc3plus-ensure-no-code-changes: - modified_files=$(git status -s) - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi - # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- @@ -993,7 +991,7 @@ codec-comparison-on-main-push: # helper variable - "|| true" to prevent failures from grep not finding anything # write to temporary file as workaround for failures observed with piping echo - - echo $CI_COMMIT_MESSAGE > tmp.txt + - echo $CI_COMMIT_MESSAGE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true @@ -1060,7 +1058,6 @@ codec-comparison-on-main-push: extends: - .sanitizer-test-template - sanitizer-test-mono: extends: .sanitizer-test-schedule-A rules: @@ -1275,7 +1272,6 @@ sanitizer-test-ism+4: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py ISM+4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS - # GCOV/LCOV coverage analysis of self_test suite coverage-test-on-main-scheduled: extends: @@ -1291,7 +1287,7 @@ coverage-test-on-main-scheduled: script: - *print-common-info - make GCOV=1 -j - - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script + - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script - python3 tests/create_short_testvectors.py - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec @@ -1310,13 +1306,12 @@ coverage-test-on-main-scheduled: - coverage.info - coverage - # --------------------------------------------------------------- # Complexity measurement jobs # --------------------------------------------------------------- -.complexity-measurements-setup: &complexity-measurements-setup - # create necessary environment +.complexity-measurements-setup: + &complexity-measurements-setup # create necessary environment - mkdir -p wmops/logs - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME) @@ -1331,8 +1326,8 @@ coverage-test-on-main-scheduled: - rm artifacts.zip - rm -rf $public_dir -.complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts - # prepare artifacts -> move to public directory +.complexity-measurements-prepare-artifacts: + &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory - public_dir="$CI_JOB_NAME-public" - mkdir $public_dir - mv -f wmops/log_*_all.txt wmops/*.js ${public_dir}/ @@ -1478,8 +1473,8 @@ complexity-StereoDmxEVS-stereo-in-mono-out: upload-selection-BE-log: rules: - - if: $UPLOAD_SELECTION_BE_RESULTS && $CI_PIPELINE_SOURCE == 'trigger' - when: always + - if: $UPLOAD_SELECTION_BE_RESULTS && $CI_PIPELINE_SOURCE == 'trigger' + when: always timeout: 5 minutes tags: - ericsson-windows-runner diff --git a/apps/decoder.c b/apps/decoder.c index a3e9fb000a36132ccabbfc1a4bf4c140b121252c..8c06e7c168dc87c464ff608fc85ce7805631e1cd 100644 --- a/apps/decoder.c +++ b/apps/decoder.c @@ -82,6 +82,10 @@ static #define VARIABLE_SPEED_FETCH_FRAMESIZE_MS 20 #endif #define JBM_FRONTEND_FETCH_FRAMESIZE_MS 20 +#ifdef API_5MS +#define HEADROTATION_FETCH_FRAMESIZE_MS 5 +#define DEFAULT_FETCH_FRAMESIZE_MS 20 +#endif /*------------------------------------------------------------------------------------------* @@ -126,7 +130,12 @@ typedef struct char *outputMdFilename; #endif IVAS_DEC_COMPLEXITY_LEVEL complexityLevel; - +#ifdef API_5MS + bool tsmEnabled; +#ifdef API_5MS_BASELINE + bool enable5ms; +#endif +#endif #ifdef DEBUGGING IVAS_DEC_FORCED_REND_MODE forcedRendMode; #ifdef DEBUG_FOA_AGC @@ -137,7 +146,9 @@ typedef struct uint16_t frontendFetchSizeMs; #endif #ifdef VARIABLE_SPEED_DECODING +#ifndef API_5MS bool variableSpeedMode; +#endif bool tsmScaleFileEnabled; char *tsmScaleFileName; uint16_t tsmScale; @@ -429,6 +440,45 @@ int main( } } +#ifdef API_5MS + /*-------------------------------------------------------------------* + * Load renderer configuration from file + *--------------------------------------------------------------------*/ + + IVAS_RENDER_CONFIG_DATA renderConfigSplit; + if ( ( error = IVAS_DEC_GetDefaultRenderConfig( &renderConfigSplit ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_GetDefaultRenderConfig failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + if ( arg.renderConfigEnabled ) + { + /* sanity check */ +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( arg.outputConfig != AUDIO_CONFIG_BINAURAL && arg.outputConfig != AUDIO_CONFIG_BINAURAL_ROOM_IR && arg.outputConfig != AUDIO_CONFIG_BINAURAL_ROOM_REVERB && + arg.outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_CODED && + arg.outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + fprintf( stderr, "\nExternal Renderer Config is supported only when binaural output configurations is used as output OR when Split rendering mode is enabled. Exiting. \n" ); + exit( -1 ); + } +#else + if ( arg.outputConfig != AUDIO_CONFIG_BINAURAL && arg.outputConfig != AUDIO_CONFIG_BINAURAL_ROOM_IR && arg.outputConfig != AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) + { + fprintf( stderr, "\nExternal Renderer Config is supported only for binaural output configurations. Exiting. \n\n" ); + goto cleanup; + } +#endif + if ( RenderConfigReader_read( renderConfigReader, + arg.renderConfigFilename, + &renderConfigSplit ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Failed to read renderer configuration from file %s\n\n", arg.renderConfigFilename ); + goto cleanup; + } + } +#endif + #ifdef SPLIT_REND_WITH_HEAD_ROT /*------------------------------------------------------------------------------------------* * Binaural split rendering sanity checks @@ -447,32 +497,74 @@ int main( } #endif - /*------------------------------------------------------------------------------------------* - * Configure the decoder - *------------------------------------------------------------------------------------------*/ +/*------------------------------------------------------------------------------------------* + * Configure the decoder + *------------------------------------------------------------------------------------------*/ +#ifndef API_5MS +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + arg.enableHeadRotation = true; +#ifdef API_5MS_BASELINE + arg.enable5ms = renderConfig.split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE || renderConfig.split_rend_config.dof == 0; +#endif + } +#endif +#endif +#ifdef API_5MS + if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputConfig, arg.tsmEnabled, +#ifdef API_5MS_BASELINE + arg.enable5ms, +#endif + arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.enableExternalOrientation, arg.orientation_tracking, arg.renderConfigEnabled, arg.Opt_non_diegetic_pan, arg.non_diegetic_pan_gain, arg.delayCompensationEnabled ) ) != IVAS_ERR_OK ) +#else if ( ( error = IVAS_DEC_Configure( hIvasDec, arg.output_Fs, arg.outputConfig, arg.customLsOutputEnabled, arg.hrtfReaderEnabled, arg.enableHeadRotation, arg.enableExternalOrientation, arg.orientation_tracking, arg.renderConfigEnabled, arg.Opt_non_diegetic_pan, arg.non_diegetic_pan_gain, arg.delayCompensationEnabled ) ) != IVAS_ERR_OK ) +#endif { fprintf( stderr, "\nConfigure failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } + + /*------------------------------------------------------------------------------------------* + * Configure VoIP mode + *------------------------------------------------------------------------------------------*/ + +#ifdef API_5MS +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + if ( ( error = IVAS_DEC_EnableSplitRendering( hIvasDec ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nConfigure failed: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } +#endif +#endif + /*------------------------------------------------------------------------------------------* * Configure VoIP mode *------------------------------------------------------------------------------------------*/ if ( arg.voipMode ) { -#ifdef VARIABLE_SPEED_DECODING - if ( ( error = IVAS_DEC_EnableVoIP( hIvasDec, IVAS_DEC_VOIP_MODE_VOIP, 100, 60, arg.inputFormat ) ) != IVAS_ERR_OK ) +#ifdef API_5MS + if ( ( error = IVAS_DEC_EnableVoIP( hIvasDec, 60, arg.inputFormat ) ) != IVAS_ERR_OK ) #else +#ifndef VARIABLE_SPEED_DECODING if ( ( error = IVAS_DEC_EnableVoIP( hIvasDec, 60, arg.inputFormat ) ) != IVAS_ERR_OK ) +#else + if ( ( error = IVAS_DEC_EnableVoIP( hIvasDec, IVAS_DEC_VOIP_MODE_VOIP, 100, 60, arg.inputFormat ) ) != IVAS_ERR_OK ) +#endif #endif { fprintf( stderr, "\nCould not enable VOIP: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } } +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING else if ( arg.variableSpeedMode ) { @@ -483,6 +575,7 @@ int main( } } #endif +#endif #ifdef DEBUGGING /*-----------------------------------------------------------------* @@ -516,7 +609,11 @@ int main( IVAS_DEC_PrintConfigWithBitstream( hIvasDec, arg.quietModeEnabled, bit_stream, num_bits ); #ifdef VARIABLE_SPEED_DECODING +#ifdef API_5MS + if ( arg.tsmEnabled ) +#else if ( arg.variableSpeedMode ) +#endif { if ( arg.tsmScaleFileEnabled ) { @@ -747,11 +844,13 @@ int main( { error = decodeVoIP( arg, hBsReader, hIvasDec ); } +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING else if ( arg.variableSpeedMode ) { error = decodeVariableSpeed( arg, hBsReader, headRotReader, externalOrientationFileReader, refRotReader, referenceVectorReader, hIvasDec ); } +#endif #endif else { @@ -1004,10 +1103,17 @@ static bool parseCmdlIVAS_dec( arg->inputFormat = IVAS_DEC_INPUT_FORMAT_G192; arg->Opt_non_diegetic_pan = 0; arg->non_diegetic_pan_gain = 0.f; - +#ifdef API_5MS + arg->tsmEnabled = false; +#ifdef API_5MS_BASELINE + arg->enable5ms = false; +#endif +#endif #ifdef DEBUGGING #ifdef VARIABLE_SPEED_DECODING +#ifndef API_5MS arg->variableSpeedMode = false; +#endif arg->tsmScale = 100; arg->tsmScaleFileEnabled = false; arg->tsmScaleFileName = NULL; @@ -1161,7 +1267,14 @@ static bool parseCmdlIVAS_dec( { i++; int32_t tmp = 100; +#ifdef API_5MS + arg->tsmEnabled = true; +#ifdef API_5MS_BASELINE + arg->enable5ms = true; +#endif +#else arg->variableSpeedMode = true; +#endif if ( i < argc - 3 ) { if ( !is_digits_only( argv[i] ) ) @@ -1224,6 +1337,9 @@ static bool parseCmdlIVAS_dec( else if ( strcmp( argv_to_upper, "-T" ) == 0 ) { arg->enableHeadRotation = true; +#ifdef API_5MS_BASELINE + arg->enable5ms = true; +#endif i++; if ( argc - i <= 4 || argv[i][0] == '-' ) @@ -1304,6 +1420,9 @@ static bool parseCmdlIVAS_dec( else if ( strcmp( argv_to_upper, "-EXOF" ) == 0 ) { arg->enableExternalOrientation = true; +#ifdef API_5MS_BASELINE + arg->enable5ms = true; +#endif i++; if ( argc - i <= 4 || argv[i][0] == '-' ) @@ -1628,10 +1747,15 @@ static int16_t app_own_random( int16_t *seed ) *---------------------------------------------------------------------*/ static ivas_error initOnFirstGoodFrame( - IVAS_DEC_HANDLE hIvasDec, /* i/o: */ - const DecArguments arg, /* i : */ - const int16_t numInitialBadFrames, /* i : */ - const uint16_t numOutSamples, /* i : */ + IVAS_DEC_HANDLE hIvasDec, /* i/o: */ + const DecArguments arg, /* i : */ + const int16_t numInitialBadFrames, /* i : */ +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + uint16_t *numOutSamples, /* i/o: */ + int16_t *vec_pos_len, /* i/o: */ +#else + const uint16_t numOutSamples, /* i : */ +#endif int16_t *pFullDelayNumSamples, /* o : */ int16_t *pRemainingDelayNumSamples, /* o : */ int32_t *delayTimeScale, /* o : */ @@ -1736,10 +1860,22 @@ static ivas_error initOnFirstGoodFrame( if ( *hSplitRendFileReadWrite != NULL ) { IVAS_SPLIT_REND_BITS_DATA splitRendBitsZero; - splitRendBitsZero.bits_written = 0; + splitRendBitsZero.bits_buf = NULL; splitRendBitsZero.bits_read = 0; - - if ( split_rend_write_bitstream_to_file( *hSplitRendFileReadWrite, splitRendBitsZero.bits_buf, &splitRendBitsZero.bits_read, &splitRendBitsZero.bits_written, -1, IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) != IVAS_ERR_OK ) + splitRendBitsZero.bits_written = 0; + splitRendBitsZero.buf_len = 0; + splitRendBitsZero.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; + splitRendBitsZero.pose_correction = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE; +#ifdef API_5MS + splitRendBitsZero.codec_frame_size_ms = 20; +#endif + if ( split_rend_write_bitstream_to_file( *hSplitRendFileReadWrite, splitRendBitsZero.bits_buf, &splitRendBitsZero.bits_read, &splitRendBitsZero.bits_written, + -1, IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE +#ifdef API_5MS + , + splitRendBitsZero.codec_frame_size_ms +#endif + ) != IVAS_ERR_OK ) { fprintf( stderr, "\nUnable to write to bitstream file!\n" ); exit( -1 ); @@ -1748,9 +1884,22 @@ static ivas_error initOnFirstGoodFrame( else { #endif - if ( *pRemainingDelayNumSamples < numOutSamples ) + if ( *pRemainingDelayNumSamples < +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + *numOutSamples +#else + numOutSamples +#endif + ) { - if ( ( error = AudioFileWriter_write( *ppAfWriter, zeroBuf, numOutSamples * *pNumOutChannels - ( *pRemainingDelayNumSamples * *pNumOutChannels ) ) ) != IVAS_ERR_OK ) + if ( ( error = AudioFileWriter_write( *ppAfWriter, zeroBuf, +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + *numOutSamples +#else + numOutSamples +#endif + * *pNumOutChannels - + ( *pRemainingDelayNumSamples * *pNumOutChannels ) ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nOutput audio file writer error\n" ); return error; @@ -1759,7 +1908,13 @@ static ivas_error initOnFirstGoodFrame( } else { - *pRemainingDelayNumSamples -= numOutSamples; + *pRemainingDelayNumSamples -= +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + *numOutSamples +#else + numOutSamples +#endif + ; } #ifdef SPLIT_REND_WITH_HEAD_ROT } @@ -1849,6 +2004,33 @@ static ivas_error initOnFirstGoodFrame( } } +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + if ( *hSplitRendFileReadWrite != NULL ) + { + + int16_t enable5ms; + + if ( numOutSamples == NULL || vec_pos_len == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + IVAS_DEC_Get5msFlag( hIvasDec, &enable5ms ); + + /* real setting of the 5ms mode for split rendering is only known after the decoded first good frame, reset the variables needed in the main decoding loop accordingly here*/ + if ( enable5ms ) + { + *numOutSamples = (int16_t) ( arg.output_Fs / 1000 * HEADROTATION_FETCH_FRAMESIZE_MS ); + *vec_pos_len = IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; + } + else + { + *numOutSamples = (int16_t) ( arg.output_Fs / 1000 * DEFAULT_FETCH_FRAMESIZE_MS ); + *vec_pos_len = 1; + } + } +#endif + return IVAS_ERR_OK; } @@ -1857,7 +2039,7 @@ static ivas_error initOnFirstGoodFrame( * * Read G.192 bitstream and decode in regular decoder *---------------------------------------------------------------------*/ - +#ifdef API_5MS static ivas_error decodeG192( DecArguments arg, BS_READER_HANDLE hBsReader, @@ -1885,22 +2067,66 @@ static ivas_error decodeG192( int16_t numInitialBadFrames = 0; /* Number of bad frames received until first good frame is decoded */ int16_t nOutChannels = 0; int16_t delayNumSamples = -1; - int16_t delayNumSamples_orig[3]; /* stores: overall delay, dec+rend delay, and binauralization delay */ + int16_t delayNumSamples_orig[3]; +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + uint16_t nOutSamples = 0; +#else int16_t nOutSamples = 0; +#endif int32_t delayTimeScale = 0; ivas_error error = IVAS_ERR_UNKNOWN; uint16_t numObj = 0; IVAS_DEC_BS_FORMAT bsFormat = IVAS_DEC_BS_UNKOWN; - IVAS_VECTOR3 Pos[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; -#ifdef SPLIT_REND_WITH_HEAD_ROT - SplitFileReadWrite *splitRendWriter = NULL; + uint16_t nSamplesAvailableNext; + bool needNewFrame; + int16_t nSamplesRendered, nSamplesRendered_loop, nSamplesToRender; +#ifdef DEBUGGING +#ifdef VARIABLE_SPEED_DECODING + TsmScaleFileReader *tsmScaleFileReader = NULL; + int16_t scale; +#endif #endif IsmFileWriter *ismWriters[IVAS_MAX_NUM_OBJECTS]; + IVAS_VECTOR3 Pos = { 0, 0, 0 }; + int16_t vec_pos_update, vec_pos_len; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + IVAS_SPLIT_REND_BITS_DATA splitRendBits; + SplitFileReadWrite *hSplitRendFileReadWrite; +#endif + for ( i = 0; i < IVAS_MAX_NUM_OBJECTS; ++i ) { ismWriters[i] = NULL; } + /* we always start with needing a new frame */ + needNewFrame = true; + +#ifdef DEBUGGING +#ifdef VARIABLE_SPEED_DECODING + /*------------------------------------------------------------------------------------------* + * Open TSM scale file or set global TSM scale + *------------------------------------------------------------------------------------------*/ + + if ( arg.tsmEnabled ) + { + if ( arg.tsmScaleFileEnabled ) + { + if ( ( tsmScaleFileReader = TsmScaleFileReader_open( arg.tsmScaleFileName ) ) == NULL ) + { + fprintf( stderr, "\nError: Can't open TSM scale file %s \n\n", arg.tsmScaleFileName ); + goto cleanup; + } + } + else + { + IVAS_DEC_VoIP_SetScale( hIvasDec, arg.tsmScale, arg.tsmScale ); + } + } +#endif +#endif + if ( !arg.quietModeEnabled ) { fprintf( stdout, "\n------ Running the decoder ------\n\n" ); @@ -1917,6 +2143,34 @@ static ivas_error decodeG192( reset_stack(); reset_wmops(); #endif + nSamplesAvailableNext = 0; + + vec_pos_update = 0; + if ( arg.enableHeadRotation +#ifdef API_5MS_BASELINE + && arg.enable5ms +#endif + ) + { + nOutSamples = (int16_t) ( arg.output_Fs / 1000 * HEADROTATION_FETCH_FRAMESIZE_MS ); + vec_pos_len = IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; + } + else + { + nOutSamples = (int16_t) ( arg.output_Fs / 1000 * DEFAULT_FETCH_FRAMESIZE_MS ); + vec_pos_len = 1; + } + +#ifdef SPLIT_REND_WITH_HEAD_ROT + splitRendBits.bits_buf = splitRendBitsBuf; + splitRendBits.bits_read = 0; + splitRendBits.bits_written = 0; + splitRendBits.buf_len = IVAS_MAX_SPLIT_REND_BITS_BUFFER_SIZE_IN_BYTES; + splitRendBits.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; + splitRendBits.pose_correction = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE; + splitRendBits.codec_frame_size_ms = 0; + hSplitRendFileReadWrite = NULL; +#endif /*------------------------------------------------------------------------------------------* * Loop for every packet (frame) of bitstream data @@ -1927,42 +2181,10 @@ static ivas_error decodeG192( while ( 1 ) { - /* Read next frame */ - if ( ( error = BS_Reader_ReadFrame_short( hBsReader, bit_stream, &num_bits, &bfi ) ) != IVAS_ERR_OK ) - { - if ( error == IVAS_ERR_END_OF_FILE ) - { - break; - } - fprintf( stderr, "\nError: input bitstream file couldn't be read: %s \n\n", arg.inputBitstreamFilename ); - goto cleanup; - } - -#ifdef DEBUGGING - /* Random FEC simulation */ - if ( arg.FER > 0.0f ) - { - float ftmp = (float) app_own_random( &fec_seed ) + 32768.0f; - if ( ftmp <= arg.FER / 100.0f * 65535.0f ) - { - bfi = 1; - } - else - { - bfi = 0; - } - } -#endif - - /* Feed into decoder */ - if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, bit_stream, num_bits, bfi ) ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nError: could not feed frame to decoder: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); - goto cleanup; - } + /* Read next frame if not enough samples availble */ /* reference vector */ - if ( arg.enableReferenceVectorTracking ) + if ( arg.enableReferenceVectorTracking && vec_pos_update == 0 ) { IVAS_VECTOR3 listenerPosition, referencePosition; if ( ( error = Vector3PairFileReader_read( referenceVectorReader, &listenerPosition, &referencePosition ) ) != IVAS_ERR_OK ) @@ -1979,13 +2201,12 @@ static ivas_error decodeG192( } /* Reference rotation */ - if ( arg.enableReferenceRotation ) + if ( arg.enableReferenceRotation && vec_pos_update == 0 ) { IVAS_QUATERNION quaternion; if ( ( error = HeadRotationFileReading( refRotReader, &quaternion, NULL ) ) != IVAS_ERR_OK ) { - fprintf( stderr, "\nError %s while reading reference rotation from %s\n", IVAS_DEC_GetErrorMessage( error ), - RotationFileReader_getFilePath( refRotReader ) ); + fprintf( stderr, "\nError %s while reading reference rotation from %s\n", IVAS_DEC_GetErrorMessage( error ), RotationFileReader_getFilePath( refRotReader ) ); goto cleanup; } @@ -1995,42 +2216,40 @@ static ivas_error decodeG192( goto cleanup; } } + /* Head-tracking input simulation */ if ( arg.enableHeadRotation ) { - IVAS_QUATERNION Quaternions[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + IVAS_QUATERNION Quaternion; #ifdef SPLIT_REND_WITH_HEAD_ROT if ( headRotReader == NULL ) { - for ( i = 0; i < IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) - { - Quaternions[i].w = -3.0f; - Quaternions[i].x = 0.0f; - Quaternions[i].y = 0.0f; - Quaternions[i].z = 0.0f; - Pos[i].x = 0.0f; - Pos[i].y = 0.0f; - Pos[i].z = 0.0f; - } + Quaternion.w = -3.0f; + Quaternion.x = 0.0f; + Quaternion.y = 0.0f; + Quaternion.z = 0.0f; + +#ifdef API_5MS + Pos.x = 0.0f; + Pos.y = 0.0f; + Pos.z = 0.0f; +#endif } else { #endif - for ( i = 0; i < IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) + if ( ( error = HeadRotationFileReading( headRotReader, &Quaternion, &Pos ) ) != IVAS_ERR_OK ) { - if ( ( error = HeadRotationFileReading( headRotReader, &Quaternions[i], &Pos[i] ) ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nError %s while reading head orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), - RotationFileReader_getFilePath( headRotReader ) ); - goto cleanup; - } + fprintf( stderr, "\nError %s while reading head orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), RotationFileReader_getFilePath( headRotReader ) ); + goto cleanup; } #ifdef SPLIT_REND_WITH_HEAD_ROT } #endif - if ( ( error = IVAS_DEC_FeedHeadTrackData( hIvasDec, Quaternions, Pos + + if ( ( error = IVAS_DEC_FeedHeadTrackData( hIvasDec, Quaternion, Pos #ifdef SPLIT_REND_WITH_HEAD_ROT , DEFAULT_AXIS @@ -2040,45 +2259,824 @@ static ivas_error decodeG192( fprintf( stderr, "\nIVAS_DEC_FeedHeadTrackData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } +#if 0 // defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + int16_t enable5ms; + IVAS_DEC_Get5msFlag( hIvasDec, &enable5ms ); + arg.enable5ms = enable5ms; +#endif + if ( !arg.enable5ms ) + { + /* Skip over 3 following head positions - they are given on 5ms grid */ + for ( i = 0; i < 3; ++i ) + { + if ( ( error = HeadRotationFileReading( headRotReader, &Quaternion, &Pos ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in Head Rotation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } } if ( arg.enableExternalOrientation ) { - IVAS_QUATERNION Quaternions[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; - int8_t enableHeadRotation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; - int8_t enableExternalOrientation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; - int8_t enableRotationInterpolation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; - int16_t numFramesToTargetOrientation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + IVAS_QUATERNION Quaternion; + int8_t enableHeadRotation; + int8_t enableExternalOrientation; + int8_t enableRotationInterpolation; + int16_t numFramesToTargetOrientation; - for ( i = 0; i < IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) - { - if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &Quaternions[i], &enableHeadRotation[i], &enableExternalOrientation[i], &enableRotationInterpolation[i], &numFramesToTargetOrientation[i] ) ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nError %s while reading external orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), - RotationFileReader_getFilePath( externalOrientationFileReader ) ); - goto cleanup; - } + if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &Quaternion, &enableHeadRotation, &enableExternalOrientation, &enableRotationInterpolation, &numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading external orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), + RotationFileReader_getFilePath( externalOrientationFileReader ) ); + goto cleanup; } - if ( ( error = IVAS_DEC_FeedExternalOrientationData( hIvasDec, Quaternions, enableHeadRotation, enableExternalOrientation, enableRotationInterpolation, numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + + if ( ( error = IVAS_DEC_FeedExternalOrientationData( hIvasDec, Quaternion, enableHeadRotation, enableExternalOrientation, enableRotationInterpolation, numFramesToTargetOrientation * vec_pos_len ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nIVAS_DEC_FeedExternalOrientationData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; } - } - /* Run decoder for one frame (get rendered output) */ - if ( ( error = IVAS_DEC_GetSamples( hIvasDec, pcmBuf, &nOutSamples -#ifdef SPLIT_REND_WITH_HEAD_ROT - , - splitRendBitsBuf +#if 0 // defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + int16_t enable5ms; + IVAS_DEC_Get5msFlag( hIvasDec, &enable5ms ); + arg.enable5ms = enable5ms; #endif - ) ) != IVAS_ERR_OK ) - { - fprintf( stderr, "\nError: could not get samples from decoder: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); - goto cleanup; - } + if ( !arg.enable5ms ) + { + /* Skip over 3 following entries in file - they are given on 5ms grid */ + for ( i = 0; i < 3; ++i ) + { + if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &Quaternion, &enableHeadRotation, &enableExternalOrientation, &enableRotationInterpolation, &numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in External Orientation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } + } + + /* decode and get samples */ + nSamplesRendered = 0; + nSamplesToRender = nOutSamples; + do + { + if ( needNewFrame ) + { +#ifdef DEBUGGING +#ifdef VARIABLE_SPEED_DECODING + if ( arg.tsmScaleFileEnabled ) + { + if ( ( error = TsmScaleFileReader_readScale( tsmScaleFileReader, &scale ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: input bitstream file couldn't be read: %s \n\n", arg.inputBitstreamFilename ); + goto cleanup; + } + IVAS_DEC_VoIP_SetScale( hIvasDec, scale, scale ); + } +#endif +#endif + + if ( ( error = BS_Reader_ReadFrame_short( hBsReader, bit_stream, &num_bits, &bfi ) ) != IVAS_ERR_OK ) + { + if ( error == IVAS_ERR_END_OF_FILE ) + { + break; + } + fprintf( stderr, "\nError: input bitstream file couldn't be read: %s \n\n", arg.inputBitstreamFilename ); + goto cleanup; + } + +#ifdef DEBUGGING + /* Random FEC simulation */ + if ( arg.FER > 0.0f ) + { + float ftmp = (float) app_own_random( &fec_seed ) + 32768.0f; + if ( ftmp <= arg.FER / 100.0f * 65535.0f ) + { + bfi = 1; + } + else + { + bfi = 0; + } + } +#endif + + /* Feed into decoder */ + if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, bit_stream, num_bits, bfi ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: could not feed frame to decoder: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + if ( arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + error = IVAS_DEC_GetSplitBinauralBitstream( hIvasDec, + (void *) ( pcmBuf + nOutChannels * nSamplesRendered ), + &splitRendBits, + nSamplesToRender, + &nSamplesRendered_loop, + &needNewFrame ); + nSamplesRendered += nSamplesRendered_loop; + nSamplesToRender -= nSamplesRendered_loop; + if ( error != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetSplitBinauralBitstream: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + else + { +#endif + error = IVAS_DEC_GetSamples( hIvasDec, nSamplesToRender, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_INT16, + (void *) +#endif + ( pcmBuf + nOutChannels * nSamplesRendered ), + &nSamplesRendered_loop, + &needNewFrame ); + nSamplesRendered += nSamplesRendered_loop; + nSamplesToRender -= nSamplesRendered_loop; + if ( error != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetSamples: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif + + + } while ( nSamplesRendered < nOutSamples && error == IVAS_ERR_OK ); + + if ( error == IVAS_ERR_END_OF_FILE ) + { + break; + } + + /* Continue checking for first good frame until it is found */ + if ( !decodedGoodFrame ) + { + if ( ( error = IVAS_DEC_HasDecodedFirstGoodFrame( hIvasDec, &decodedGoodFrame ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_HasDecodedFirstGoodFrame, code: %d\n", error ); + goto cleanup; + } + + /* Once good frame decoded, catch up */ + if ( decodedGoodFrame ) + { + error = initOnFirstGoodFrame( + hIvasDec, + arg, + numInitialBadFrames, +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + &nOutSamples, + &vec_pos_len, +#else + nOutSamples, +#endif + delayNumSamples_orig, + &delayNumSamples, + &delayTimeScale, + &bsFormat, + &afWriter, + &masaWriter, + ismWriters, + &nOutChannels, + &numObj +#ifdef SPLIT_REND_WITH_HEAD_ROT + , + &hSplitRendFileReadWrite +#endif + ); + if ( error != IVAS_ERR_OK ) + { + goto cleanup; + } +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + int16_t enable5ms; + IVAS_DEC_Get5msFlag( hIvasDec, &enable5ms ); + /* we assumed 5ms but are really 20ms, we need to throw away some head and ext positions */ + if ( !enable5ms && arg.enable5ms ) + { + /* Skip over 3 following entries in file - they are given on 5ms grid */ + if ( arg.enableExternalOrientation ) + { + IVAS_QUATERNION Quaternion; + int8_t enableHeadRotation; + int8_t enableExternalOrientation; + int8_t enableRotationInterpolation; + int16_t numFramesToTargetOrientation; + for ( i = 0; i < 3; ++i ) + { + if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &Quaternion, &enableHeadRotation, &enableExternalOrientation, &enableRotationInterpolation, &numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in External Orientation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } + if ( arg.enableHeadRotation ) + { + IVAS_QUATERNION Quaternion; + /* Skip over 3 following head positions - they are given on 5ms grid */ + for ( i = 0; i < 3; ++i ) + { + if ( ( error = HeadRotationFileReading( headRotReader, &Quaternion, &Pos ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in Head Rotation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } + } + arg.enable5ms = enable5ms; +#endif + } + else + { + ++numInitialBadFrames; + } + } + + /* Write current frame */ + if ( decodedGoodFrame ) + { +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( hSplitRendFileReadWrite != NULL ) && ( arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) ) + { + if ( splitRendBits.bits_written > 0 ) + { + if ( split_rend_write_bitstream_to_file( hSplitRendFileReadWrite, splitRendBits.bits_buf, &splitRendBits.bits_read, &splitRendBits.bits_written, + splitRendBits.codec, splitRendBits.pose_correction +#ifdef API_5MS + , + splitRendBits.codec_frame_size_ms +#endif + ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nUnable to write to bitstream file!\n" ); + goto cleanup; + } + } + } + else + { + if ( ( hSplitRendFileReadWrite != NULL ) && ( arg.outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) + { + if ( split_rend_write_bitstream_to_file( hSplitRendFileReadWrite, splitRendBits.bits_buf, &splitRendBits.bits_read, &splitRendBits.bits_written, + splitRendBits.codec, splitRendBits.pose_correction +#ifdef API_5MS + , + splitRendBits.codec_frame_size_ms +#endif + ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nUnable to write to bitstream file!\n" ); + goto cleanup; + } + } +#endif + if ( delayNumSamples < nOutSamples ) + { + if ( ( error = AudioFileWriter_write( afWriter, &pcmBuf[delayNumSamples * nOutChannels], nOutSamples * nOutChannels - ( delayNumSamples * nOutChannels ) ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nOutput audio file writer error\n" ); + goto cleanup; + } + delayNumSamples = 0; + } + else + { + delayNumSamples -= nOutSamples; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif + } + + /* Write ISm metadata to external file(s) */ + if ( decodedGoodFrame && arg.outputConfig == AUDIO_CONFIG_EXTERNAL ) + { + if ( bsFormat == IVAS_DEC_BS_OBJ ) + { + if ( ( error = IVAS_DEC_GetNumObjects( hIvasDec, &numObj ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetNumObjects: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + for ( i = 0; i < numObj; ++i ) + { + IVAS_ISM_METADATA IsmMetadata; + + if ( ( error = IVAS_DEC_GetObjectMetadata( hIvasDec, &IsmMetadata, 0, i ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetObjectMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + if ( ( IsmFileWriter_writeFrame( IsmMetadata, ismWriters[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError writing ISM metadata to file %s\n", IsmFileWriter_getFilePath( ismWriters[i] ) ); + goto cleanup; + } + } + } + else if ( bsFormat == IVAS_DEC_BS_MASA ) + { + MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta; + if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); + goto cleanup; + } + } + } + + frame++; + vec_pos_update = ( vec_pos_update + 1 ) % vec_pos_len; + if ( !arg.quietModeEnabled ) + { + fprintf( stdout, "%-8d\b\b\b\b\b\b\b\b", frame ); +#ifdef DEBUGGING + if ( IVAS_DEC_GetBerDetectFlag( hIvasDec ) ) + { + fprintf( stdout, "\n Decoding error: BER detected in frame %d !!!!!\n", frame - 1 ); + } +#endif + } +#ifdef WMOPS + if ( vec_pos_update == 0 ) + { + update_wmops(); + } +#ifdef MEM_COUNT_DETAILS + export_mem( "mem_analysis.csv" ); +#endif +#endif + } + + + /*------------------------------------------------------------------------------------------* + * Flush what is still left in the VoIP Buffers.... + *------------------------------------------------------------------------------------------*/ + + while ( nSamplesAvailableNext > 0 ) + { + int16_t nSamplesFlushed; + + /* Feed into decoder */ + + /* reference vector */ + if ( arg.enableReferenceVectorTracking ) + { + IVAS_VECTOR3 listenerPosition, referencePosition; + if ( ( error = Vector3PairFileReader_read( referenceVectorReader, &listenerPosition, &referencePosition ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading listener and reference positions from %s\n", IVAS_DEC_GetErrorMessage( error ), Vector3PairFileReader_getFilePath( referenceVectorReader ) ); + goto cleanup; + } + + if ( ( error = IVAS_DEC_FeedRefVectorData( hIvasDec, listenerPosition, referencePosition ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedRefVectorData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + /* Reference rotation */ + if ( arg.enableReferenceRotation ) + { + IVAS_QUATERNION quaternion; + if ( ( error = HeadRotationFileReading( refRotReader, &quaternion, NULL ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading reference rotation from %s\n", IVAS_DEC_GetErrorMessage( error ), RotationFileReader_getFilePath( refRotReader ) ); + goto cleanup; + } + + if ( ( error = IVAS_DEC_FeedRefRotData( hIvasDec, quaternion ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedRefRotData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + /* Head-tracking input simulation */ + if ( arg.enableHeadRotation ) + { + IVAS_QUATERNION Quaternion; + + + if ( ( error = HeadRotationFileReading( headRotReader, &Quaternion, &Pos ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading head orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), RotationFileReader_getFilePath( headRotReader ) ); + goto cleanup; + } + + + if ( ( error = IVAS_DEC_FeedHeadTrackData( hIvasDec, Quaternion, Pos +#ifdef SPLIT_REND_WITH_HEAD_ROT + , + DEFAULT_AXIS +#endif + ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedHeadTrackData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + + /* decode and get samples */ + if ( ( error = IVAS_DEC_Flush( hIvasDec, nOutSamples, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_INT16, + (void *) +#endif + pcmBuf, + &nSamplesFlushed ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_VoIP_Flush: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + /* Write current frame */ + if ( ( error = AudioFileWriter_write( afWriter, pcmBuf, nSamplesFlushed * nOutChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nOutput audio file writer error\n" ); + goto cleanup; + } + + /* Write ISm metadata to external file(s) */ + if ( decodedGoodFrame && arg.outputConfig == AUDIO_CONFIG_EXTERNAL ) + { + if ( bsFormat == IVAS_DEC_BS_OBJ ) + { + if ( ( error = IVAS_DEC_GetNumObjects( hIvasDec, &numObj ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetNumObjects: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + for ( i = 0; i < numObj; ++i ) + { + IVAS_ISM_METADATA IsmMetadata; + + if ( ( error = IVAS_DEC_GetObjectMetadata( hIvasDec, &IsmMetadata, 0, i ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetObjectMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + if ( ( IsmFileWriter_writeFrame( IsmMetadata, ismWriters[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError writing ISM metadata to file %s\n", IsmFileWriter_getFilePath( ismWriters[i] ) ); + goto cleanup; + } + } + } + else if ( bsFormat == IVAS_DEC_BS_MASA ) + { + MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta; + if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + if ( ( error = MasaFileWriter_writeFrame( masaWriter, hMasaExtOutMeta ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError writing MASA metadata to file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); + goto cleanup; + } + } + } + + frame++; + if ( !arg.quietModeEnabled ) + { + fprintf( stdout, "%-8d\b\b\b\b\b\b\b\b", frame ); +#ifdef DEBUGGING + if ( IVAS_DEC_GetBerDetectFlag( hIvasDec ) ) + { + fprintf( stdout, "\n Decoding error: BER detected in frame %d !!!!!\n", frame - 1 ); + } +#endif + } + } + + /*------------------------------------------------------------------------------------------* + * Printouts after decoding has finished + *------------------------------------------------------------------------------------------*/ + + if ( !arg.quietModeEnabled ) + { + printf( "\n\nDecoder+renderer delay: %4.2f ms (%3u samples at timescale %5u)\n", 1000.f * delayNumSamples_orig[1] / (float) delayTimeScale, delayNumSamples_orig[1], delayTimeScale ); + + if ( delayNumSamples_orig[2] > 0 ) + { + printf( "HRIR/BRIR delay: %4.2f ms (%3u samples at timescale %5u)\n", 1000.f * delayNumSamples_orig[2] / (float) delayTimeScale, delayNumSamples_orig[2], delayTimeScale ); + printf( "Total delay: %4.2f ms (%3u samples at timescale %5u)\n", 1000.f * ( delayNumSamples_orig[1] + delayNumSamples_orig[2] ) / (float) delayTimeScale, delayNumSamples_orig[1] + delayNumSamples_orig[2], delayTimeScale ); + } + } + + /* Print output metadata file name(s) */ + if ( arg.outputConfig == AUDIO_CONFIG_EXTERNAL ) + { + if ( bsFormat == IVAS_DEC_BS_OBJ ) + { + for ( i = 0; i < numObj; i++ ) + { + fprintf( stdout, "\nOutput metadata file: %s", IsmFileWriter_getFilePath( ismWriters[i] ) ); + } + fprintf( stdout, "\n" ); + } + else if ( bsFormat == IVAS_DEC_BS_MASA ) + { + fprintf( stdout, "\nOutput metadata file: %s\n", MasaFileWriter_getFilePath( masaWriter ) ); + } + } + + /* add zeros at the end to have equal length of synthesized signals */ + memset( pcmBuf, 0, delayNumSamples_orig[0] * nOutChannels * sizeof( int16_t ) ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( afWriter != NULL ) + { +#endif + if ( ( error = AudioFileWriter_write( afWriter, pcmBuf, delayNumSamples_orig[0] * nOutChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError writing output file: %s\n", ivas_error_to_string( error ) ); + goto cleanup; + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif + + /*------------------------------------------------------------------------------------------* + * Close files and deallocate resources + *------------------------------------------------------------------------------------------*/ + + decodingFailed = false; /* This will stay set to true if cleanup is reached via a goto due to an error */ + +cleanup: + +#ifdef SPLIT_REND_WITH_HEAD_ROT + split_rend_reader_writer_close( &hSplitRendFileReadWrite ); +#endif + AudioFileWriter_close( &afWriter ); + MasaFileWriter_close( &masaWriter ); +#ifdef DEBUGGING +#ifdef VARIABLE_SPEED_DECODING + TsmScaleFileReader_close( &tsmScaleFileReader ); +#endif +#endif + for ( i = 0; i < IVAS_MAX_NUM_OBJECTS; i++ ) + { + IsmFileWriter_close( &ismWriters[i] ); + } + + if ( decodingFailed && error == IVAS_ERR_OK ) + { + return IVAS_ERR_UNKNOWN; + } + + return error; +} +#else +static ivas_error decodeG192( + DecArguments arg, + BS_READER_HANDLE hBsReader, + RotFileReader *headRotReader, + RotFileReader *externalOrientationFileReader, + RotFileReader *refRotReader, + Vector3PairFileReader *referenceVectorReader, +#ifdef SPLIT_REND_WITH_HEAD_ROT + uint8_t *splitRendBitsBuf, +#endif + IVAS_DEC_HANDLE hIvasDec, + int16_t *pcmBuf ) + +{ + bool decodingFailed = true; /* Assume failure until cleanup is reached without errors */ + uint16_t bit_stream[IVAS_MAX_BITS_PER_FRAME + 4 * 8]; + int16_t i, num_bits; + int16_t bfi = 0; +#ifdef DEBUGGING + int16_t fec_seed = 12558; /* FEC_SEED */ +#endif + AudioFileWriter *afWriter = NULL; + MasaFileWriter *masaWriter = NULL; + bool decodedGoodFrame = false; + int16_t numInitialBadFrames = 0; /* Number of bad frames received until first good frame is decoded */ + int16_t nOutChannels = 0; + int16_t delayNumSamples = -1; + int16_t delayNumSamples_orig[3]; /* stores: overall delay, dec+rend delay, and binauralization delay */ + int16_t nOutSamples = 0; + int32_t delayTimeScale = 0; + ivas_error error = IVAS_ERR_UNKNOWN; + uint16_t numObj = 0; + IVAS_DEC_BS_FORMAT bsFormat = IVAS_DEC_BS_UNKOWN; + IVAS_VECTOR3 Pos[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; +#ifdef SPLIT_REND_WITH_HEAD_ROT + SplitFileReadWrite *splitRendWriter = NULL; +#endif + IsmFileWriter *ismWriters[IVAS_MAX_NUM_OBJECTS]; + for ( i = 0; i < IVAS_MAX_NUM_OBJECTS; ++i ) + { + ismWriters[i] = NULL; + } + + if ( !arg.quietModeEnabled ) + { + fprintf( stdout, "\n------ Running the decoder ------\n\n" ); + fprintf( stdout, "Frames processed: " ); + } + else + { + fprintf( stdout, "\n-- Start the decoder (quiet mode) --\n\n" ); + } + + delayNumSamples_orig[0] = -1; + +#ifdef WMOPS + reset_stack(); + reset_wmops(); +#endif + + /*------------------------------------------------------------------------------------------* + * Loop for every packet (frame) of bitstream data + * - Read the bitstream packet + * - Run the decoder + * - Write the synthesized signal into output file + *------------------------------------------------------------------------------------------*/ + + while ( 1 ) + { + /* Read next frame */ + if ( ( error = BS_Reader_ReadFrame_short( hBsReader, bit_stream, &num_bits, &bfi ) ) != IVAS_ERR_OK ) + { + if ( error == IVAS_ERR_END_OF_FILE ) + { + break; + } + fprintf( stderr, "\nError: input bitstream file couldn't be read: %s \n\n", arg.inputBitstreamFilename ); + goto cleanup; + } + +#ifdef DEBUGGING + /* Random FEC simulation */ + if ( arg.FER > 0.0f ) + { + float ftmp = (float) app_own_random( &fec_seed ) + 32768.0f; + if ( ftmp <= arg.FER / 100.0f * 65535.0f ) + { + bfi = 1; + } + else + { + bfi = 0; + } + } +#endif + + /* Feed into decoder */ + if ( ( error = IVAS_DEC_FeedFrame_Serial( hIvasDec, bit_stream, num_bits, bfi ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: could not feed frame to decoder: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + + /* reference vector */ + if ( arg.enableReferenceVectorTracking ) + { + IVAS_VECTOR3 listenerPosition, referencePosition; + if ( ( error = Vector3PairFileReader_read( referenceVectorReader, &listenerPosition, &referencePosition ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading listener and reference positions from %s\n", IVAS_DEC_GetErrorMessage( error ), Vector3PairFileReader_getFilePath( referenceVectorReader ) ); + goto cleanup; + } + + if ( ( error = IVAS_DEC_FeedRefVectorData( hIvasDec, listenerPosition, referencePosition ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedRefVectorData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + + /* Reference rotation */ + if ( arg.enableReferenceRotation ) + { + IVAS_QUATERNION quaternion; + if ( ( error = HeadRotationFileReading( refRotReader, &quaternion, NULL ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading reference rotation from %s\n", IVAS_DEC_GetErrorMessage( error ), + RotationFileReader_getFilePath( refRotReader ) ); + goto cleanup; + } + + if ( ( error = IVAS_DEC_FeedRefRotData( hIvasDec, quaternion ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedRefRotData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + /* Head-tracking input simulation */ + if ( arg.enableHeadRotation ) + { + IVAS_QUATERNION Quaternions[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + +#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( headRotReader == NULL ) + { + for ( i = 0; i < IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) + { + Quaternions[i].w = -3.0f; + Quaternions[i].x = 0.0f; + Quaternions[i].y = 0.0f; + Quaternions[i].z = 0.0f; + Pos[i].x = 0.0f; + Pos[i].y = 0.0f; + Pos[i].z = 0.0f; + } + } + else + { +#endif + for ( i = 0; i < IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) + { + if ( ( error = HeadRotationFileReading( headRotReader, &Quaternions[i], &Pos[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading head orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), + RotationFileReader_getFilePath( headRotReader ) ); + goto cleanup; + } + } +#ifdef SPLIT_REND_WITH_HEAD_ROT + } +#endif + + if ( ( error = IVAS_DEC_FeedHeadTrackData( hIvasDec, Quaternions, Pos +#ifdef SPLIT_REND_WITH_HEAD_ROT + , + DEFAULT_AXIS +#endif + ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedHeadTrackData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + + if ( arg.enableExternalOrientation ) + { + IVAS_QUATERNION Quaternions[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + int8_t enableHeadRotation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + int8_t enableExternalOrientation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + int8_t enableRotationInterpolation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + int16_t numFramesToTargetOrientation[IVAS_MAX_PARAM_SPATIAL_SUBFRAMES]; + + for ( i = 0; i < IVAS_MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) + { + + if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &Quaternions[i], &enableHeadRotation[i], &enableExternalOrientation[i], &enableRotationInterpolation[i], &numFramesToTargetOrientation[i] ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError %s while reading external orientation from %s\n", IVAS_DEC_GetErrorMessage( error ), + RotationFileReader_getFilePath( externalOrientationFileReader ) ); + goto cleanup; + } + } + + if ( ( error = IVAS_DEC_FeedExternalOrientationData( hIvasDec, Quaternions, enableHeadRotation, enableExternalOrientation, enableRotationInterpolation, numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nIVAS_DEC_FeedExternalOrientationData failed: %s\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } + } + + /* Run decoder for one frame (get rendered output) */ + if ( ( error = IVAS_DEC_GetSamples( hIvasDec, pcmBuf, &nOutSamples +#ifdef SPLIT_REND_WITH_HEAD_ROT + , + splitRendBitsBuf +#endif + ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError: could not get samples from decoder: %s\n\n", IVAS_DEC_GetErrorMessage( error ) ); + goto cleanup; + } /* Continue checking for first good frame until it is found */ if ( !decodedGoodFrame ) @@ -2307,6 +3305,7 @@ cleanup: return error; } +#endif #ifdef DEBUGGING /*---------------------------------------------------------------------* @@ -2577,8 +3576,14 @@ static ivas_error decodeVoIP( while ( 1 ) { +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + uint16_t nOutSamples = 0; +#else int16_t nOutSamples = 0; +#endif +#ifndef API_5MS uint16_t nSamplesAvailableNext = 0; +#endif #ifdef DEBUG_JBM_CMD_OPTION nOutSamples = (int16_t) ( arg.output_Fs / 1000 * arg.frontendFetchSizeMs ); #else @@ -2635,7 +3640,17 @@ static ivas_error decodeVoIP( /* decode and get samples */ - if ( ( error = IVAS_DEC_VoIP_GetSamples( hIvasDec, nOutSamples, pcmBuf, systemTime_ms, &nSamplesAvailableNext + if ( ( error = IVAS_DEC_VoIP_GetSamples( hIvasDec, nOutSamples, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_INT16, + (void *) +#endif + pcmBuf, + systemTime_ms +#ifndef API_5MS + , + &nSamplesAvailableNext +#endif #ifdef SUPPORT_JBM_TRACEFILE , writeJbmTraceFileFrameWrapper, @@ -2686,7 +3701,12 @@ static ivas_error decodeVoIP( hIvasDec, arg, numInitialBadFrames, +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + &nOutSamples, + NULL, +#else nOutSamples, +#endif delayNumSamples_orig, &delayNumSamples, &delayTimeScale, @@ -2853,6 +3873,7 @@ cleanup: #ifdef DEBUGGING +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING /*---------------------------------------------------------------------* * decodeVariableSpeed() @@ -3059,7 +4080,11 @@ static ivas_error decodeVariableSpeed( fprintf( stderr, "\nError: input bitstream file couldn't be read: %s \n\n", arg.inputBitstreamFilename ); goto cleanup; } +#ifdef API_5MS + IVAS_DEC_VoIP_SetScale( hIvasDec, scale, scale ); +#else IVAS_DEC_VoIP_SetScale( hIvasDec, scale ); +#endif } if ( ( error = BS_Reader_ReadFrame_short( hBsReader, bit_stream, &num_bits, &bfi ) ) != IVAS_ERR_OK ) @@ -3329,7 +4354,7 @@ static ivas_error decodeVariableSpeed( } /* decode and get samples */ - if ( ( error = IVAS_DEC_VoIP_Flush( hIvasDec, nOutSamples, pcmBuf, &nSamplesAvailableNext, &nSamplesFlushed ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_DEC_Flush( hIvasDec, nOutSamples, pcmBuf, &nSamplesAvailableNext, &nSamplesFlushed ) ) != IVAS_ERR_OK ) { fprintf( stderr, "\nError in IVAS_DEC_VoIP_Flush: %s\n", IVAS_DEC_GetErrorMessage( error ) ); goto cleanup; @@ -3473,7 +4498,7 @@ cleanup: return error; } #endif - +#endif /*---------------------------------------------------------------------* * parseForcedRendModeDec() diff --git a/apps/renderer.c b/apps/renderer.c index 0ab2de02166b6e7c4b34b7d160b5bc9ca4ba40a4..767f94661f5b8804396c3d5865c835b30c72ce89 100644 --- a/apps/renderer.c +++ b/apps/renderer.c @@ -777,7 +777,13 @@ int main( int32_t delayTimeScale = 0; int16_t i, numChannels; ivas_error error = IVAS_ERR_OK; +#ifndef API_5MS IVAS_VECTOR3 Pos[RENDERER_HEAD_POSITIONS_PER_FRAME]; +#endif +#if defined( API_5MS ) && defined( SPLIT_REND_WITH_HEAD_ROT ) + bool splitBinNeedsNewAudioFrame = true; + bool splitBinNeedsNewBitstreamFrame = true; +#endif #ifdef WMOPS reset_wmops(); @@ -996,7 +1002,13 @@ int main( #ifdef SPLIT_REND_WITH_HEAD_ROT } #endif - const int16_t frameSize_smpls = (int16_t) ( 20 * args.sampleRate / 1000 ); + const int16_t frameSize_smpls = (int16_t) ( +#ifdef API_5MS + ( args.framing_5ms ? 5 : 20 ) +#else + 20 +#endif + * args.sampleRate / 1000 ); IVAS_REND_InputId mcIds[RENDERER_MAX_MC_INPUTS] = { 0 }; IVAS_REND_InputId ismIds[RENDERER_MAX_ISM_INPUTS] = { 0 }; @@ -1408,6 +1420,11 @@ int main( bitsBuffer.config.bitsRead = 0; bitsBuffer.config.bitsWritten = 0; bitsBuffer.config.bufLenInBytes = bitsBufferSize; + bitsBuffer.config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; + bitsBuffer.config.poseCorrection = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE; +#ifdef API_5MS + bitsBuffer.config.codec_frame_size_ms = 20; +#endif #else inFloatBuffer = malloc( inBufferSize * sizeof( float ) ); outInt16Buffer = malloc( outBufferSize * sizeof( int16_t ) ); @@ -1437,19 +1454,50 @@ int main( fprintf( stdout, "\n\n-- Start the renderer (quiet mode) --\n\n" ); } +#ifdef API_5MS + ObjectPositionBuffer mtdBuffer; +#endif + while ( 1 ) { int16_t num_in_channels; num_in_channels = inBuffer.config.numChannels; +#ifdef API_5MS + const bool isCurrentFrameMultipleOf20ms = !args.framing_5ms || frame % 4 == 0; #ifdef SPLIT_REND_WITH_HEAD_ROT - numSamplesRead = 0; if ( ( hSplitRendFileReadWrite != NULL ) && is_split_post_rend_mode( &args ) ) + { + if ( args.inConfig.binBuses[0].audioConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + /* this is always 20 ms*/ + splitBinNeedsNewBitstreamFrame = isCurrentFrameMultipleOf20ms; + } + else + { + splitBinNeedsNewBitstreamFrame = splitBinNeedsNewAudioFrame; + } + } +#endif +#endif + +#ifdef SPLIT_REND_WITH_HEAD_ROT + numSamplesRead = 0; + if ( ( hSplitRendFileReadWrite != NULL ) && is_split_post_rend_mode( &args ) +#ifdef API_5MS + && splitBinNeedsNewBitstreamFrame +#endif + ) { ivas_error error_tmp; numSamplesRead = (int16_t) inBufferSize; error_tmp = split_rend_read_bits_from_file( hSplitRendFileReadWrite, bitsBuffer.bits, &bitsBuffer.config.bitsRead, &bitsBuffer.config.bitsWritten, - &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection ); + &bitsBuffer.config.codec, &bitsBuffer.config.poseCorrection +#ifdef API_5MS + , + &bitsBuffer.config.codec_frame_size_ms +#endif + ); if ( error_tmp != IVAS_ERR_OK ) { if ( error_tmp == IVAS_ERR_END_OF_FILE ) @@ -1477,7 +1525,11 @@ int main( } #endif - if ( numSamplesRead == 0 ) + if ( numSamplesRead == 0 +#if defined( API_5MS ) && defined( SPLIT_REND_WITH_HEAD_ROT ) + && splitBinNeedsNewAudioFrame +#endif + ) { /* end of input data */ break; @@ -1491,43 +1543,84 @@ int main( #endif ); +#ifdef API_5MS + if ( isCurrentFrameMultipleOf20ms ) + { +#else ObjectPositionBuffer mtdBuffer; - IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer ); +#endif + IsmPositionProvider_getNextFrame( positionProvider, &mtdBuffer ); - if ( referenceVectorReader != NULL ) - { - IVAS_VECTOR3 listenerPos, refPos; - if ( ( error = Vector3PairFileReader_read( referenceVectorReader, &listenerPos, &refPos ) ) != IVAS_ERR_OK ) + if ( referenceVectorReader != NULL ) { - fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); - exit( -1 ); + IVAS_VECTOR3 listenerPos, refPos; + if ( ( error = Vector3PairFileReader_read( referenceVectorReader, &listenerPos, &refPos ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + if ( ( error = IVAS_REND_SetReferenceVector( hIvasRend, listenerPos, refPos ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } } - if ( ( error = IVAS_REND_SetReferenceVector( hIvasRend, listenerPos, refPos ) ) != IVAS_ERR_OK ) + /* Read from reference rotation trajectory file if specified */ + if ( referenceRotReader != NULL ) { - fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); - exit( -1 ); + IVAS_QUATERNION quaternion; + if ( ( error = HeadRotationFileReading( referenceRotReader, &quaternion, NULL ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in Head Rotation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( ( error = IVAS_REND_SetReferenceRotation( hIvasRend, quaternion ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error setting Reference Rotation: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } } +#ifdef API_5MS } - /* Read from reference rotation trajectory file if specified */ - if ( referenceRotReader != NULL ) +#endif + + /* Read from head rotation trajectory file if specified */ + if ( headRotReader != NULL ) { - IVAS_QUATERNION quaternion; - if ( ( error = HeadRotationFileReading( referenceRotReader, &quaternion, NULL ) ) != IVAS_ERR_OK ) +#ifdef API_5MS + IVAS_QUATERNION headRot; + IVAS_VECTOR3 Pos; + + if ( ( error = HeadRotationFileReading( headRotReader, &headRot, &Pos ) ) != IVAS_ERR_OK ) { fprintf( stderr, "Error in Head Rotation File Reading: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } - - if ( ( error = IVAS_REND_SetReferenceRotation( hIvasRend, quaternion ) ) != IVAS_ERR_OK ) + if ( ( error = IVAS_REND_SetHeadRotation( hIvasRend, headRot, Pos +#ifdef SPLIT_REND_WITH_HEAD_ROT + , + DEFAULT_AXIS +#endif + ) ) != IVAS_ERR_OK ) { - fprintf( stderr, "Error setting Reference Rotation: %s\n", ivas_error_to_string( error ) ); + fprintf( stderr, "Error setting Head Rotation: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } - } - /* Read from head rotation trajectory file if specified */ - if ( headRotReader != NULL ) - { + if ( !args.framing_5ms ) + { + /* Skip over 3 following head positions - they are given on 5ms grid */ + for ( i = 0; i < 3; ++i ) + { + if ( ( error = HeadRotationFileReading( headRotReader, &headRot, &Pos ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in Head Rotation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } +#else IVAS_QUATERNION quatBuffer[RENDERER_HEAD_POSITIONS_PER_FRAME]; for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) @@ -1549,9 +1642,17 @@ int main( fprintf( stderr, "Error setting Head Rotation: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } +#endif } else { +#ifdef API_5MS + if ( ( error = IVAS_REND_DisableHeadRotation( hIvasRend ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error disabling head rotation: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } +#else /* API_5MS */ error = IVAS_REND_SetHeadRotation( hIvasRend, NULL, NULL #ifdef SPLIT_REND_WITH_HEAD_ROT , @@ -1563,11 +1664,16 @@ int main( fprintf( stderr, "Error setting Head Rotation: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } +#endif /* API_5MS */ } #ifdef SPLIT_REND_WITH_HEAD_ROT /* Read from split renderer bfi file if specified */ - if ( splitRendBFIReader != NULL ) + if ( splitRendBFIReader != NULL +#ifdef API_5MS + && splitBinNeedsNewBitstreamFrame +#endif + ) { int16_t bfi; SplitRendBFIFileReading( splitRendBFIReader, &bfi ); @@ -1578,6 +1684,38 @@ int main( /* Read from external orientation file if specified */ if ( externalOrientationFileReader != NULL ) { +#ifdef API_5MS + IVAS_QUATERNION quat; + int8_t enableHeadRotation; + int8_t enableExternalOrientation; + int8_t enableRotationInterpolation; + int16_t numFramesToTargetOrientation; + + if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &quat, &enableHeadRotation, &enableExternalOrientation, &enableRotationInterpolation, &numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in External Orientation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( ( error = IVAS_REND_SetExternalOrientation( hIvasRend, &quat, enableHeadRotation, enableExternalOrientation, enableRotationInterpolation, numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error setting External Orientation: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + + if ( !args.framing_5ms ) + { + /* Skip over 3 following entries in file - they are given on 5ms grid */ + for ( i = 0; i < 3; ++i ) + { + if ( ( error = ExternalOrientationFileReading( externalOrientationFileReader, &quat, &enableHeadRotation, &enableExternalOrientation, &enableRotationInterpolation, &numFramesToTargetOrientation ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error in External Orientation File Reading: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } + } +#else IVAS_QUATERNION quatBuffer[RENDERER_HEAD_POSITIONS_PER_FRAME]; int8_t enableHeadRotation[RENDERER_HEAD_POSITIONS_PER_FRAME]; int8_t enableExternalOrientation[RENDERER_HEAD_POSITIONS_PER_FRAME]; @@ -1598,6 +1736,7 @@ int main( fprintf( stderr, "Error setting External Orientation: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } +#endif } /* Combine external orientations and head rotation */ @@ -1711,35 +1850,77 @@ int main( #ifdef SPLIT_REND_WITH_HEAD_ROT for ( i = 0; i < args.inConfig.numBinBuses; ++i ) { - if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, splitBinIds[i], &numChannels ) ) != IVAS_ERR_OK ) +#ifdef API_5MS + if ( splitBinNeedsNewAudioFrame ) + { +#endif + if ( ( error = IVAS_REND_GetInputNumChannels( hIvasRend, splitBinIds[i], &numChannels ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.binBuses[i].inputChannelIndex, numChannels ); + + if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, splitBinIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + +#ifdef API_5MS + } + if ( splitBinNeedsNewBitstreamFrame ) + { + if ( ( error = IVAS_REND_FeedSplitBinauralBitstream( hIvasRend, splitBinIds[i], &bitsBuffer ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); + exit( -1 ); + } + } +#endif + } +#endif + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + if ( args.inConfig.numBinBuses != 0 ) + { + if ( ( error = IVAS_REND_GetSplitBinauralSamples( hIvasRend, outBuffer, &splitBinNeedsNewAudioFrame ) ) != IVAS_ERR_OK ) { fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } - IVAS_REND_ReadOnlyAudioBuffer tmpBuffer = getReadOnlySubBuffer( inBuffer, (int16_t) args.inConfig.binBuses[i].inputChannelIndex, numChannels ); - - if ( ( error = IVAS_REND_FeedInputAudio( hIvasRend, splitBinIds[i], tmpBuffer ) ) != IVAS_ERR_OK ) + } + else if ( args.outConfig.audioConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || + args.outConfig.audioConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + if ( ( error = IVAS_REND_GetSplitBinauralBitstream( hIvasRend, outBuffer, &bitsBuffer ) ) != IVAS_ERR_OK ) { fprintf( stderr, "Error: %s\n", ivas_error_to_string( error ) ); exit( -1 ); } } + else + { #endif - - if ( ( error = IVAS_REND_GetSamples( hIvasRend, outBuffer + if ( ( error = IVAS_REND_GetSamples( hIvasRend, outBuffer +#ifndef API_5MS #ifdef SPLIT_REND_WITH_HEAD_ROT - , - &bitsBuffer + , + &bitsBuffer #endif - ) ) != IVAS_ERR_OK ) - { +#endif + ) ) != IVAS_ERR_OK ) + { #ifdef SPLIT_REND_WITH_HEAD_ROT - fprintf( stderr, "Error %s\n", ivas_error_to_string( error ) ); + fprintf( stderr, "Error %s\n", ivas_error_to_string( error ) ); #else fprintf( stderr, "Error in getting samples\n" ); #endif - exit( -1 ); + exit( -1 ); + } +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT } +#endif int16_t num_out_channels; num_out_channels = outBuffer.config.numChannels; @@ -1788,7 +1969,12 @@ int main( if ( ( hSplitRendFileReadWrite != NULL ) && is_split_pre_rend_mode( &args ) ) { if ( split_rend_write_bitstream_to_file( hSplitRendFileReadWrite, bitsBuffer.bits, &bitsBuffer.config.bitsRead, &bitsBuffer.config.bitsWritten, - bitsBuffer.config.codec, bitsBuffer.config.poseCorrection ) != IVAS_ERR_OK ) + bitsBuffer.config.codec, bitsBuffer.config.poseCorrection +#ifdef API_5MS + , + bitsBuffer.config.codec_frame_size_ms +#endif + ) != IVAS_ERR_OK ) { fprintf( stderr, "\nUnable to write to bitstream file!\n" ); exit( -1 ); @@ -2686,7 +2872,9 @@ static void parseOption( case CmdLnOptionId_framing5ms: assert( numOptionValues == 0 ); args->framing_5ms = true; +#ifndef API_5MS fprintf( stderr, "Warning: this is a placeholder for 5ms framing.\n" ); +#endif break; case CmdLnOptionId_directivityPatternId: assert( numOptionValues <= RENDERER_MAX_ISM_INPUTS ); @@ -2695,7 +2883,6 @@ static void parseOption( args->directivityPatternId[i] = (int16_t) strtol( optionValues[i], NULL, 10 ); } break; - case CmdLnOptionId_acousticEnvironmentId: assert( numOptionValues == 1 ); args->acousticEnvironmentId = (int16_t) strtol( optionValues[0], NULL, 10 ); diff --git a/lib_com/common_api_types.h b/lib_com/common_api_types.h index da51606209ff631d7adf3b46e8f9b1139b816e4b..5f28e53ffe0a9cdba19f413c3d54284f3a5ffd0c 100644 --- a/lib_com/common_api_types.h +++ b/lib_com/common_api_types.h @@ -55,8 +55,13 @@ #define IVAS_CLDFB_NO_CHANNELS_MAX 60 #define IVAS_MAX_INPUT_LFE_CHANNELS 4 -#define IVAS_MAX_PARAM_SPATIAL_SUBFRAMES 4 +#ifdef API_5MS +#define RENDERER_SUBFRAMES_PER_FRAME 4 +#endif +#define IVAS_MAX_PARAM_SPATIAL_SUBFRAMES 4 +#ifndef API_5MS #define RENDERER_HEAD_POSITIONS_PER_FRAME 4 // ToDo: should it be harmonized with IVAS_MAX_PARAM_SPATIAL_SUBFRAMES? +#endif #define QC_ABS_COEFF 6 #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -178,6 +183,9 @@ typedef struct ivas_split_rend_bits_t int32_t buf_len; /*size of bits_buf in bytes. This field should be set by allocator of bits_buf*/ int32_t bits_written; int32_t bits_read; +#ifdef API_5MS + int16_t codec_frame_size_ms; +#endif IVAS_SPLIT_REND_CODEC codec; IVAS_SPLIT_REND_POSE_CORRECTION_MODE pose_correction; } ivas_split_rend_bits_t, IVAS_SPLIT_REND_BITS_DATA, *IVAS_SPLIT_REND_BITS_HANDLE; @@ -260,6 +268,9 @@ typedef struct _IVAS_SPLIT_REND_CONFIG 3 - (3dof correction. By default YAW, PITCH and ROLL correction) */ int16_t codec_delay_ms; /*PLACEHOLDER (currently being ignored) : look ahead delay of the codec that is used to code BIN signal output of pre-renderer*/ +#ifdef API_5MS + int16_t codec_frame_size_ms; /*Codec frame size in milliseconds, only relevant with LC3plus */ +#endif IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode; IVAS_SPLIT_REND_CODEC codec; IVAS_SPLIT_REND_RENDERER_SELECTION rendererSelection; diff --git a/lib_com/ivas_cnst.h b/lib_com/ivas_cnst.h index ff944358454284adc6f1ad8636656ae9597b846b..49194aa54cd739d6ddf84971c6aa331ea2a9485d 100755 --- a/lib_com/ivas_cnst.h +++ b/lib_com/ivas_cnst.h @@ -121,6 +121,15 @@ typedef enum RENDERER_OSBA_LS } RENDERER_TYPE; + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +typedef enum +{ + PCM_INT16, + PCM_FLOAT32, + PCM_NOT_KNOW = 0xffff +} PCM_RESOLUTION; +#endif #ifdef DEBUGGING typedef enum { diff --git a/lib_com/ivas_error.h b/lib_com/ivas_error.h index e9c8692bde148e6b411be04e079631c8b176133c..813b1d570fd631313186dee425b7f6b71fbd0d18 100644 --- a/lib_com/ivas_error.h +++ b/lib_com/ivas_error.h @@ -86,6 +86,10 @@ typedef enum IVAS_ERR_ISM_INVALID_METADATA_VALUE, IVAS_ERR_INVALID_MASA_FORMAT_METADATA_FILE, IVAS_ERR_IO_CONFIG_PAIR_NOT_SUPPORTED, +#ifdef API_5MS + IVAS_ERR_TSM_NOT_ENABLED, + IVAS_ERR_FETCH_SIZE_NO_MULTIPLE_OF_5MS, +#endif #ifdef DEBUGGING IVAS_ERR_INVALID_FORCE_MODE, #ifdef DEBUG_AGC_ENCODER_CMD_OPTION diff --git a/lib_com/ivas_prot.h b/lib_com/ivas_prot.h index a0d220df2abf0714b766d34a8c3d1a4e38b05493..26f69a451580822f15815b84cb2c2baf843bb1df 100755 --- a/lib_com/ivas_prot.h +++ b/lib_com/ivas_prot.h @@ -299,19 +299,31 @@ void stereo_dmx_evs_close_encoder( STEREO_DMX_EVS_ENC_HANDLE *hStereoDmxEVS /* i/o: Stereo downmix for EVS encoder handle */ ); +#if !defined(API_5MS) || defined (API_5MS_BASELINE) ivas_error ivas_dec( Decoder_Struct *st_ivas, /* i : IVAS decoder structure */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else int16_t *data /* o : output synthesis signal */ -#ifdef SPLIT_REND_WITH_HEAD_ROT +#endif +#if defined SPLIT_REND_WITH_HEAD_ROT && !defined(API_5MS) , uint8_t *splitRendBitsBuf /* o : output split rendering bits */ #endif ); +#endif ivas_error ivas_dec_setup( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ - int16_t *data /* o : flushed PCM samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ); ivas_error create_sce_dec( @@ -651,7 +663,12 @@ ivas_error ivas_mc_dec_config( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const int16_t idx, /* i : LS config. index */ uint16_t *nSamplesRendered, /* o : samples flushed from last frame (JBM) */ - int16_t *data /* o : flushed samples (JBM) */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ); /*! r: MC format mode (MCT, McMASA, ParamMC) */ @@ -766,7 +783,12 @@ ivas_error ivas_jbm_dec_render( const uint16_t nSamplesAsked, /* i : number of samples wanted */ uint16_t *nSamplesRendered, /* o : number of samples rendered */ uint16_t *nSamplesAvailableNext, /* o : number of samples still available in the rendering pipeline */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else int16_t *data /* o : output synthesis signal */ +#endif ); ivas_error ivas_jbm_dec_flush_renderer( @@ -778,7 +800,12 @@ ivas_error ivas_jbm_dec_flush_renderer( const MC_MODE mc_mode_old, /* i : old MC mode */ const ISM_MODE ism_mode_old, /* i : old ISM mode */ uint16_t *nSamplesRendered, /* o : number of samples flushed */ - int16_t *data /* o : rendered samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ); ivas_error ivas_jbm_dec_feed_tc_to_renderer( @@ -817,6 +844,13 @@ int16_t ivas_jbm_dec_get_num_tc_channels( Decoder_Struct *st_ivas /* i : IVAS decoder handle */ ); +#ifdef API_5MS +void ivas_jbm_dec_copy_tc_no_tsm( + Decoder_Struct *st_ivas, + float *tc[], + const int16_t output_frame ); +#endif + void ivas_jbm_dec_get_md_map_even_spacing( const int16_t len, /* i : length of the modfied frames in metadata slots */ const int16_t subframe_len, /* i : default length of a subframe */ @@ -1037,7 +1071,12 @@ ivas_error ivas_ism_dec_config( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const ISM_MODE last_ism_mode, /* i/o: last ISM mode */ uint16_t *nSamplesRendered, /* o : number of samples flushed on renderer change*/ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : flushed PCM samples */ +#else int16_t *data /* o : flushed PCM samples */ +#endif ); ivas_error ivas_param_ism_dec_open( @@ -1065,6 +1104,15 @@ void ivas_param_ism_dec_digest_tc( float *transport_channels_f[] /* i : synthesized core-coder transport channels/DirAC output */ ); +#ifdef API_5MS +void ivas_ism_param_dec_tc_gain_ajust( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamples, /* i : number of samples to be compensate */ + const uint16_t nFadeLength, /* i : length of the crossfade in samples */ + float *transport_channels_f[] /* i : synthesized core-coder transport channels/DirAC output */ +); +#endif + void ivas_param_ism_dec_render( Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ @@ -3632,6 +3680,9 @@ void ivas_dirac_dec( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ const int16_t nchan_transport /* i : number of transport channels */ +#ifdef API_5MS + ,const int16_t num_subframes /* i : number of subframes to render */ +#endif ); void ivas_dirac_dec_render( @@ -4944,7 +4995,12 @@ void ivas_masa_enc_reconfigure( ivas_error ivas_masa_dec_reconfigure( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : flushed PCM samples */ +#else int16_t *data /* o : flushed PCM samples */ +#endif ); ivas_error ivas_masa_encode( @@ -5202,7 +5258,9 @@ void ivas_binRenderer( HEAD_TRACK_DATA_HANDLE hPostRendHeadTrackData, #endif COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined head and external orientation handle */ +#ifndef API_5MS int16_t subframe_idx, /* i : subframe index */ +#endif const int16_t numTimeSlots, /* i : number of time slots to process */ #ifdef SPLIT_REND_WITH_HEAD_ROT float Cldfb_RealBuffer_Binaural[][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* o : Rotated Binaural signals */ @@ -5211,8 +5269,8 @@ void ivas_binRenderer( float Cldfb_RealBuffer_Binaural[][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* o : Binaural signals */ float Cldfb_ImagBuffer_Binaural[][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* o : Binaural signals */ #endif - float RealBuffer[][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i : LS signals */ - float ImagBuffer[][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX] /* i : LS signals */ + float RealBuffer[][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i : LS signals */ + float ImagBuffer[][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX] /* i : LS signals */ ); void ivas_binaural_add_LFE( @@ -5668,7 +5726,12 @@ ivas_error ivas_omasa_enc_config( ivas_error ivas_omasa_dec_config( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : flushed PCM samples */ +#else int16_t *data /* o : flushed PCM samples */ +#endif ); void ivas_omasa_set_config( diff --git a/lib_com/options.h b/lib_com/options.h index b556709497b466555e18aa2946b71a595c918a24..52e171299344023b7bab70bff732d102573ce919 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -129,6 +129,8 @@ /*#define DEBUG_AGC_ENCODER_CMD_OPTION*/ /* Ability to force enable or disable AGC behaviour in DIRAC/SPAR via command line option */ /*#define DEBUG_JBM_CMD_OPTION*/ /* ability for telling the decoder the frontend fetch size and to not delay compensate for bad frames at the beginning */ /*#define VARIABLE_SPEED_DECODING*/ /* variable speed decoding employing the JBM functioniality; move to DEBUGGING after build for disabled is fixed */ +#define DISABLE_LIMITER /* disable the limiter to aid BE debugging */ +#define DEBUG_SIMULATE_EXTORIENTATION_MAIN_FOR_BE /* simulate the problem discussed in*/ /*Split Rendering Debug switches*/ /*#define DBG_WAV_WRITER*/ /* add debugging function dbgwrite_wav() */ @@ -150,6 +152,22 @@ /*#define FIX_I4_OL_PITCH*/ /* fix open-loop pitch used for EVS core switching */ +/* ### API_5MS switches start ### */ +#define API_5MS /* FhG: 5ms rendering capability */ +#ifdef API_5MS +#define API_5MS_BASELINE /* FhG: baseline with 20ms rendering and split rendering through 20ms branch */ +#endif +/* BE Fixes */ +#define FIX_676_JBM_USAN /* FhG: fix usan in acessing transport channel buffers */ +#define FIX_RENDER_CONFIG_BITSTREAM_READER +#define FIX_REND_DELAY_COMP /* FhG: fix delay compensation in renderer.c - Doesn't affect BE as the broken code was never executed on main */ +#define LIB_REND_FIX_HRTFPARAMBIN_MEMLEAK +#define FIX_CQMFPREDDEC_FREE /* FhG: avoid double free / free of NULL in DeletePredictionDecoder() */ +#define FIX_PLANAR_SBA_JBM_RS +#define FIX_RENDMC_LOCAL_ORIENTATION +/* ### API_5MS switches ### */ + + /*#define SPLIT_REND_WITH_HEAD_ROT*/ /* Dlb,FhG: Split Rendering contributions 21 and 35 */ #ifdef SPLIT_REND_WITH_HEAD_ROT #define SPLIT_REND_PRED_QUANT_63_PNTS diff --git a/lib_dec/ivas_binRenderer_internal.c b/lib_dec/ivas_binRenderer_internal.c index d884f8f8e500a0537c14facec395a99efe658346..d7755460903b8d9a03a8b522f6bfc65aaf0e2f87 100644 --- a/lib_dec/ivas_binRenderer_internal.c +++ b/lib_dec/ivas_binRenderer_internal.c @@ -1649,17 +1649,21 @@ void ivas_binaural_cldfb( } /* Implement binaural rendering */ - ivas_binRenderer( st_ivas->hBinRenderer, + ivas_binRenderer( + st_ivas->hBinRenderer, #ifdef SPLIT_REND_WITH_HEAD_ROT - &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, + &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, #endif - st_ivas->hCombinedOrientationData, - subframeIdx, JBM_CLDFB_SLOTS_IN_SUBFRAME, + st_ivas->hCombinedOrientationData, +#ifndef API_5MS + subframeIdx, +#endif + JBM_CLDFB_SLOTS_IN_SUBFRAME, #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG - NULL, + NULL, #endif - Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, - Cldfb_RealBuffer, Cldfb_ImagBuffer ); + Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, + Cldfb_RealBuffer, Cldfb_ImagBuffer ); #ifdef SPLIT_REND_WITH_HEAD_ROT if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) @@ -1824,11 +1828,20 @@ void ivas_binaural_cldfb_sf( } /* Implement binaural rendering */ - ivas_binRenderer( st_ivas->hBinRenderer, + ivas_binRenderer( + st_ivas->hBinRenderer, #ifdef SPLIT_REND_WITH_HEAD_ROT - &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, + &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, +#endif + st_ivas->hCombinedOrientationData, +#ifndef API_5MS + subframeIdx, #endif - st_ivas->hCombinedOrientationData, subframeIdx, st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, Cldfb_RealBuffer, Cldfb_ImagBuffer ); + st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + Cldfb_RealBuffer, + Cldfb_ImagBuffer ); #ifdef SPLIT_REND_WITH_HEAD_ROT if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) @@ -1885,13 +1898,15 @@ void ivas_binaural_cldfb_sf( *-------------------------------------------------------------------------*/ void ivas_binRenderer( - BINAURAL_RENDERER_HANDLE hBinRenderer, /* i/o: binaural renderer handle */ + BINAURAL_RENDERER_HANDLE hBinRenderer, /* i/o: binaural renderer handle */ #ifdef SPLIT_REND_WITH_HEAD_ROT const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, #endif - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined head and external orientation handle */ - int16_t subframe_idx, /* i : subframe index */ - const int16_t numTimeSlots, + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined head and external orientation handle*/ +#ifndef API_5MS + int16_t subframe_idx, /* i : subframe index */ +#endif + const int16_t numTimeSlots, /* i : number of time slots to render */ #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG HEAD_TRACK_DATA_HANDLE hPostRendHeadTrackData, #endif @@ -1942,25 +1957,40 @@ void ivas_binRenderer( #endif /* Head rotation in HOA3 or CICPx */ - if ( - hCombinedOrientationData != NULL && hCombinedOrientationData->enableCombinedOrientation[subframe_idx] && hBinRenderer->rotInCldfb ) +#ifdef API_5MS + if ( hCombinedOrientationData != NULL && hCombinedOrientationData->enableCombinedOrientation && hBinRenderer->rotInCldfb ) +#else + if ( hCombinedOrientationData != NULL && hCombinedOrientationData->enableCombinedOrientation[subframe_idx] && hBinRenderer->rotInCldfb ) +#endif { if ( hBinRenderer->hInputSetup->is_loudspeaker_setup == 0 ) { /* Rotation in SHD (HOA3) */ if ( hCombinedOrientationData->shd_rot_max_order == -1 ) { +#ifdef API_5MS + rotateFrame_shd_cldfb( RealBuffer, ImagBuffer, hCombinedOrientationData->Rmat, hBinRenderer->hInputSetup->nchan_out_woLFE, numTimeSlots, 3 ); +#else rotateFrame_shd_cldfb( RealBuffer, ImagBuffer, hCombinedOrientationData->Rmat[subframe_idx], hBinRenderer->hInputSetup->nchan_out_woLFE, numTimeSlots, 3 ); +#endif } else if ( hCombinedOrientationData->shd_rot_max_order > 0 ) { +#ifdef API_5MS + rotateFrame_shd_cldfb( RealBuffer, ImagBuffer, hCombinedOrientationData->Rmat, hBinRenderer->hInputSetup->nchan_out_woLFE, numTimeSlots, hCombinedOrientationData->shd_rot_max_order ); +#else rotateFrame_shd_cldfb( RealBuffer, ImagBuffer, hCombinedOrientationData->Rmat[subframe_idx], hBinRenderer->hInputSetup->nchan_out_woLFE, numTimeSlots, hCombinedOrientationData->shd_rot_max_order ); +#endif } } else { /* Rotation in SD (CICPx) */ +#ifdef API_5MS + rotateFrame_sd_cldfb( hCombinedOrientationData->Rmat, RealBuffer, ImagBuffer, hBinRenderer->hInputSetup, hBinRenderer->hEFAPdata, numTimeSlots, hBinRenderer->conv_band ); +#else rotateFrame_sd_cldfb( hCombinedOrientationData->Rmat[subframe_idx], RealBuffer, ImagBuffer, hBinRenderer->hInputSetup, hBinRenderer->hEFAPdata, numTimeSlots, hBinRenderer->conv_band ); +#endif } } @@ -1987,7 +2017,11 @@ void ivas_binRenderer( if ( hCombinedOrientationData && hBinRenderer->rotInCldfb ) { +#ifdef API_5MS + Quaternions_ref = &hCombinedOrientationData->Quaternion; +#else Quaternions_ref = &hCombinedOrientationData->Quaternions[0]; +#endif Quaternions_rel.w = -3.0f; /*euler*/ Quaternions_abs.w = -3.0f; /*euler*/ Quat2EulerDegree( *Quaternions_ref, &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ @@ -2102,9 +2136,15 @@ void ivas_rend_CldfbMultiBinRendProcess( float Cldfb_In_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_Out_Real[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* o : Binaural signals */ float Cldfb_Out_Imag[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - const int16_t low_res_pre_rend_rot ) + const int16_t low_res_pre_rend_rot +#ifdef API_5MS + , + int16_t num_subframes +#endif +) { - int16_t sf_idx, slot_idx, ch_idx, idx, pose_idx, i, j; + int16_t slot_idx, ch_idx, idx, pose_idx, i, j; + int16_t sf_idx; float Cldfb_RealBuffer_sfIn[MAX_INPUT_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_sfIn[MAX_INPUT_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG @@ -2116,7 +2156,14 @@ void ivas_rend_CldfbMultiBinRendProcess( float Cldfb_ImagBuffer_Binaural[MAX_HEAD_ROT_POSES][BINAURAL_CHANNELS][MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; #endif - for ( sf_idx = 0; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) + for ( sf_idx = 0; sf_idx < +#ifdef API_5MS + num_subframes +#else + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ; + sf_idx++ ) { for ( slot_idx = 0; slot_idx < MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) { @@ -2132,12 +2179,18 @@ void ivas_rend_CldfbMultiBinRendProcess( { if ( ( low_res_pre_rend_rot ) && ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) ) { +#ifdef API_5MS + ( *pCombinedOrientationData )->Quaternion = ( *pCombinedOrientationData )->Quaternion; +#else ( *pCombinedOrientationData )->Quaternions[sf_idx] = ( *pCombinedOrientationData )->Quaternions[0]; +#endif for ( i = 0; i < 3; i++ ) { for ( j = 0; j < 3; j++ ) { - ( *pCombinedOrientationData )->Rmat[sf_idx][i][j] = ( *pCombinedOrientationData )->Rmat[0][i][j]; +#ifdef API_5MS + ( *pCombinedOrientationData )->Rmat[i][j] = ( *pCombinedOrientationData )->Rmat[i][j]; +#endif } } } @@ -2150,7 +2203,13 @@ void ivas_rend_CldfbMultiBinRendProcess( head_track_post.Quaternions[0] = ivas_split_rend_get_sf_rot_data( pHeadRotData->headPositionsPostRend, sf_idx ); #endif - ivas_binRenderer( hCldfbRend, pMultiBinPoseData, *pCombinedOrientationData, sf_idx, MAX_PARAM_SPATIAL_SUBFRAMES, + ivas_binRenderer( hCldfbRend, + pMultiBinPoseData, + *pCombinedOrientationData, +#ifndef API_5MS + sf_idx, +#endif + MAX_PARAM_SPATIAL_SUBFRAMES, #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG &head_track_post, #endif diff --git a/lib_dec/ivas_dec.c b/lib_dec/ivas_dec.c index ae8b1f158ea7428297123efd4550c8686380a9cf..db5d52251b246086e3190c7b5de99e8c2c24bd4e 100644 --- a/lib_dec/ivas_dec.c +++ b/lib_dec/ivas_dec.c @@ -32,6 +32,7 @@ #include #include "options.h" +#if !defined( API_5MS ) || defined( API_5MS_BASELINE ) #include "cnst.h" #include "ivas_cnst.h" #include "rom_com.h" @@ -53,8 +54,13 @@ ivas_error ivas_dec( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ - int16_t *data /* o : output synthesis signal */ -#ifdef SPLIT_REND_WITH_HEAD_ROT +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif +#if defined SPLIT_REND_WITH_HEAD_ROT && !defined( API_5MS ) , uint8_t *splitRendBitsBuf /* o : output split rendering bits */ #endif @@ -72,11 +78,15 @@ ivas_error ivas_dec( float *p_output[MAX_OUTPUT_CHANNELS + MAX_NUM_OBJECTS]; int16_t num_md_sub_frames; int32_t ism_total_brate; +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + int16_t nchan_out_syn_output; +#endif error = IVAS_ERR_OK; push_wmops( "ivas_dec" ); +#ifndef API_5MS_BASELINE /*----------------------------------------------------------------* * IVAS decoder setup * - read IVAS format signaling @@ -87,11 +97,16 @@ ivas_error ivas_dec( if ( st_ivas->bfi == 0 ) { - if ( ( error = ivas_dec_setup( st_ivas, NULL, NULL ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_dec_setup( st_ivas, NULL, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + PCM_NOT_KNOW, +#endif + NULL ) ) != IVAS_ERR_OK ) { return error; } } +#endif /*----------------------------------------------------------------* * Initialization of local vars after struct has been set @@ -120,7 +135,7 @@ ivas_error ivas_dec( return error; } -#ifdef SPLIT_REND_WITH_HEAD_ROT +#if defined SPLIT_REND_WITH_HEAD_ROT && !defined( API_5MS ) /*----------------------------------------------------------------* * Split rendering setup *----------------------------------------------------------------*/ @@ -140,7 +155,10 @@ ivas_error ivas_dec( ( st_ivas->ivas_format == MC_FORMAT && st_ivas->mc_mode != MC_MODE_MCMASA ) ) && ( output_Fs == 48000 ) && "split binaural mode is currently supported with SBA, discrete ISM, or MCT-MC formats and 48 kHz sampling rate only" ); #endif - if ( ( error = ivas_set_split_rend_setup( &st_ivas->hSplitBinRend, &st_ivas->hRenderConfig->split_rend_config, st_ivas->hCombinedOrientationData, splitRendBitsBuf ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_set_split_rend_setup( &st_ivas->hSplitBinRend, + &st_ivas->hRenderConfig->split_rend_config, + st_ivas->hCombinedOrientationData, + splitRendBitsBuf ) ) != IVAS_ERR_OK ) { return error; } @@ -267,7 +285,12 @@ ivas_error ivas_dec( ivas_param_ism_params_to_masa_param_mapping( st_ivas ); - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } else if ( st_ivas->renderer_type == RENDERER_MONO_DOWNMIX ) { @@ -346,6 +369,10 @@ ivas_error ivas_dec( { if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper, AUDIO_CONFIG_7_1_4, AUDIO_CONFIG_BINAURAL_ROOM_IR, NULL, NULL, NULL, NULL, p_output, output_Fs +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , 0 @@ -497,7 +524,12 @@ ivas_error ivas_dec( /* Loudspeakers, Ambisonics or Binaural rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, nchan_remapped ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, nchan_remapped +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } else if ( st_ivas->ivas_format == MASA_FORMAT ) { @@ -512,7 +544,12 @@ ivas_error ivas_dec( } else if ( st_ivas->renderer_type == RENDERER_DIRAC ) { - ivas_dirac_dec( st_ivas, output, nchan_remapped ); + ivas_dirac_dec( st_ivas, output, nchan_remapped +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } } else if ( !st_ivas->sba_dirac_stereo_flag && nchan_out != 1 ) @@ -599,7 +636,12 @@ ivas_error ivas_dec( } else { - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } } else if ( st_ivas->renderer_type == RENDERER_MONO_DOWNMIX ) @@ -739,7 +781,12 @@ ivas_error ivas_dec( /* Loudspeakers, Ambisonics or Binaural rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, &output[sba_ch_idx], nchan_remapped ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, &output[sba_ch_idx], nchan_remapped +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } else if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV ) { @@ -867,6 +914,10 @@ ivas_error ivas_dec( #endif if ( ( error = ivas_rend_crendProcess( st_ivas->hCrendWrapper, st_ivas->intern_config, st_ivas->hOutSetup.output_config, st_ivas->hDecoderConfig, st_ivas->hCombinedOrientationData, &st_ivas->hIntSetup, st_ivas->hEFAPdata, p_output, output_Fs +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , 0 @@ -1104,11 +1155,21 @@ ivas_error ivas_dec( /* Rendering */ if ( st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC || st_ivas->renderer_type == RENDERER_BINAURAL_PARAMETRIC_ROOM || st_ivas->renderer_type == RENDERER_STEREO_PARAMETRIC ) { - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } else if ( st_ivas->renderer_type == RENDERER_DIRAC || st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) /* rendering to CICPxx and Ambisonics */ { - ivas_dirac_dec( st_ivas, output, st_ivas->nchan_transport ); + ivas_dirac_dec( st_ivas, output, st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); if ( st_ivas->renderer_type == RENDERER_SBA_LINEAR_ENC ) { @@ -1130,7 +1191,7 @@ ivas_error ivas_dec( } -#ifdef SPLIT_REND_WITH_HEAD_ROT +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && !defined( API_5MS ) /*----------------------------------------------------------------* * Split rendering process calls *----------------------------------------------------------------*/ @@ -1173,13 +1234,59 @@ ivas_error ivas_dec( * - float to integer conversion *----------------------------------------------------------------*/ - ivas_limiter_dec( st_ivas->hLimiter, p_output, nchan_out, output_frame, st_ivas->BER_detect ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + if ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + nchan_out_syn_output = BINAURAL_CHANNELS * st_ivas->hSplitBinRend.splitrend.multiBinPoseData.num_poses; + } + else + { + nchan_out_syn_output = nchan_out; + } + + if ( st_ivas->hDecoderConfig->Opt_Limiter ) +#endif + { +#ifndef DISABLE_LIMITER + ivas_limiter_dec( st_ivas->hLimiter, p_output, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + nchan_out_syn_output, +#else + nchan_out, +#endif + output_frame, st_ivas->BER_detect ); +#endif + } +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + switch ( pcm_resolution ) + { + case PCM_INT16: +#endif #ifdef DEBUGGING - st_ivas->noClipping += + st_ivas->noClipping += +#endif + ivas_syn_output( p_output, output_frame, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + nchan_out_syn_output, +#else + nchan_out, +#endif +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + (int16_t *) +#endif + data ); + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + break; + case PCM_FLOAT32: + ivas_syn_output_f( p_output, output_frame, nchan_out_syn_output, (float *) data ); + break; + default: + error = IVAS_ERR_UNKNOWN; + break; + } #endif - ivas_syn_output( p_output, output_frame, nchan_out, data ); - /*----------------------------------------------------------------* * Common updates *----------------------------------------------------------------*/ @@ -1224,3 +1331,4 @@ ivas_error ivas_dec( pop_wmops(); return error; } +#endif diff --git a/lib_dec/ivas_dirac_dec.c b/lib_dec/ivas_dirac_dec.c index dbabd37154cfa5cfdfa1d23b4852b0d4e71872a1..c65c053a129c645d73bc0ebf90f85fa723649b94 100644 --- a/lib_dec/ivas_dirac_dec.c +++ b/lib_dec/ivas_dirac_dec.c @@ -974,7 +974,15 @@ ivas_error ivas_dirac_dec_config( /* Allocate transport channel buffers for SBA format when in JBM */ if ( dec_config_flag == DIRAC_OPEN ) { - if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms && +#endif +#else + st_ivas->hDecoderConfig->voip_active == 1 && +#endif + st_ivas->hTcBuffer == NULL ) { if ( st_ivas->ivas_format == SBA_FORMAT ) { @@ -1548,6 +1556,15 @@ void ivas_dirac_dec_set_md_map( hSpatParamRendCom->subframes_rendered = 0; ivas_jbm_dec_get_adapted_subframes( nCldfbTs, hSpatParamRendCom->subframe_nbslots, &hSpatParamRendCom->nb_subframes ); +#ifdef API_5MS + /* copy also to tc buffer */ + /* only for non-combined formats and combinded formats w/o discrete objects */ + if ( st_ivas->ivas_format != MASA_ISM_FORMAT || st_ivas->ism_mode != ISM_MASA_MODE_DISC ) + { + st_ivas->hTcBuffer->nb_subframes = hSpatParamRendCom->nb_subframes; + mvs2s( hSpatParamRendCom->subframe_nbslots, st_ivas->hTcBuffer->subframe_nbslots, hSpatParamRendCom->nb_subframes ); + } +#endif /* set mapping according to dirac_read_idx */ @@ -1600,6 +1617,10 @@ void ivas_dirac_dec( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ const int16_t nchan_transport /* i : number of transport channels */ +#ifdef API_5MS + , + const int16_t num_subframes /* i : number of subframes to render */ +#endif ) { int16_t subframe_idx; @@ -1632,7 +1653,11 @@ void ivas_dirac_dec( ivas_dirac_dec_set_md_map( st_ivas, DEFAULT_JBM_CLDFB_TIMESLOTS ); +#ifdef API_5MS + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) +#endif { ivas_dirac_dec_render_sf( st_ivas, output_f_local, nchan_transport, NULL, NULL ); for ( n = 0; n < nchan_out; n++ ) @@ -1671,12 +1696,12 @@ void ivas_dirac_dec( *------------------------------------------------------------------------*/ void ivas_dirac_dec_render( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const int16_t nchan_transport, /* i : number of transport channels */ - const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ - uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ - uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ - float *output_f[] /* o : rendered time signal */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const int16_t nchan_transport, /* i : number of transport channels */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailableNext, /* o : number of CLDFB slots still to render */ + float *output_f[] /* o : rendered time signal */ ) { int16_t slots_to_render, first_sf, last_sf, subframe_idx; @@ -1733,7 +1758,7 @@ void ivas_dirac_dec_render( } } - *nSamplesAvailable = ( hSpatParamRendCom->num_slots - hSpatParamRendCom->slots_rendered ) * slot_size; + *nSamplesAvailableNext = ( hSpatParamRendCom->num_slots - hSpatParamRendCom->slots_rendered ) * slot_size; return; } @@ -1866,9 +1891,17 @@ void ivas_dirac_dec_render_sf( set_zero( onset_filter_subframe, hSpatParamRendCom->num_freq_bands ); } +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation ) +#else if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation[subframe_idx] ) +#endif { +#ifdef API_5MS + p_Rmat = &st_ivas->hCombinedOrientationData->Rmat[0][0]; +#else p_Rmat = &st_ivas->hCombinedOrientationData->Rmat[subframe_idx][0][0]; +#endif if ( st_ivas->hCombinedOrientationData->shd_rot_max_order == 0 ) { @@ -1932,7 +1965,11 @@ void ivas_dirac_dec_render_sf( set_zero( surCohRatio, hSpatParamRendCom->num_freq_bands ); } } +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation && st_ivas->hCombinedOrientationData->shd_rot_max_order == 1 ) +#else if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation[subframe_idx] && st_ivas->hCombinedOrientationData->shd_rot_max_order == 1 ) +#endif { ivas_dirac_dec_compute_directional_responses( hSpatParamRendCom, hDirACRend, @@ -2059,7 +2096,11 @@ void ivas_dirac_dec_render_sf( if ( hDirACRend->synthesisConf == DIRAC_SYNTHESIS_GAIN_SHD ) { +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation && st_ivas->hCombinedOrientationData->shd_rot_max_order == 0 ) +#else if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation[subframe_idx] && st_ivas->hCombinedOrientationData->shd_rot_max_order == 0 ) +#endif { protoSignalComputation_shd( Cldfb_RealBuffer, Cldfb_ImagBuffer, hDirACRend->h_output_synthesis_psd_state.proto_direct_buffer_f, @@ -2290,8 +2331,12 @@ void ivas_dirac_dec_render_sf( ivas_dirac_dec_compute_diffuse_proto( hDirACRend, hSpatParamRendCom->num_freq_bands, slot_idx ); } - /*Compute PSDs*/ +/*Compute PSDs*/ +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation && st_ivas->hCombinedOrientationData->shd_rot_max_order > 0 ) +#else if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation[subframe_idx] && st_ivas->hCombinedOrientationData->shd_rot_max_order > 0 ) +#endif { ivas_dirac_dec_output_synthesis_process_slot( reference_power, p_onset_filter, @@ -2409,7 +2454,15 @@ void ivas_dirac_dec_render_sf( #ifdef SPLIT_REND_WITH_HEAD_ROT &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, #endif - st_ivas->hCombinedOrientationData, subframe_idx, hSpatParamRendCom->subframe_nbslots[subframe_idx], Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, Cldfb_RealBuffer, Cldfb_ImagBuffer ); + st_ivas->hCombinedOrientationData, +#ifndef API_5MS + subframe_idx, +#endif + hSpatParamRendCom->subframe_nbslots[subframe_idx], + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + Cldfb_RealBuffer, + Cldfb_ImagBuffer ); #ifdef SPLIT_REND_WITH_HEAD_ROT if ( st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || st_ivas->hDecoderConfig->output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) diff --git a/lib_dec/ivas_init_dec.c b/lib_dec/ivas_init_dec.c index 1f401bb42084a678bf330a5141f702a12e35c0ba..c5e49191b6c9b2b8e7cd15147e677632545240f4 100644 --- a/lib_dec/ivas_init_dec.c +++ b/lib_dec/ivas_init_dec.c @@ -247,7 +247,27 @@ static ivas_error ivas_dec_init_split_rend( } } +#ifdef API_5MS + if ( ( error = ivas_split_rend_choose_default_codec( &st_ivas->hRenderConfig->split_rend_config.codec, + &st_ivas->hRenderConfig->split_rend_config.codec_frame_size_ms, + ( cldfb_in == 0 ), + pcm_out ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( st_ivas->hRenderConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && + ( st_ivas->hRenderConfig->split_rend_config.dof == 0 || st_ivas->hRenderConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) ) + { + st_ivas->hDecoderConfig->Opt_5ms = true; + } + else + { + st_ivas->hDecoderConfig->Opt_5ms = false; + } +#else ivas_split_rend_choose_default_codec( &st_ivas->hRenderConfig->split_rend_config.codec, ( cldfb_in == 0 ), pcm_out ); +#endif #ifdef OSBA_SPLIT_RENDERING if ( st_ivas->renderer_type == RENDERER_BINAURAL_FASTCONV && st_ivas->ivas_format == SBA_ISM_FORMAT ) { @@ -286,7 +306,12 @@ static ivas_error ivas_dec_init_split_rend( ivas_error ivas_dec_setup( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ - int16_t *data /* o : flushed PCM samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { int16_t k, idx, num_bits_read; @@ -337,7 +362,11 @@ ivas_error ivas_dec_setup( st_ivas->nchan_ism = nchan_ism; - if ( ( error = ivas_ism_dec_config( st_ivas, st_ivas->ism_mode, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_ism_dec_config( st_ivas, st_ivas->ism_mode, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -394,7 +423,11 @@ ivas_error ivas_dec_setup( } else { - if ( ( error = ivas_masa_dec_reconfigure( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_masa_dec_reconfigure( st_ivas, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -402,7 +435,11 @@ ivas_error ivas_dec_setup( } else { - if ( ( error = ivas_omasa_dec_config( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_omasa_dec_config( st_ivas, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -423,7 +460,11 @@ ivas_error ivas_dec_setup( /* reconfigure in case a change of operation mode is detected */ if ( ( ivas_total_brate > IVAS_SID_5k2 && ivas_total_brate != st_ivas->hDecoderConfig->last_ivas_total_brate ) || ( st_ivas->ini_active_frame == 0 ) ) { - if ( ( error = ivas_omasa_dec_config( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_omasa_dec_config( st_ivas, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -495,7 +536,12 @@ ivas_error ivas_dec_setup( num_bits_read += MC_LS_SETUP_BITS; /* select MC format mode; reconfigure the MC format decoder */ - ivas_mc_dec_config( st_ivas, idx, nSamplesRendered, data ); + ivas_mc_dec_config( st_ivas, idx, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + + data ); } /*-------------------------------------------------------------------* @@ -610,7 +656,11 @@ ivas_error ivas_dec_setup( st_ivas->ism_mode = (ISM_MODE) ( idx + 1 ); } - if ( ( error = ivas_ism_dec_config( st_ivas, st_ivas->ism_mode, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_ism_dec_config( st_ivas, st_ivas->ism_mode, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -1913,7 +1963,13 @@ ivas_error ivas_init_decoder( } +#ifdef API_5MS_BASELINE +#ifndef API_5MS if ( st_ivas->hDecoderConfig->voip_active ) +#else + if ( st_ivas->hDecoderConfig->Opt_5ms ) +#endif +#endif { granularity = NS2SA( st_ivas->hDecoderConfig->output_Fs, FRAME_SIZE_NS / MAX_PARAM_SPATIAL_SUBFRAMES ); n_channels_transport_jbm = ivas_jbm_dec_get_num_tc_channels( st_ivas ); @@ -1961,7 +2017,13 @@ ivas_error ivas_init_decoder( st_ivas->binaural_latency_ns = st_ivas->hCrendWrapper->binaural_latency_ns; +#ifdef API_5MS_BASELINE +#ifndef API_5MS if ( st_ivas->hDecoderConfig->voip_active ) +#else + if ( st_ivas->hDecoderConfig->Opt_5ms ) +#endif +#endif { if ( ( st_ivas->ivas_format == MC_FORMAT ) && ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) ) { @@ -2135,8 +2197,16 @@ ivas_error ivas_init_decoder( /*-----------------------------------------------------------------* * Allocate and initialize JBM struct + buffer *-----------------------------------------------------------------*/ + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms && +#endif +#else + st_ivas->hDecoderConfig->voip_active == 1 && +#endif + st_ivas->hTcBuffer == NULL ) - if ( st_ivas->hDecoderConfig->voip_active && st_ivas->hTcBuffer == NULL ) { /* no module has yet open the TC buffer, open a default one */ n_channels_transport_jbm = ivas_jbm_dec_get_num_tc_channels( st_ivas ); @@ -2147,6 +2217,7 @@ ivas_error ivas_init_decoder( } } +#if !defined( API_5MS ) || defined( API_5MS_BASELINE ) if ( st_ivas->hTcBuffer == NULL ) { /* we need the handle anyway, but without the buffer*/ @@ -2155,6 +2226,7 @@ ivas_error ivas_init_decoder( return error; } } +#endif if ( st_ivas->hJbmMetadata == NULL ) { @@ -2171,7 +2243,11 @@ ivas_error ivas_init_decoder( * Allocate floating-point output audio buffers *-----------------------------------------------------------------*/ +#ifdef API_5MS + if ( !st_ivas->hDecoderConfig->Opt_5ms ) +#else if ( !st_ivas->hDecoderConfig->voip_active ) +#endif { for ( n = 0; n < ivas_get_nchan_buffers_dec( st_ivas ); n++ ) { @@ -2394,6 +2470,17 @@ void ivas_initialize_handles_dec( /* rendering handles */ st_ivas->hBinRenderer = NULL; +#ifdef SPLIT_REND_WITH_HEAD_ROT + st_ivas->hSplitBinRend.hMultiBinCldfbData = NULL; + st_ivas->hSplitBinRend.hSplitRendBits = NULL; + st_ivas->hSplitBinRend.hCldfbDataOut = NULL; +#ifdef API_5MS + st_ivas->hSplitBinRend.tdDataOut = NULL; + st_ivas->hSplitBinRend.numTdSamplesPerChannelCached = 0; + st_ivas->hSplitBinRend.numSamplesCollected = 0; +#endif + ivas_init_split_rend_handles( &st_ivas->hSplitBinRend.splitrend ); +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN for ( i = 0; i < MAX_HEAD_ROT_POSES; i++ ) { @@ -2589,6 +2676,12 @@ void ivas_destroy_dec( free( st_ivas->hSplitBinRend.hSplitRendBits ); st_ivas->hSplitBinRend.hSplitRendBits = NULL; } +#ifdef API_5MS + if ( st_ivas->hSplitBinRend.tdDataOut != NULL ) + { + free( st_ivas->hSplitBinRend.tdDataOut ); + } +#endif #endif /* Parametric binaural renderer handle */ diff --git a/lib_dec/ivas_ism_dec.c b/lib_dec/ivas_ism_dec.c index 073b01824d853a1ea13b80683f75446161af9c4f..1e9007920306c643966a3b65a87631e527daaad7 100644 --- a/lib_dec/ivas_ism_dec.c +++ b/lib_dec/ivas_ism_dec.c @@ -51,7 +51,12 @@ static ivas_error ivas_ism_bitrate_switching_dec( const int16_t nchan_transport_old, /* i : last number of transport channels */ const ISM_MODE last_ism_mode, /* i : last ISM mode */ uint16_t *nSamplesRendered, /* o : number of samples rendered */ - int16_t *data /* o : rendered samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { ivas_error error; @@ -122,7 +127,13 @@ static ivas_error ivas_ism_bitrate_switching_dec( ivas_output_init( &( st_ivas->hIntSetup ), st_ivas->hDecoderConfig->output_config ); } +#ifdef API_5MS_BASELINE +#ifndef API_5MS if ( st_ivas->hDecoderConfig->voip_active ) +#else + if ( st_ivas->hDecoderConfig->Opt_5ms ) +#endif +#endif { /* transfer subframe info from DirAC or ParamMC to central tc buffer */ if ( last_ism_mode == ISM_MODE_PARAM && st_ivas->hSpatParamRendCom != NULL && ( st_ivas->renderer_type != RENDERER_MONO_DOWNMIX && st_ivas->renderer_type != RENDERER_DISABLE ) ) @@ -139,7 +150,11 @@ static ivas_error ivas_ism_bitrate_switching_dec( tc_granularity_new = ivas_jbm_dec_get_render_granularity( st_ivas->renderer_type, st_ivas->ivas_format, st_ivas->mc_mode, st_ivas->hDecoderConfig->output_Fs ); if ( tc_granularity_new < st_ivas->hTcBuffer->n_samples_granularity ) { - if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, tc_granularity_new, renderer_type_old, intern_config_old, &hIntSetupOld, MC_MODE_NONE, last_ism_mode, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, tc_granularity_new, renderer_type_old, intern_config_old, &hIntSetupOld, MC_MODE_NONE, last_ism_mode, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -330,7 +345,11 @@ static ivas_error ivas_ism_bitrate_switching_dec( * floating-point output audio buffers *-----------------------------------------------------------------*/ +#ifdef API_5MS + if ( !st_ivas->hDecoderConfig->Opt_5ms ) +#else if ( !st_ivas->hDecoderConfig->voip_active ) +#endif { nchan_out_buff = ivas_get_nchan_buffers_dec( st_ivas ); @@ -358,8 +377,13 @@ static ivas_error ivas_ism_bitrate_switching_dec( /*-----------------------------------------------------------------* * Reconfigure TC buffer *-----------------------------------------------------------------*/ - +#ifdef API_5MS +#ifdef API_5MS_BASELINE + if ( st_ivas->hDecoderConfig->Opt_5ms ) +#endif +#else if ( st_ivas->hDecoderConfig->voip_active == 1 ) +#endif { int16_t tc_nchan_full_new; DECODER_TC_BUFFER_HANDLE hTcBuffer; @@ -418,7 +442,12 @@ ivas_error ivas_ism_dec_config( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const ISM_MODE last_ism_mode, /* i/o: last ISM mode */ uint16_t *nSamplesRendered, /* o : number of samples flushed when the renderer granularity changes */ - int16_t *data /* o : flushed PCM samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { int32_t ivas_total_brate; @@ -457,7 +486,11 @@ ivas_error ivas_ism_dec_config( { if ( ( st_ivas->ism_mode != last_ism_mode ) || ( st_ivas->hDecoderConfig->ivas_total_brate != st_ivas->hDecoderConfig->last_ivas_total_brate ) ) { - if ( ( error = ivas_ism_bitrate_switching_dec( st_ivas, nchan_transport_old, last_ism_mode, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_ism_bitrate_switching_dec( st_ivas, nchan_transport_old, last_ism_mode, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -480,7 +513,11 @@ ivas_error ivas_ism_dec_config( /* ISM mode switching */ if ( st_ivas->ism_mode != last_ism_mode ) { - if ( ( error = ivas_ism_bitrate_switching_dec( st_ivas, nchan_transport_old, last_ism_mode, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_ism_bitrate_switching_dec( st_ivas, nchan_transport_old, last_ism_mode, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } diff --git a/lib_dec/ivas_ism_dtx_dec.c b/lib_dec/ivas_ism_dtx_dec.c index fada7caaecd56a2c8658c8188d6133b2af0cf6b1..682ba615fa3b57449965a326f0f1e9a56690ca39 100644 --- a/lib_dec/ivas_ism_dtx_dec.c +++ b/lib_dec/ivas_ism_dtx_dec.c @@ -96,7 +96,11 @@ ivas_error ivas_ism_dtx_dec( st_ivas->ism_mode = ism_mode_bstr; } - if ( ( error = ivas_ism_dec_config( st_ivas, last_ism_mode, NULL, NULL ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_ism_dec_config( st_ivas, last_ism_mode, NULL, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + PCM_NOT_KNOW, +#endif + NULL ) ) != IVAS_ERR_OK ) { return error; } diff --git a/lib_dec/ivas_ism_param_dec.c b/lib_dec/ivas_ism_param_dec.c index a1f8534f98a6e87d9e1fdc5710afffd27b0d8e4e..32368601fd3aecb7d71abe3f1873b1d76b1813b7 100644 --- a/lib_dec/ivas_ism_param_dec.c +++ b/lib_dec/ivas_ism_param_dec.c @@ -528,7 +528,11 @@ ivas_error ivas_param_ism_dec_open( if ( !( output_config == AUDIO_CONFIG_MONO || output_config == AUDIO_CONFIG_STEREO ) ) { /* Initialize Param ISM Rendering handle */ +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm ) +#else if ( st_ivas->hDecoderConfig->voip_active ) +#endif { if ( ( error = ivas_param_ism_rendering_init( hDirAC->hParamIsmRendering, hOutSetup, st_ivas->nchan_transport, MAX_JBM_CLDFB_TIMESLOTS, output_config ) ) != IVAS_ERR_OK ) { @@ -590,7 +594,13 @@ ivas_error ivas_param_ism_dec_open( st_ivas->hDirAC = hDirAC; st_ivas->hSpatParamRendCom = hSpatParamRendCom; +#ifndef API_5MS if ( st_ivas->hDecoderConfig->voip_active ) +#else +#ifdef API_5MS_BASELINE + if ( st_ivas->hDecoderConfig->Opt_5ms ) +#endif +#endif { if ( st_ivas->renderer_type != RENDERER_MONO_DOWNMIX && st_ivas->renderer_type != RENDERER_DISABLE ) { @@ -604,7 +614,31 @@ ivas_error ivas_param_ism_dec_open( } else { +#ifdef API_5MS + int16_t n_slots_to_alloc; + if ( st_ivas->hDecoderConfig->Opt_tsm == 1 ) + { + n_slots_to_alloc = MAX_JBM_CLDFB_TIMESLOTS; + } + else + { + n_slots_to_alloc = CLDFB_SLOTS_PER_SUBFRAME * MAX_PARAM_SPATIAL_SUBFRAMES; + } + if ( ( hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc = (float *) malloc( n_slots_to_alloc * nchan_transport * hSpatParamRendCom->num_freq_bands * sizeof( float ) ) ) == NULL ) + + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Param ISM JBM Rendering handle\n" ) ); + } + set_zero( hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc, n_slots_to_alloc * nchan_transport * hSpatParamRendCom->num_freq_bands ); + + if ( ( hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc = (float *) malloc( n_slots_to_alloc * nchan_transport * hSpatParamRendCom->num_freq_bands * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Param ISM JBM Rendering handle\n" ) ); + } + set_zero( hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc, n_slots_to_alloc * nchan_transport * hSpatParamRendCom->num_freq_bands ); +#else if ( ( hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * nchan_transport * hSpatParamRendCom->num_freq_bands * sizeof( float ) ) ) == NULL ) + { return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Param ISM JBM Rendering handle\n" ) ); } @@ -615,6 +649,7 @@ ivas_error ivas_param_ism_dec_open( return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Param ISM JBM Rendering handle\n" ) ); } set_zero( hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc, MAX_JBM_CLDFB_TIMESLOTS * nchan_transport * hSpatParamRendCom->num_freq_bands ); +#endif } if ( st_ivas->hTcBuffer == NULL ) { @@ -638,11 +673,13 @@ ivas_error ivas_param_ism_dec_open( } } } +#if !defined( API_5MS ) || defined API_5MS_BASELINE else { hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc = NULL; hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc = NULL; } +#endif pop_wmops(); return error; @@ -740,8 +777,12 @@ void ivas_param_ism_dec( int16_t subframe_idx, slot_idx, index_slot, bin_idx; int32_t ivas_total_brate; int16_t output_frame; +#ifndef API_5MS float gain, ene_tc, ene_sum, grad; float last_gain; +#else + float *p_tc[PARAM_ISM_MAX_DMX]; +#endif float ref_power[CLDFB_NO_CHANNELS_MAX]; float cx_diag[CLDFB_NO_CHANNELS_MAX][PARAM_ISM_MAX_DMX]; /* CLDFB Input Buffers */ @@ -768,9 +809,17 @@ void ivas_param_ism_dec( assert( hDirAC ); hSpatParamRendCom = st_ivas->hSpatParamRendCom; assert( hSpatParamRendCom ); +#ifdef API_5MS + for ( i = 0; i < PARAM_ISM_MAX_DMX; i++ ) + { + p_tc[i] = output_f[i]; + } +#else + ene_tc = 0.0f; ene_sum = 0.0f; last_gain = st_ivas->hDirAC->hParamIsm->last_dmx_gain; +#endif output_frame = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); nchan_transport = st_ivas->nchan_transport; @@ -861,7 +910,9 @@ void ivas_param_ism_dec( } } } - +#ifdef API_5MS + ivas_ism_param_dec_tc_gain_ajust( st_ivas, output_frame, output_frame / 2, p_tc ); +#else /* Energy Compensation */ for ( i = 0; i < output_frame; i++ ) { @@ -895,7 +946,7 @@ void ivas_param_ism_dec( } } st_ivas->hDirAC->hParamIsm->last_dmx_gain = gain; - +#endif /* API_5MS */ for ( ch = 0; ch < nchan_transport; ch++ ) { /*-----------------------------------------------------------------* @@ -1111,27 +1162,31 @@ void ivas_param_ism_dec_digest_tc( int16_t ch, nchan_transport, nchan_out, nchan_out_woLFE, i; int16_t slot_idx, bin_idx; int32_t ivas_total_brate; +#ifndef API_5MS int16_t output_frame; + int16_t fade_len; float gain, ene_tc, ene_sum, grad; float last_gain; +#endif float ref_power[CLDFB_NO_CHANNELS_MAX]; float cx_diag[CLDFB_NO_CHANNELS_MAX][PARAM_ISM_MAX_DMX]; /* Direct Response/EFAP Gains */ float direct_response[MAX_NUM_OBJECTS][PARAM_ISM_MAX_CHAN]; DIRAC_DEC_HANDLE hDirAC; SPAT_PARAM_REND_COMMON_DATA_HANDLE hSpatParamRendCom; - int16_t fade_len; /* Initialization */ hDirAC = st_ivas->hDirAC; assert( hDirAC ); hSpatParamRendCom = st_ivas->hSpatParamRendCom; assert( hSpatParamRendCom ); +#ifndef API_5MS ene_tc = 0.0f; ene_sum = 0.0f; last_gain = st_ivas->hDirAC->hParamIsm->last_dmx_gain; output_frame = nCldfbSlots * hSpatParamRendCom->num_freq_bands; fade_len = output_frame / 2; +#endif nchan_transport = st_ivas->nchan_transport; ivas_total_brate = st_ivas->hDecoderConfig->ivas_total_brate; @@ -1224,7 +1279,16 @@ void ivas_param_ism_dec_digest_tc( } } } - +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm +#ifdef API_5MS_BASELINE + || !st_ivas->hDecoderConfig->Opt_5ms +#endif + ) + { + ivas_ism_param_dec_tc_gain_ajust( st_ivas, nCldfbSlots * hSpatParamRendCom->num_freq_bands, (int16_t) ( st_ivas->hDecoderConfig->output_Fs / ( 2 * FRAMES_PER_SEC ) ), transport_channels_f ); + } +#else /* Energy Compensation */ for ( i = 0; i < output_frame; i++ ) { @@ -1257,6 +1321,7 @@ void ivas_param_ism_dec_digest_tc( } } st_ivas->hDirAC->hParamIsm->last_dmx_gain = gain; +#endif /* API_5MS */ for ( ch = 0; ch < nchan_transport; ch++ ) { @@ -1265,14 +1330,27 @@ void ivas_param_ism_dec_digest_tc( *-----------------------------------------------------------------*/ for ( slot_idx = 0; slot_idx < nCldfbSlots; slot_idx++ ) { - float RealBuffer[CLDFB_NO_CHANNELS_MAX]; - float ImagBuffer[CLDFB_NO_CHANNELS_MAX]; +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm +#ifdef API_5MS_BASELINE + || !st_ivas->hDecoderConfig->Opt_5ms +#endif + ) + { +#endif - cldfbAnalysis_ts( &( transport_channels_f[ch][hSpatParamRendCom->num_freq_bands * slot_idx] ), RealBuffer, ImagBuffer, hSpatParamRendCom->num_freq_bands, st_ivas->cldfbAnaDec[ch] ); - mvr2r( RealBuffer, &hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], hSpatParamRendCom->num_freq_bands ); - mvr2r( ImagBuffer, &hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], hSpatParamRendCom->num_freq_bands ); + float RealBuffer[CLDFB_NO_CHANNELS_MAX]; + float ImagBuffer[CLDFB_NO_CHANNELS_MAX]; + cldfbAnalysis_ts( &( transport_channels_f[ch][hSpatParamRendCom->num_freq_bands * slot_idx] ), RealBuffer, ImagBuffer, hSpatParamRendCom->num_freq_bands, st_ivas->cldfbAnaDec[ch] ); + mvr2r( RealBuffer, &hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], hSpatParamRendCom->num_freq_bands ); + mvr2r( ImagBuffer, &hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], hSpatParamRendCom->num_freq_bands ); +#ifdef API_5MS + } + ivas_param_ism_collect_slot( hDirAC, &hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], &hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc[slot_idx * hSpatParamRendCom->num_freq_bands * nchan_transport + ch * hSpatParamRendCom->num_freq_bands], ch, ref_power, cx_diag ); +#else ivas_param_ism_collect_slot( hDirAC, RealBuffer, ImagBuffer, ch, ref_power, cx_diag ); +#endif } } @@ -1291,6 +1369,67 @@ void ivas_param_ism_dec_digest_tc( } +#ifdef API_5MS +/*-------------------------------------------------------------------------* + * ivas_ism_param_dec_tc_gain_ajust() + * + * + *-------------------------------------------------------------------------*/ + +void ivas_ism_param_dec_tc_gain_ajust( + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamples, /* i : number of samples to be compensate */ + const uint16_t nFadeLength, /* i : length of the crossfade in samples */ + float *transport_channels_f[] /* i : synthesized core-coder transport channels/DirAC output */ +) + +{ + int16_t i; + float gain, ene_tc, ene_sum, grad; + float last_gain; + + ene_tc = 0.0f; + ene_sum = 0.0f; + last_gain = st_ivas->hDirAC->hParamIsm->last_dmx_gain; + + + for ( i = 0; i < nSamples; i++ ) + { + ene_tc += transport_channels_f[0][i] * transport_channels_f[0][i] + transport_channels_f[1][i] * transport_channels_f[1][i]; // L*L + R*R + ene_sum += ( transport_channels_f[0][i] + transport_channels_f[1][i] ) * ( transport_channels_f[0][i] + transport_channels_f[1][i] ); // (L+R)*(L+R) + } + gain = sqrtf( ene_tc / ( ene_sum + EPSILON ) ); + if ( st_ivas->hSCE[0]->hCoreCoder[0]->ini_frame > 1 ) + { + /* Smoothing */ + gain = 0.75f * gain + 0.25f * last_gain; + /* 10ms ramp */ + grad = ( gain - last_gain ) / (float) nFadeLength; /* slope between two consecutive gains, 480 samples length */ + for ( i = 0; i < ( nFadeLength ); i++ ) + { + transport_channels_f[0][i] *= ( last_gain + i * grad ); + transport_channels_f[1][i] *= ( last_gain + i * grad ); + } + for ( ; i < nSamples; i++ ) + { + transport_channels_f[0][i] *= gain; + transport_channels_f[1][i] *= gain; + } + } + else + { + for ( i = 0; i < nSamples; i++ ) + { + transport_channels_f[0][i] *= gain; + transport_channels_f[1][i] *= gain; + } + } + st_ivas->hDirAC->hParamIsm->last_dmx_gain = gain; + + return; +} +#endif + /*-------------------------------------------------------------------------* * ivas_ism_param_dec_render_sf() * @@ -1396,11 +1535,11 @@ static void ivas_ism_param_dec_render_sf( *-------------------------------------------------------------------------*/ void ivas_param_ism_dec_render( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ - uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ - uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ - float *output_f[] /* o : rendered time signal */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailableNext, /* o : number of CLDFB slots still to render */ + float *output_f[] /* o : rendered time signal */ ) { int16_t ch, slots_to_render, first_sf, last_sf, subframe_idx; @@ -1484,7 +1623,7 @@ void ivas_param_ism_dec_render( } } - *nSamplesAvailable = ( hSpatParamRendCom->num_slots - hSpatParamRendCom->slots_rendered ) * slot_size; + *nSamplesAvailableNext = ( hSpatParamRendCom->num_slots - hSpatParamRendCom->slots_rendered ) * slot_size; return; } diff --git a/lib_dec/ivas_ism_renderer.c b/lib_dec/ivas_ism_renderer.c index 9cb92cd2fbb9ff7aa5766b5575d5a5b6b06dd67c..30d77ad3c554edddf8211c9a07e6b58daee85834 100644 --- a/lib_dec/ivas_ism_renderer.c +++ b/lib_dec/ivas_ism_renderer.c @@ -82,7 +82,11 @@ ivas_error ivas_ism_renderer_open( set_f( st_ivas->hIsmRendererData->gains[i], 0.0f, MAX_OUTPUT_CHANNELS ); } +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm ) +#else if ( st_ivas->hDecoderConfig->voip_active ) +#endif { init_interpolator_length = NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_CLDFB_TIMESLOTS * CLDFB_SLOT_NS ); interpolator_length = (uint16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); @@ -161,9 +165,17 @@ void ivas_ism_render( else { /* Combined rotation: rotate the object positions depending the head and external orientations */ +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData != NULL && st_ivas->hCombinedOrientationData->enableCombinedOrientation == 1 ) +#else if ( st_ivas->hCombinedOrientationData != NULL && st_ivas->hCombinedOrientationData->enableCombinedOrientation[0] == 1 ) +#endif { +#ifdef API_5MS + rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat, st_ivas->hIntSetup.is_planar_setup ); +#else rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat[0], st_ivas->hIntSetup.is_planar_setup ); +#endif } else { @@ -244,7 +256,11 @@ void ivas_ism_render_sf( set_f( output_f[i], 0.0f, n_samples_to_render ); } +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation ) +#else if ( st_ivas->hCombinedOrientationData && st_ivas->hCombinedOrientationData->enableCombinedOrientation[0] ) +#endif { ivas_jbm_dec_get_adapted_linear_interpolator( n_samples_to_render, n_samples_to_render, @@ -256,9 +272,17 @@ void ivas_ism_render_sf( { /* Combined rotation: rotate the object positions depending the head and external orientations */ +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData != NULL && st_ivas->hCombinedOrientationData->enableCombinedOrientation == 1 ) +#else if ( st_ivas->hCombinedOrientationData != NULL && st_ivas->hCombinedOrientationData->enableCombinedOrientation[0] == 1 ) +#endif { +#ifdef API_5MS + rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat, st_ivas->hIntSetup.is_planar_setup ); +#else rotateAziEle( st_ivas->hIsmMetaData[i]->azimuth, st_ivas->hIsmMetaData[i]->elevation, &azimuth, &elevation, st_ivas->hCombinedOrientationData->Rmat[0], st_ivas->hIntSetup.is_planar_setup ); +#endif if ( st_ivas->hEFAPdata != NULL ) { efap_determine_gains( st_ivas->hEFAPdata, st_ivas->hIsmRendererData->gains[i], azimuth, elevation, EFAP_MODE_EFAP ); @@ -287,7 +311,11 @@ void ivas_ism_render_sf( } /* update here only in case of head rotation */ +#ifdef API_5MS + if ( st_ivas->hCombinedOrientationData != NULL && st_ivas->hCombinedOrientationData->enableCombinedOrientation == 1 ) +#else if ( st_ivas->hCombinedOrientationData != NULL && st_ivas->hCombinedOrientationData->enableCombinedOrientation[0] == 1 ) +#endif { st_ivas->hIsmRendererData->prev_gains[i][j] = gain; } @@ -370,6 +398,8 @@ ivas_error ivas_omasa_separate_object_renderer_open( set_f( st_ivas->hIsmRendererData->prev_gains[i], 0.0f, MAX_OUTPUT_CHANNELS ); } +#ifndef API_5MS + // Todo OMASA JBM: This needs touches for VOIP path at least. Current version is mostly an adapted copy from ivas_ism_renderer_open() if ( st_ivas->hDecoderConfig->voip_active ) { init_interpolator_length = NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_CLDFB_TIMESLOTS * CLDFB_SLOT_NS ); @@ -377,9 +407,12 @@ ivas_error ivas_omasa_separate_object_renderer_open( } else { +#endif init_interpolator_length = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ); interpolator_length = init_interpolator_length; +#ifndef API_5MS } +#endif st_ivas->hIsmRendererData->interpolator = (float *) malloc( sizeof( float ) * init_interpolator_length ); for ( i = 0; i < interpolator_length; i++ ) diff --git a/lib_dec/ivas_jbm_dec.c b/lib_dec/ivas_jbm_dec.c index 419cd56195d6d26205c03339aefa0ff14a1ef60d..1d8e5fb3cc447ec4e0de1fd262ffb48d5f24d263 100644 --- a/lib_dec/ivas_jbm_dec.c +++ b/lib_dec/ivas_jbm_dec.c @@ -113,7 +113,7 @@ ivas_error ivas_jbm_dec_tc( /* zero output when first frame(s) is lost */ for ( n = 0; n < nchan_out; n++ ) { - set_f( output[n], 0.0f, output_frame ); + set_f( p_output[n], 0.0f, output_frame ); } #ifdef DEBUG_MODE_INFO @@ -591,15 +591,44 @@ ivas_error ivas_jbm_dec_tc( ivas_mono_stereo_downmix_mcmasa( st_ivas, output, output_frame ); } } +#ifdef API_5MS + else if ( st_ivas->mc_mode == MC_MODE_PARAMUPMIX ) + { + + /* at least decode everything here, the rest is ToDo, for this we just output zeroes atm */ + ivas_lfe_dec( st_ivas->hLFE, st, output_frame, st_ivas->bfi, output[LFE_CHANNEL] ); + + ivas_mc_paramupmix_dec_read_BS( st_ivas, st, st_ivas->hMCParamUpmix, &nb_bits_metadata[0] ); + + if ( ( error = ivas_mct_dec( st_ivas, output, output_frame, nb_bits_metadata[0] ) ) != IVAS_ERR_OK ) + { + return error; + } + } +#endif } /*----------------------------------------------------------------* * Write IVAS transport channels *----------------------------------------------------------------*/ - - ivas_syn_output_f( p_output, output_frame, st_ivas->hTcBuffer->nchan_transport_jbm, data ); - +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm == 1 +#ifdef API_5MS_BASELINE + || !st_ivas->hDecoderConfig->Opt_5ms +#endif + ) + { +#endif + ivas_syn_output_f( p_output, output_frame, st_ivas->hTcBuffer->nchan_transport_jbm, data ); +#ifdef API_5MS + } + else + { + /* directly copy to tc buffers */ + ivas_jbm_dec_copy_tc_no_tsm( st_ivas, p_output, output_frame ); + } +#endif /*----------------------------------------------------------------* * Common updates @@ -665,7 +694,22 @@ ivas_error ivas_jbm_dec_feed_tc_to_renderer( { p_data_f[n] = &data_f[n][0]; } - ivas_jbm_dec_copy_tc( st_ivas, nSamplesForRendering, nSamplesResidual, data, p_data_f ); +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm +#ifdef API_5MS_BASELINE + || !st_ivas->hDecoderConfig->Opt_5ms +#endif + ) + { +#endif + ivas_jbm_dec_copy_tc( st_ivas, nSamplesForRendering, nSamplesResidual, data, p_data_f ); +#ifdef API_5MS + } + else + { + *nSamplesResidual = 0; + } +#endif n_render_timeslots = st_ivas->hTcBuffer->n_samples_available / st_ivas->hTcBuffer->n_samples_granularity; if ( st_ivas->hTcBuffer->tc_buffer_mode == TC_BUFFER_MODE_BUFFER ) @@ -754,7 +798,12 @@ ivas_error ivas_jbm_dec_render( const uint16_t nSamplesAsked, /* i : number of samples wanted */ uint16_t *nSamplesRendered, /* o : number of samples rendered */ uint16_t *nSamplesAvailableNext, /* o : number of samples still available in the rendering pipeline */ - int16_t *data /* o : output synthesis signal */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { int16_t n, nchan_out; @@ -790,7 +839,11 @@ ivas_error ivas_jbm_dec_render( p_output[n] = &output[n][0]; } +#ifdef FIX_676_JBM_USAN + for ( n = 0; n < st_ivas->hTcBuffer->nchan_buffer_full; n++ ) +#else for ( n = 0; n < st_ivas->hTcBuffer->nchan_transport_internal; n++ ) +#endif { p_tc[n] = &st_ivas->hTcBuffer->tc[n][st_ivas->hTcBuffer->n_samples_rendered]; } @@ -1142,15 +1195,41 @@ ivas_error ivas_jbm_dec_render( st_ivas->hTcBuffer->n_samples_discard = 0; } - ivas_limiter_dec( st_ivas->hLimiter, p_output, nchan_out, *nSamplesRendered, st_ivas->BER_detect ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + if ( st_ivas->hDecoderConfig->Opt_Limiter ) +#endif + { + +#ifndef DISABLE_LIMITER + ivas_limiter_dec( st_ivas->hLimiter, p_output, nchan_out, *nSamplesRendered, st_ivas->BER_detect ); +#endif + } +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + switch ( pcm_resolution ) + { + case PCM_INT16: +#endif #ifdef DEBUGGING - st_ivas->noClipping += + st_ivas->noClipping += +#endif + ivas_syn_output( p_output, *nSamplesRendered, nchan_out, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + (int16_t *) #endif - ivas_syn_output( p_output, *nSamplesRendered, nchan_out, data ); + data ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + break; + case PCM_FLOAT32: + ivas_syn_output_f( p_output, *nSamplesRendered, nchan_out, (float *) data ); + break; + default: + error = IVAS_ERR_UNKNOWN; + break; + } +#endif *nSamplesAvailableNext = st_ivas->hTcBuffer->n_samples_available; - pop_wmops(); return error; } @@ -1171,7 +1250,12 @@ ivas_error ivas_jbm_dec_flush_renderer( const MC_MODE mc_mode_old, /* i : old MC mode */ const ISM_MODE ism_mode_old, /* i : old ISM mode */ uint16_t *nSamplesRendered, /* o : number of samples flushed */ - int16_t *data /* o : rendered samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { ivas_error error; @@ -1323,13 +1407,39 @@ ivas_error ivas_jbm_dec_flush_renderer( } /* Only write out the valid data*/ - ivas_limiter_dec( st_ivas->hLimiter, p_output, st_ivas->hDecoderConfig->nchan_out, *nSamplesRendered, st_ivas->BER_detect ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + if ( st_ivas->hDecoderConfig->Opt_Limiter ) +#endif + { +#ifndef DISABLE_LIMITER + ivas_limiter_dec( st_ivas->hLimiter, p_output, st_ivas->hDecoderConfig->nchan_out, *nSamplesRendered, st_ivas->BER_detect ); +#endif + } +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + switch ( pcm_resolution ) + { + case PCM_INT16: +#endif #ifdef DEBUGGING - st_ivas->noClipping += + st_ivas->noClipping += #endif - ivas_syn_output( p_output, *nSamplesRendered, st_ivas->hDecoderConfig->nchan_out, data ); + ivas_syn_output( p_output, *nSamplesRendered, st_ivas->hDecoderConfig->nchan_out, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + (int16_t *) +#endif + data ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + break; + case PCM_FLOAT32: + ivas_syn_output_f( p_output, *nSamplesRendered, st_ivas->hDecoderConfig->nchan_out, (float *) data ); + break; + default: + error = IVAS_ERR_UNKNOWN; + break; + } +#endif return error; } @@ -1618,7 +1728,16 @@ int16_t ivas_jbm_dec_get_num_tc_channels( } if ( st_ivas->ivas_format == SBA_FORMAT ) { - if ( ( st_ivas->sba_planar && num_tc >= 3 ) || ( num_tc == 3 ) ) + if ( +#ifndef FIX_PLANAR_SBA_JBM_RS + ( st_ivas->sba_planar && num_tc >= 3 ) || + ( +#endif + num_tc == 3 +#ifndef FIX_PLANAR_SBA_JBM_RS + ) +#endif + ) { num_tc++; } @@ -1683,6 +1802,12 @@ int16_t ivas_jbm_dec_get_num_tc_channels( } } } +#ifdef API_5MS + else if ( st_ivas->ivas_format == MONO_FORMAT && st_ivas->renderer_type == RENDERER_NON_DIEGETIC_DOWNMIX ) + { + num_tc = MAX_OUTPUT_CHANNELS_IN_DIEGETIC_PAN; + } +#endif return num_tc; } @@ -1741,6 +1866,9 @@ static void ivas_jbm_dec_copy_tc( } hTcBuffer->n_samples_rendered = 0; +#ifdef API_5MS + hTcBuffer->subframes_rendered = 0; +#endif return; } @@ -1843,34 +1971,66 @@ ivas_error ivas_jbm_dec_tc_buffer_open( } else { +#ifdef API_5MS + int16_t n_samp_full, n_samp_residual; +#else int16_t n_samp_full = ( NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_L_FRAME_NS ) + hTcBuffer->n_samples_granularity - 1 ); int16_t n_samp_residual = hTcBuffer->n_samples_granularity - 1; +#endif int32_t offset; - - nsamp_to_allocate = hTcBuffer->nchan_buffer_full * n_samp_full; - nsamp_to_allocate += nchan_residual * n_samp_residual; - - if ( ( hTcBuffer->tc_buffer = (float *) malloc( nsamp_to_allocate * sizeof( float ) ) ) == NULL ) +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm ) { - return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for JBM TC Buffer\n" ) ); + n_samp_full = ( NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_L_FRAME_NS ) + hTcBuffer->n_samples_granularity - 1 ); + n_samp_residual = hTcBuffer->n_samples_granularity - 1; } - set_zero( hTcBuffer->tc_buffer, nsamp_to_allocate ); - - offset = 0; - for ( ch_idx = 0; ch_idx < hTcBuffer->nchan_buffer_full; ch_idx++ ) + else { - hTcBuffer->tc[ch_idx] = &hTcBuffer->tc_buffer[offset]; - offset += n_samp_full; + n_samp_full = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); + n_samp_residual = 0; } - for ( ; ch_idx < hTcBuffer->nchan_transport_internal; ch_idx++ ) +#endif + + + nsamp_to_allocate = hTcBuffer->nchan_buffer_full * n_samp_full; + nsamp_to_allocate += nchan_residual * n_samp_residual; + +#ifdef API_5MS + if ( nsamp_to_allocate == 0 ) { - hTcBuffer->tc[ch_idx] = &hTcBuffer->tc_buffer[offset]; - offset += n_samp_residual; + hTcBuffer->tc_buffer = NULL; + for ( ch_idx = 0; ch_idx < MAX_TRANSPORT_CHANNELS; ch_idx++ ) + { + hTcBuffer->tc[ch_idx] = NULL; + } } - for ( ; ch_idx < MAX_TRANSPORT_CHANNELS; ch_idx++ ) + else { - hTcBuffer->tc[ch_idx] = NULL; +#endif + if ( ( hTcBuffer->tc_buffer = (float *) malloc( nsamp_to_allocate * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for JBM TC Buffer\n" ) ); + } + set_zero( hTcBuffer->tc_buffer, nsamp_to_allocate ); + + offset = 0; + for ( ch_idx = 0; ch_idx < hTcBuffer->nchan_buffer_full; ch_idx++ ) + { + hTcBuffer->tc[ch_idx] = &hTcBuffer->tc_buffer[offset]; + offset += n_samp_full; + } + for ( ; ch_idx < hTcBuffer->nchan_transport_internal; ch_idx++ ) + { + hTcBuffer->tc[ch_idx] = &hTcBuffer->tc_buffer[offset]; + offset += n_samp_residual; + } + for ( ; ch_idx < MAX_TRANSPORT_CHANNELS; ch_idx++ ) + { + hTcBuffer->tc[ch_idx] = NULL; + } +#ifdef API_5MS } +#endif } st_ivas->hTcBuffer = hTcBuffer; @@ -1949,8 +2109,21 @@ ivas_error ivas_jbm_dec_tc_buffer_reconfigure( /* realloc buffers */ free( hTcBuffer->tc_buffer ); +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm ) + { + n_samp_full = ( NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_L_FRAME_NS ) + hTcBuffer->n_samples_granularity - 1 ); + n_samp_residual = hTcBuffer->n_samples_granularity - 1; + } + else + { + n_samp_full = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); + n_samp_residual = 0; + } +#else n_samp_full = ( NS2SA( st_ivas->hDecoderConfig->output_Fs, MAX_JBM_L_FRAME_NS ) + hTcBuffer->n_samples_granularity - 1 ); n_samp_residual = hTcBuffer->n_samples_granularity - 1; +#endif nsamp_to_allocate = hTcBuffer->nchan_buffer_full * n_samp_full; nsamp_to_allocate += nchan_residual * n_samp_residual; @@ -2133,11 +2306,26 @@ TC_BUFFER_MODE ivas_jbm_dec_get_tc_buffer_mode( case RENDERER_PARAM_ISM: case RENDERER_BINAURAL_MIXER_CONV: case RENDERER_BINAURAL_MIXER_CONV_ROOM: +#ifdef API_5MS + buffer_mode = TC_BUFFER_MODE_RENDERER; + break; + case RENDERER_NON_DIEGETIC_DOWNMIX: + if ( st_ivas->ivas_format == MONO_FORMAT ) + { + buffer_mode = TC_BUFFER_MODE_BUFFER; + } + else + { + buffer_mode = TC_BUFFER_MODE_RENDERER; + } + break; +#else case RENDERER_NON_DIEGETIC_DOWNMIX: case RENDERER_OSBA_AMBI: case RENDERER_OSBA_LS: buffer_mode = TC_BUFFER_MODE_RENDERER; break; +#endif case RENDERER_MC_PARAMMC: if ( st_ivas->hParamMC->synthesis_conf == PARAM_MC_SYNTH_MONO_STEREO ) { @@ -2183,6 +2371,76 @@ TC_BUFFER_MODE ivas_jbm_dec_get_tc_buffer_mode( return buffer_mode; } +#ifdef API_5MS +void ivas_jbm_dec_copy_tc_no_tsm( + Decoder_Struct *st_ivas, + float *tc[], + const int16_t output_frame ) +{ + int16_t n_ch_full_copy; + int16_t n_ch_cldfb; + int16_t ch_idx; + DECODER_TC_BUFFER_HANDLE hTcBuffer; + + hTcBuffer = st_ivas->hTcBuffer; + hTcBuffer->n_samples_buffered = output_frame; + hTcBuffer->n_samples_available = hTcBuffer->n_samples_buffered; + n_ch_full_copy = min( hTcBuffer->nchan_transport_jbm, hTcBuffer->nchan_buffer_full ); + n_ch_cldfb = hTcBuffer->nchan_transport_jbm - hTcBuffer->nchan_buffer_full; + /* copy full tcs*/ + for ( ch_idx = 0; ch_idx < n_ch_full_copy; ch_idx++ ) + { + mvr2r( tc[ch_idx], st_ivas->hTcBuffer->tc[ch_idx], hTcBuffer->n_samples_buffered ); + } + + /* CLDFB ana for ParamMC/ParamISM */ + if ( n_ch_cldfb > 0 ) + { + float *cldfb_real_buffer; + float *cldfb_imag_buffer; + int16_t cldfb_ch, slot_idx, num_freq_bands; + + cldfb_real_buffer = NULL; + cldfb_imag_buffer = NULL; + num_freq_bands = 0; + + if ( st_ivas->ivas_format == ISM_FORMAT ) + { + cldfb_real_buffer = st_ivas->hDirAC->hParamIsmRendering->Cldfb_RealBuffer_tc; + cldfb_imag_buffer = st_ivas->hDirAC->hParamIsmRendering->Cldfb_ImagBuffer_tc; + num_freq_bands = st_ivas->hSpatParamRendCom->num_freq_bands; + ivas_ism_param_dec_tc_gain_ajust( st_ivas, output_frame, output_frame / 2, tc ); + } + else if ( st_ivas->ivas_format == MC_FORMAT ) + { + cldfb_real_buffer = st_ivas->hParamMC->Cldfb_RealBuffer_tc; + cldfb_imag_buffer = st_ivas->hParamMC->Cldfb_ImagBuffer_tc; + num_freq_bands = st_ivas->hParamMC->num_freq_bands; + } +#ifdef DEBUGGING + else + { + assert( 0 && "Residual (direct CLDFB transport channels) only possible for ParamMC/ParamISM!" ); + } +#endif + /* CLDFB Analysis*/ + + for ( cldfb_ch = 0; cldfb_ch < n_ch_cldfb; cldfb_ch++, ch_idx++ ) + { + for ( slot_idx = 0; slot_idx < DEFAULT_JBM_CLDFB_TIMESLOTS; slot_idx++ ) + { + cldfbAnalysis_ts( &( tc[ch_idx][num_freq_bands * slot_idx] ), + &cldfb_real_buffer[slot_idx * num_freq_bands * n_ch_cldfb + cldfb_ch * num_freq_bands], + &cldfb_imag_buffer[slot_idx * num_freq_bands * n_ch_cldfb + cldfb_ch * num_freq_bands], + num_freq_bands, st_ivas->cldfbAnaDec[cldfb_ch] ); + } + } + } + hTcBuffer->n_samples_rendered = 0; + hTcBuffer->subframes_rendered = 0; +} +#endif + /*--------------------------------------------------------------------------* * ivas_jbm_dec_metadata_open() diff --git a/lib_dec/ivas_masa_dec.c b/lib_dec/ivas_masa_dec.c index 6bc88b06c8a097f6788ad0c45961c2a35b73aec5..ba87227569ebd9e5d8fae154a3b46ea42bf1e99d 100644 --- a/lib_dec/ivas_masa_dec.c +++ b/lib_dec/ivas_masa_dec.c @@ -622,7 +622,15 @@ ivas_error ivas_masa_dec_open( st_ivas->hMasa = hMasa; /* allocate transport channels*/ - if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL && st_ivas->renderer_type != RENDERER_DISABLE && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms && +#endif +#else + st_ivas->hDecoderConfig->voip_active == 1 && +#endif + st_ivas->hTcBuffer == NULL && st_ivas->renderer_type != RENDERER_DISABLE && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC ) { int16_t nchan_to_allocate; TC_BUFFER_MODE buffer_mode; @@ -1231,7 +1239,12 @@ static int16_t decode_lfe_to_total_energy_ratio( ivas_error ivas_masa_dec_reconfigure( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ - int16_t *data /* o : flushed PCM samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : flushed PCM samples */ +#else + int16_t *data /* o : flushed PCM samples */ +#endif ) { int16_t n, tmp, num_bits; @@ -1251,7 +1264,11 @@ ivas_error ivas_masa_dec_reconfigure( ivas_total_brate = st_ivas->hDecoderConfig->ivas_total_brate; last_ivas_total_brate = st_ivas->hDecoderConfig->last_ivas_total_brate; +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_5ms == 1 ) +#else if ( st_ivas->hDecoderConfig->voip_active == 1 ) +#endif { if ( st_ivas->hSpatParamRendCom != NULL && st_ivas->hSpatParamRendCom->slot_size == st_ivas->hTcBuffer->n_samples_granularity ) { @@ -1444,7 +1461,13 @@ ivas_error ivas_masa_dec_reconfigure( st_ivas->ism_mode = ISM_MODE_NONE; } +#ifdef API_5MS +#ifdef API_5MS_BASELINE + if ( st_ivas->hDecoderConfig->Opt_5ms == 1 ) +#endif +#else if ( st_ivas->hDecoderConfig->voip_active == 1 ) +#endif { int16_t tc_nchan_to_allocate; int16_t tc_nchan_transport; @@ -1483,7 +1506,11 @@ ivas_error ivas_masa_dec_reconfigure( { if ( n_samples_granularity < st_ivas->hTcBuffer->n_samples_granularity ) { - if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, n_samples_granularity, st_ivas->renderer_type, st_ivas->intern_config, &st_ivas->hIntSetup, MC_MODE_NONE, ISM_MASA_MODE_DISC, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, n_samples_granularity, st_ivas->renderer_type, st_ivas->intern_config, &st_ivas->hIntSetup, MC_MODE_NONE, ISM_MASA_MODE_DISC, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } diff --git a/lib_dec/ivas_mc_param_dec.c b/lib_dec/ivas_mc_param_dec.c index 25663a1241907f01ce074e24534eb35ee2324ab2..d7a6136c1887a46cd11c29575231849a089315f5 100644 --- a/lib_dec/ivas_mc_param_dec.c +++ b/lib_dec/ivas_mc_param_dec.c @@ -459,8 +459,36 @@ ivas_error ivas_param_mc_dec_open( ivas_param_mc_dec_init( hParamMC, nchan_transport, nchan_out_cov ); - if ( st_ivas->hDecoderConfig->voip_active && hParamMC->synthesis_conf != PARAM_MC_SYNTH_MONO_STEREO ) + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms && +#endif +#else + st_ivas->hDecoderConfig->voip_active && +#endif + hParamMC->synthesis_conf != PARAM_MC_SYNTH_MONO_STEREO ) { +#ifdef API_5MS + int16_t n_cldfb_slots; + + n_cldfb_slots = DEFAULT_JBM_CLDFB_TIMESLOTS; + if ( st_ivas->hDecoderConfig->Opt_tsm ) + { + n_cldfb_slots = MAX_JBM_CLDFB_TIMESLOTS; + } + if ( ( hParamMC->Cldfb_RealBuffer_tc = (float *) malloc( n_cldfb_slots * nchan_transport * hParamMC->num_freq_bands * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Parametric MC JBM\n" ) ); + } + set_zero( hParamMC->Cldfb_RealBuffer_tc, n_cldfb_slots * nchan_transport * hParamMC->num_freq_bands ); + + if ( ( hParamMC->Cldfb_ImagBuffer_tc = (float *) malloc( n_cldfb_slots * nchan_transport * hParamMC->num_freq_bands * sizeof( float ) ) ) == NULL ) + { + return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Parametric MC JBM\n" ) ); + } + set_zero( hParamMC->Cldfb_ImagBuffer_tc, n_cldfb_slots * nchan_transport * hParamMC->num_freq_bands ); +#else if ( ( hParamMC->Cldfb_RealBuffer_tc = (float *) malloc( MAX_JBM_CLDFB_TIMESLOTS * nchan_transport * hParamMC->num_freq_bands * sizeof( float ) ) ) == NULL ) { return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Parametric MC JBM\n" ) ); @@ -472,7 +500,7 @@ ivas_error ivas_param_mc_dec_open( return ( IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for Parametric MC JBM\n" ) ); } set_zero( hParamMC->Cldfb_ImagBuffer_tc, MAX_JBM_CLDFB_TIMESLOTS * nchan_transport * hParamMC->num_freq_bands ); - +#endif if ( st_ivas->hTcBuffer == NULL ) { if ( ( error = ivas_jbm_dec_tc_buffer_open( st_ivas, TC_BUFFER_MODE_RENDERER, nchan_transport, nchan_transport, 0, NS2SA( st_ivas->hDecoderConfig->output_Fs, CLDFB_SLOT_NS ) ) ) != IVAS_ERR_OK ) @@ -1451,6 +1479,10 @@ void ivas_param_mc_dec_digest_tc( hParamMC->slots_rendered = 0; hParamMC->subframes_rendered = 0; ivas_jbm_dec_get_adapted_subframes( nCldfbSlots, hParamMC->subframe_nbslots, &hParamMC->nb_subframes ); +#ifdef API_5MS + st_ivas->hTcBuffer->nb_subframes = hParamMC->nb_subframes; + mvs2s( hParamMC->subframe_nbslots, st_ivas->hTcBuffer->subframe_nbslots, hParamMC->nb_subframes ); +#endif ivas_param_mc_dec_compute_interpolator( hParamMC->hMetadataPMC->bAttackPresent, hParamMC->hMetadataPMC->attackIndex, nCldfbSlots, hParamMC->h_output_synthesis_params.interpolator ); @@ -1463,18 +1495,28 @@ void ivas_param_mc_dec_digest_tc( /* slot loop for gathering the input data */ for ( slot_idx = 0; slot_idx < nCldfbSlots; slot_idx++ ) { - float RealBuffer[CLDFB_NO_CHANNELS_MAX]; - float ImagBuffer[CLDFB_NO_CHANNELS_MAX]; - - /* CLDFB Analysis*/ - for ( ch = 0; ch < nchan_transport; ch++ ) +#ifdef API_5MS + if ( st_ivas->hDecoderConfig->Opt_tsm +#ifdef API_5MS_BASELINE + || !st_ivas->hDecoderConfig->Opt_5ms +#endif + ) { - cldfbAnalysis_ts( &( transport_channels_f[ch][hParamMC->num_freq_bands * slot_idx] ), RealBuffer, ImagBuffer, hParamMC->num_freq_bands, st_ivas->cldfbAnaDec[ch] ); +#endif + float RealBuffer[CLDFB_NO_CHANNELS_MAX]; + float ImagBuffer[CLDFB_NO_CHANNELS_MAX]; - mvr2r( RealBuffer, &hParamMC->Cldfb_RealBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport + ch * hParamMC->num_freq_bands], hParamMC->num_freq_bands ); - mvr2r( ImagBuffer, &hParamMC->Cldfb_ImagBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport + ch * hParamMC->num_freq_bands], hParamMC->num_freq_bands ); - } + /* CLDFB Analysis*/ + for ( ch = 0; ch < nchan_transport; ch++ ) + { + cldfbAnalysis_ts( &( transport_channels_f[ch][hParamMC->num_freq_bands * slot_idx] ), RealBuffer, ImagBuffer, hParamMC->num_freq_bands, st_ivas->cldfbAnaDec[ch] ); + mvr2r( RealBuffer, &hParamMC->Cldfb_RealBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport + ch * hParamMC->num_freq_bands], hParamMC->num_freq_bands ); + mvr2r( ImagBuffer, &hParamMC->Cldfb_ImagBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport + ch * hParamMC->num_freq_bands], hParamMC->num_freq_bands ); + } +#ifdef API_5MS + } +#endif if ( slot_idx >= 2 * hParamMC->hMetadataPMC->attackIndex ) { ivas_dirac_dec_output_synthesis_cov_param_mc_collect_slot( &hParamMC->Cldfb_RealBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport], &hParamMC->Cldfb_ImagBuffer_tc[slot_idx * hParamMC->num_freq_bands * nchan_transport], cx, cx_imag, hParamMC, nchan_transport ); @@ -1536,11 +1578,11 @@ void ivas_param_mc_dec_digest_tc( *------------------------------------------------------------------------*/ void ivas_param_mc_dec_render( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ - uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ - uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ - float *output_f[] /* o : rendered time signal */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailableNext, /* o : number of CLDFB slots still to render */ + float *output_f[] /* o : rendered time signal */ ) { PARAM_MC_DEC_HANDLE hParamMC; @@ -1758,9 +1800,11 @@ void ivas_param_mc_dec_render( #ifdef SPLIT_REND_WITH_HEAD_ROT &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, #endif - st_ivas->hCombinedOrientationData, subframe_idx, + st_ivas->hCombinedOrientationData, +#ifndef API_5MS + subframe_idx, +#endif hParamMC->subframe_nbslots[subframe_idx], - #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG NULL, #endif @@ -1842,7 +1886,7 @@ void ivas_param_mc_dec_render( param_mc_update_mixing_matrices( hParamMC, hParamMC->h_output_synthesis_cov_state.mixing_matrix, hParamMC->h_output_synthesis_cov_state.mixing_matrix_res, nchan_transport, nchan_out_cov ); } hParamMC->subframes_rendered = last_sf; - *nSamplesAvailable = ( hParamMC->num_slots - hParamMC->slots_rendered ) * NS2SA( output_Fs, CLDFB_SLOT_NS ); + *nSamplesAvailableNext = ( hParamMC->num_slots - hParamMC->slots_rendered ) * NS2SA( output_Fs, CLDFB_SLOT_NS ); pop_wmops(); return; @@ -1863,7 +1907,7 @@ void ivas_param_mc_dec( PARAM_MC_DEC_HANDLE hParamMC; float Cldfb_RealBuffer_in[PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_NSLOTS * CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_in[PARAM_MC_MAX_TRANSPORT_CHANS * PARAM_MC_MAX_NSLOTS * CLDFB_NO_CHANNELS_MAX]; - uint16_t nSamplesAsked, nSamplesAvailable, nSamplesRendered; + uint16_t nSamplesAsked, nSamplesAvailableNext, nSamplesRendered; hParamMC = st_ivas->hParamMC; assert( hParamMC ); @@ -1875,10 +1919,10 @@ void ivas_param_mc_dec( nSamplesAsked = (int16_t) ( st_ivas->hDecoderConfig->output_Fs / FRAMES_PER_SEC ); ivas_param_mc_dec_digest_tc( st_ivas, DEFAULT_JBM_CLDFB_TIMESLOTS, output_f ); - ivas_param_mc_dec_render( st_ivas, nSamplesAsked, &nSamplesRendered, &nSamplesAvailable, output_f ); + ivas_param_mc_dec_render( st_ivas, nSamplesAsked, &nSamplesRendered, &nSamplesAvailableNext, output_f ); #ifdef DEBUGGING assert( nSamplesRendered == nSamplesAsked ); - assert( nSamplesAvailable == 0 ); + assert( nSamplesAvailableNext == 0 ); #endif /* set handle pointers back to NULL */ diff --git a/lib_dec/ivas_mc_paramupmix_dec.c b/lib_dec/ivas_mc_paramupmix_dec.c index bd0be4b14d750bac02885f2d97fb81ad2b527df3..563ed4d2e4cdd1451e43f7b345f437d593472017 100644 --- a/lib_dec/ivas_mc_paramupmix_dec.c +++ b/lib_dec/ivas_mc_paramupmix_dec.c @@ -353,7 +353,10 @@ void ivas_mc_paramupmix_dec( &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, #endif st_ivas->hCombinedOrientationData, - subframeIdx, JBM_CLDFB_SLOTS_IN_SUBFRAME, +#ifndef API_5MS + subframeIdx, /* TODO (5ms) : tmu2Dlb please verify */ +#endif + JBM_CLDFB_SLOTS_IN_SUBFRAME, #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG NULL, #endif @@ -661,7 +664,15 @@ ivas_error ivas_mc_paramupmix_dec_open( /* allocate transport channels*/ hMCParamUpmix->free_param_interpolator = 0; hMCParamUpmix->param_interpolator = NULL; - if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms == 1 && +#endif +#else + st_ivas->hDecoderConfig->voip_active == 1 && +#endif + st_ivas->hTcBuffer == NULL ) { int16_t nchan_to_allocate; int16_t nchan_tc; @@ -1142,7 +1153,10 @@ static void ivas_mc_paramupmix_dec_sf( &st_ivas->hSplitBinRend.splitrend.multiBinPoseData, #endif st_ivas->hCombinedOrientationData, - subframeIdx, st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], +#ifndef API_5MS + subframeIdx, /* TODO (5ms) : tmu2Dlb please verify */ +#endif + st_ivas->hTcBuffer->subframe_nbslots[subframeIdx], Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, Cldfb_RealBuffer_subfr, Cldfb_ImagBuffer_subfr ); diff --git a/lib_dec/ivas_mct_dec.c b/lib_dec/ivas_mct_dec.c index c9c372072fb01820261beba6ba17baade76c0a6f..02710355267aa6766922213c9c37852eac30ed19 100644 --- a/lib_dec/ivas_mct_dec.c +++ b/lib_dec/ivas_mct_dec.c @@ -54,7 +54,14 @@ * Local function prototypes *-----------------------------------------------------------------------*/ -static ivas_error ivas_mc_dec_reconfig( Decoder_Struct *st_ivas, uint16_t *nSamplesRendered, int16_t *data ); +static ivas_error ivas_mc_dec_reconfig( Decoder_Struct *st_ivas, uint16_t *nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif +); /*--------------------------------------------------------------------------* * ivas_mct_dec() @@ -627,7 +634,12 @@ ivas_error ivas_mc_dec_config( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ const int16_t idx, /* i : LS config. index */ uint16_t *nSamplesRendered, /* o : samples flushed from last frame (JBM) */ - int16_t *data /* o : flushed samples (JBM) */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { AUDIO_CONFIG signaled_config; @@ -657,7 +669,11 @@ ivas_error ivas_mc_dec_config( { if ( st_ivas->hDecoderConfig->last_ivas_total_brate != st_ivas->hDecoderConfig->ivas_total_brate || st_ivas->transport_config != signaled_config || last_mc_mode != st_ivas->mc_mode ) { - ivas_mc_dec_reconfig( st_ivas, nSamplesRendered, data ); + ivas_mc_dec_reconfig( st_ivas, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ); } } @@ -677,7 +693,12 @@ ivas_error ivas_mc_dec_config( static ivas_error ivas_mc_dec_reconfig( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the last frame (JBM) */ - int16_t *data /* o : flushed samples (JBM) */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { int16_t nchan_transport_old, nSCE_old, nCPE_old, sba_dirac_stereo_flag_old, nchan_hp20_old; @@ -734,8 +755,13 @@ static ivas_error ivas_mc_dec_reconfig( /* side effect of the renderer selection can be a changed internal config */ ivas_output_init( &( st_ivas->hIntSetup ), st_ivas->intern_config ); - +#ifdef API_5MS_BASELINE +#ifndef API_5MS if ( st_ivas->hDecoderConfig->voip_active ) +#else + if ( st_ivas->hDecoderConfig->Opt_5ms ) +#endif +#endif { /* transfer subframe info from DirAC or ParamMC to central tc buffer */ if ( last_mc_mode == MC_MODE_PARAMMC ) @@ -760,7 +786,11 @@ static ivas_error ivas_mc_dec_reconfig( tc_granularity_new = ivas_jbm_dec_get_render_granularity( st_ivas->renderer_type, st_ivas->ivas_format, st_ivas->mc_mode, st_ivas->hDecoderConfig->output_Fs ); if ( tc_granularity_new < st_ivas->hTcBuffer->n_samples_granularity ) { - if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, tc_granularity_new, renderer_type_old, intern_config_old, &hIntSetupOld, last_mc_mode, ISM_MODE_NONE, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_jbm_dec_flush_renderer( st_ivas, tc_granularity_new, renderer_type_old, intern_config_old, &hIntSetupOld, last_mc_mode, ISM_MODE_NONE, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -1278,8 +1308,13 @@ static ivas_error ivas_mc_dec_reconfig( /*-----------------------------------------------------------------* * Reconfigure TC buffer *-----------------------------------------------------------------*/ - +#ifdef API_5MS +#ifdef API_5MS_BASELINE + if ( st_ivas->hDecoderConfig->Opt_5ms == 1 ) +#endif +#else if ( st_ivas->hDecoderConfig->voip_active == 1 ) +#endif { int16_t tc_nchan_full_new; DECODER_TC_BUFFER_HANDLE hTcBuffer; diff --git a/lib_dec/ivas_objectRenderer_internal.c b/lib_dec/ivas_objectRenderer_internal.c index eb04843b3ded0f655694d428592978b152a1ed62..4cdd9181d2f0cb2e2eddef4eddde6e79482e93b6 100644 --- a/lib_dec/ivas_objectRenderer_internal.c +++ b/lib_dec/ivas_objectRenderer_internal.c @@ -107,10 +107,21 @@ ivas_error ivas_td_binaural_renderer( nchan_transport, LFE_CHANNEL, st_ivas->ivas_format, st_ivas->hIsmMetaData, +#ifdef API_5MS + ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->enableCombinedOrientation : NULL, + ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->Quaternion : NULL, + ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->listenerPos : NULL, +#else ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->enableCombinedOrientation : NULL, ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->Quaternions : NULL, ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->listenerPos : NULL, - ism_md_subframe_update, output, output_frame ); +#endif + ism_md_subframe_update, output, output_frame +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } @@ -219,10 +230,17 @@ ivas_error ivas_td_binaural_renderer_sf( } /* Update the listener's location/orientation */ +#ifdef API_5MS + TDREND_Update_listener_orientation( st_ivas->hBinRendererTd, + ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->enableCombinedOrientation : 0, + ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->Quaternion : NULL, + ( st_ivas->hCombinedOrientationData != NULL ) ? &st_ivas->hCombinedOrientationData->listenerPos : NULL ); +#else TDREND_Update_listener_orientation( st_ivas->hBinRendererTd, ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->enableCombinedOrientation[subframe_idx] : 0, ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->Quaternions : NULL, ( st_ivas->hCombinedOrientationData != NULL ) ? st_ivas->hCombinedOrientationData->listenerPos : NULL ); +#endif if ( st_ivas->hRenderConfig != NULL && st_ivas->hIntSetup.output_config == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) { @@ -233,7 +251,7 @@ ivas_error ivas_td_binaural_renderer_sf( } /* Render subframe */ - if ( ( error = TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, ism_md_subframe_update_jbm ) ) != IVAS_ERR_OK ) + if ( ( error = TDREND_GetMix( st_ivas->hBinRendererTd, output_f_local, output_frame, 0, ism_md_subframe_update_jbm != subframe_idx ) ) != IVAS_ERR_OK ) { return error; } diff --git a/lib_dec/ivas_omasa_dec.c b/lib_dec/ivas_omasa_dec.c index de4a310d5a93614c3963cc2ab8b0eff7d9c7cee9..06a7e8ead384b2aa86872e624efaa6e3ddb9ea58 100644 --- a/lib_dec/ivas_omasa_dec.c +++ b/lib_dec/ivas_omasa_dec.c @@ -154,7 +154,12 @@ void ivas_omasa_data_close( ivas_error ivas_omasa_dec_config( Decoder_Struct *st_ivas, /* i/o: IVAS decoder structure */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the previous frame (JBM) */ - int16_t *data /* o : flushed PCM samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { int16_t k, sce_id, nSCE_old, nchan_hp20_old, numCldfbAnalyses_old, numCldfbSyntheses_old, n_MD; @@ -197,7 +202,11 @@ ivas_error ivas_omasa_dec_config( { st_ivas->hCPE[0]->nchan_out = 1; } - else if ( ( error = ivas_masa_dec_reconfigure( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + else if ( ( error = ivas_masa_dec_reconfigure( st_ivas, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -403,7 +412,11 @@ ivas_error ivas_omasa_dec_config( * floating-point output audio buffers *-----------------------------------------------------------------*/ +#ifdef API_5MS + if ( !st_ivas->hDecoderConfig->Opt_5ms ) +#else if ( !st_ivas->hDecoderConfig->voip_active ) +#endif { nchan_out_buff = ivas_get_nchan_buffers_dec( st_ivas ); @@ -624,7 +637,12 @@ void ivas_omasa_dirac_rend( dirac_read_idx = st_ivas->hSpatParamRendCom->dirac_read_idx; - ivas_dirac_dec( st_ivas, output, st_ivas->nchan_transport ); + ivas_dirac_dec( st_ivas, output, st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); st_ivas->hSpatParamRendCom->dirac_read_idx = dirac_read_idx; /* Original read index is needed for the next function which will update it again */ @@ -697,7 +715,12 @@ ivas_error ivas_omasa_dirac_td_binaural( delay_signal( data_separated_objects[n], output_frame, st_ivas->hMasaIsmData->delayBuffer[n], st_ivas->hMasaIsmData->delayBuffer_size ); } - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, output, st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); if ( ( error = ivas_td_binaural_renderer( st_ivas, p_sepobj, output_frame ) ) != IVAS_ERR_OK ) { diff --git a/lib_dec/ivas_osba_dec.c b/lib_dec/ivas_osba_dec.c index 47ef0efbe0fab3cdb958c0ca55259d38534f0298..d162ad6c632f6765daf066b1619aa919b953f2a6 100644 --- a/lib_dec/ivas_osba_dec.c +++ b/lib_dec/ivas_osba_dec.c @@ -183,7 +183,12 @@ ivas_error ivas_osba_dirac_td_binaural( } else { - ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, &output[channel_offset], st_ivas->nchan_transport ); + ivas_dirac_dec_binaural( st_ivas, st_ivas->hCombinedOrientationData, &output[channel_offset], st_ivas->nchan_transport +#ifdef API_5MS + , + MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); } #ifdef DEBUG_OSBA diff --git a/lib_dec/ivas_sba_dec.c b/lib_dec/ivas_sba_dec.c index 5a76aa885db77b4f87a01a340a58b1e4a8f319d6..500b95f5e82dafb8420d01d405d59935c82fd1b7 100755 --- a/lib_dec/ivas_sba_dec.c +++ b/lib_dec/ivas_sba_dec.c @@ -465,8 +465,13 @@ ivas_error ivas_sba_dec_reconfigure( /*-----------------------------------------------------------------* * JBM TC buffer *-----------------------------------------------------------------*/ - +#ifdef API_5MS +#ifdef API_5MS_BASELINE + if ( st_ivas->hDecoderConfig->Opt_5ms == 1 ) +#endif +#else if ( st_ivas->hDecoderConfig->voip_active == 1 ) +#endif { int16_t tc_nchan_to_allocate; int16_t tc_nchan_tc; @@ -510,7 +515,11 @@ ivas_error ivas_sba_dec_reconfigure( * floating-point output audio buffers *-----------------------------------------------------------------*/ +#ifdef API_5MS + if ( !st_ivas->hDecoderConfig->Opt_5ms ) +#else if ( !st_ivas->hDecoderConfig->voip_active ) +#endif { nchan_out_buff = ivas_get_nchan_buffers_dec( st_ivas ); @@ -629,11 +638,11 @@ ivas_error ivas_sba_dec_digest_tc( *-------------------------------------------------------------------*/ void ivas_sba_dec_render( - Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ - const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ - uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ - uint16_t *nSamplesAvailable, /* o : number of CLDFB slots still to render */ - float *output_f[] /* o : rendered time signal */ + Decoder_Struct *st_ivas, /* i/o: IVAS decoder handle */ + const uint16_t nSamplesAsked, /* i : number of CLDFB slots requested */ + uint16_t *nSamplesRendered, /* o : number of CLDFB slots rendered */ + uint16_t *nSamplesAvailableNext, /* o : number of CLDFB slots still to render */ + float *output_f[] /* o : rendered time signal */ ) { int16_t slots_to_render, first_sf, last_sf, subframe_idx; @@ -703,7 +712,7 @@ void ivas_sba_dec_render( } } - *nSamplesAvailable = ( hSpar->num_slots - hSpar->slots_rendered ) * slot_size; + *nSamplesAvailableNext = ( hSpar->num_slots - hSpar->slots_rendered ) * slot_size; return; } diff --git a/lib_dec/ivas_spar_decoder.c b/lib_dec/ivas_spar_decoder.c index f5131535a5166116dd18eb9c3fc38f78b64dad22..d38953212b403a90ee9d3be9558eee69d69c5da4 100644 --- a/lib_dec/ivas_spar_decoder.c +++ b/lib_dec/ivas_spar_decoder.c @@ -216,7 +216,15 @@ ivas_error ivas_spar_dec_open( } /* allocate transport channels*/ - if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms && +#endif +#else + st_ivas->hDecoderConfig->voip_active == 1 && +#endif + st_ivas->hTcBuffer == NULL ) { int16_t nchan_to_allocate; int16_t nchan_tc; @@ -1193,6 +1201,10 @@ void ivas_spar_dec_set_render_map( hSpar->subframes_rendered = 0; set_s( hSpar->render_to_md_map, 0, MAX_JBM_SUBFRAMES_5MS * JBM_CLDFB_SLOTS_IN_SUBFRAME ); ivas_jbm_dec_get_adapted_subframes( nCldfbTs, hSpar->subframe_nbslots, &hSpar->nb_subframes ); +#ifdef API_5MS + st_ivas->hTcBuffer->nb_subframes = hSpar->nb_subframes; + mvs2s( hSpar->subframe_nbslots, st_ivas->hTcBuffer->subframe_nbslots, hSpar->nb_subframes ); +#endif ivas_jbm_dec_get_md_map( DEFAULT_JBM_CLDFB_TIMESLOTS, nCldfbTs, 1, 0, DEFAULT_JBM_CLDFB_TIMESLOTS, hSpar->render_to_md_map ); return; diff --git a/lib_dec/ivas_stat_dec.h b/lib_dec/ivas_stat_dec.h index d6167f57085c5e5ec20a7a3020413b2d7ef82fcd..c05a47a49f3e262aaf7e12860cb6d6172fb22ed0 100755 --- a/lib_dec/ivas_stat_dec.h +++ b/lib_dec/ivas_stat_dec.h @@ -908,7 +908,17 @@ typedef struct decoder_config_structure #ifdef DEBUGGING int16_t force_rend; /* forced TD/CLDFB binaural renderer (for ISM and MC) */ #endif +#ifdef API_5MS + int16_t Opt_tsm; +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t Opt_Limiter; +#endif +#ifdef API_5MS_BASELINE + int16_t Opt_5ms; +#endif +#else int16_t voip_active; +#endif int16_t Opt_delay_comp; /* flag indicating delay compensation active */ } DECODER_CONFIG, *DECODER_CONFIG_HANDLE; @@ -935,6 +945,11 @@ typedef struct IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits; /*scratch buffer for frame by frame processing*/ SPLIT_REND_WRAPPER splitrend; IVAS_DEC_SPLIT_REND_CLDFB_OUT_DATA_HANDLE hCldfbDataOut; /*buffer to store cldfb data before binauralization*/ +#ifdef API_5MS + float *tdDataOut; /*buffer to store TD data before binauralization*/ + int16_t numTdSamplesPerChannelCached; + int16_t numSamplesCollected; +#endif } IVAS_DEC_SPLIT_REND_WRAPPER; #endif diff --git a/lib_dec/jbm_jb4sb.h b/lib_dec/jbm_jb4sb.h index 00f5ccbb4078ab61d69873fb1cd6c3e25f5128a6..599730975da23f620c190baeafe7062160192ef0 100644 --- a/lib_dec/jbm_jb4sb.h +++ b/lib_dec/jbm_jb4sb.h @@ -77,13 +77,13 @@ struct JB4_DATAUNIT int16_t partialCopyOffset; int16_t nextCoderType; }; - +#ifndef API_5MS typedef enum { JBM_RENDERER_NONE, JBM_RENDERER_IVAS, } JBM_RENDERER_TYPE; - +#endif typedef struct JB4_DATAUNIT *JB4_DATAUNIT_HANDLE; diff --git a/lib_dec/jbm_pcmdsp_fifo.c b/lib_dec/jbm_pcmdsp_fifo.c index a3e893646442922769dcbccb395f3dd9d21f8c4f..ad0a860944623ba10822f7c8565d79c17c378408 100644 --- a/lib_dec/jbm_pcmdsp_fifo.c +++ b/lib_dec/jbm_pcmdsp_fifo.c @@ -38,6 +38,7 @@ #include #include "options.h" +#ifndef API_5MS #include "prot.h" #include "ivas_prot.h" #ifdef DEBUGGING @@ -267,3 +268,4 @@ uint16_t pcmdsp_fifo_nReadableSamplesPerChannel( { return h->size; } +#endif diff --git a/lib_dec/jbm_pcmdsp_fifo.h b/lib_dec/jbm_pcmdsp_fifo.h index b601cc2e0e0001aba14c562a3b36f3447e124e27..62ebc15d08b46bb812845743f3da13461a196576 100644 --- a/lib_dec/jbm_pcmdsp_fifo.h +++ b/lib_dec/jbm_pcmdsp_fifo.h @@ -41,7 +41,7 @@ #include #include "options.h" - +#ifndef API_5MS /** Ringbuffer (FIFO) with fixed capacity for audio samples. */ struct PCMDSP_FIFO @@ -81,5 +81,5 @@ int16_t pcmdsp_fifo_write_zero( PCMDSP_FIFO_HANDLE h, uint16_t nSamplesPerChanne int16_t pcmdsp_fifo_read( PCMDSP_FIFO_HANDLE h, uint16_t nSamplesPerChannel, uint8_t *samples ); uint16_t pcmdsp_fifo_nReadableSamplesPerChannel( const PCMDSP_FIFO_HANDLE h ); - +#endif #endif /* JBM_PCMDSP_FIFO_H */ diff --git a/lib_dec/lib_dec.c b/lib_dec/lib_dec.c index f790be3f0b83f16ae2a92c1f886b2c2411f7ef37..554cbad5bec6a53ce7228f4a1ac6de2a01874cfa 100644 --- a/lib_dec/lib_dec.c +++ b/lib_dec/lib_dec.c @@ -31,6 +31,7 @@ *******************************************************************************************************/ #include "lib_dec.h" +#include "ivas_cnst.h" #include "ivas_prot.h" #include "ivas_prot_rend.h" #include "prot.h" @@ -53,11 +54,16 @@ struct IVAS_DEC_VOIP { uint16_t nSamplesFrame; /* Total number of samples in a frame (includes number of channels) */ JB4_HANDLE hJBM; +#ifndef API_5MS PCMDSP_APA_HANDLE hTimeScaler; +#endif uint16_t lastDecodedWasActive; - float *apaExecBuffer; /* Buffer for APA scaling */ +#ifndef API_5MS + float *apaExecBuffer; /* Buffer for APA scaling */ +#endif JB4_DATAUNIT_HANDLE hCurrentDataUnit; /* Points to the currently processed data unit */ uint16_t *bs_conversion_buf; /* Buffer for bitstream conversion from packed to serial */ +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING IVAS_DEC_VOIP_MODE voipMode; uint16_t speedFac; @@ -67,6 +73,7 @@ struct IVAS_DEC_VOIP PCMDSP_FIFO_HANDLE hFifoOut; uint8_t nTransportChannelsOld; uint16_t nSamplesAvailableNext; +#endif #ifdef SUPPORT_JBM_TRACEFILE IVAS_JBM_TRACE_DATA JbmTraceData; #endif @@ -84,8 +91,19 @@ struct IVAS_DEC bool hasDecodedFirstGoodFrame; /* False on init. Gets set to true after first good frame has been decoded -> all bitstream information is known from that point on */ bool isInitialized; - int16_t bitstreamformat; /* Bitstream format flag (G.192/MIME/VOIP_G192_RTP/VOIP_RTPDUMP) */ - bool Opt_VOIP; /* flag indicating VOIP mode with JBM */ + int16_t bitstreamformat; /* Bitstream format flag (G.192/MIME/VOIP_G192_RTP/VOIP_RTPDUMP) */ + bool Opt_VOIP; /* flag indicating VOIP mode with JBM */ +#ifdef API_5MS + int16_t tsm_scale; /* scale for TSM operation */ + int16_t tsm_max_scaling; + float *apaExecBuffer; /* Buffer for APA scaling */ + PCMDSP_APA_HANDLE hTimeScaler; + bool needNewFrame; + bool hasBeenFedFrame; + uint16_t nSamplesAvailableNext; + int16_t nSamplesRendered; + int16_t nTransportChannelsOld; +#endif int16_t amrwb_rfc4867_flag; /* MIME from rfc4867 is used */ int16_t sdp_hf_only; /* RTP payload format parameter: only Header-Full format without zero padding for size collision avoidance */ int16_t prev_ft_speech; /* RXDTX handler: previous frametype flag for G.192 format AMRWB SID_FIRST detection */ @@ -104,9 +122,36 @@ static void store_JbmData( IVAS_DEC_VOIP *hVoIP, JB4_DATAUNIT_HANDLE dataUnit, c static ivas_error evs_dec_main( Decoder_Struct *st_ivas, const int16_t nOutSamples, float *floatBuf, int16_t *pcmBuf ); static ivas_error input_format_API_to_internal( IVAS_DEC_INPUT_FORMAT input_format, int16_t *bitstream_format_internal, int16_t *sdp_hf_only, const bool is_voip_enabled ); static void init_decoder_config( DECODER_CONFIG_HANDLE hDecoderConfig ); +static int16_t IVAS_DEC_VoIP_GetRenderGranularity( Decoder_Struct *st_ivas ); +#ifndef API_5MS +static JBM_RENDERER_TYPE IVAS_DEC_VoIP_GetRendererConfig( IVAS_DEC_HANDLE hIvasDec ); +#endif static ivas_error IVAS_DEC_VoIP_reconfigure( IVAS_DEC_HANDLE hIvasDec, const uint16_t nTransportChannels, const uint16_t l_ts ); +static ivas_error IVAS_DEC_Setup( IVAS_DEC_HANDLE hIvasDec, uint16_t *nTcBufferGranularity, uint8_t *nTransportChannels, uint8_t *nOutChannels, uint16_t *nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const IVAS_DEC_PCM_TYPE pcmType, + void *data +#else + int16_t *data +#endif +); +static ivas_error IVAS_DEC_GetTcSamples( IVAS_DEC_HANDLE hIvasDec, float *pcmBuf, int16_t *nOutSamples ); +static ivas_error IVAS_DEC_RendererFeedTcSamples( IVAS_DEC_HANDLE hIvasDec, const int16_t nSamplesForRendering, int16_t *nSamplesResidual, float *pcmBuf ); +static ivas_error IVAS_DEC_GetRenderedSamples( IVAS_DEC_HANDLE hIvasDec, const uint16_t nSamplesForRendering, uint16_t *nSamplesRendered, uint16_t *nSamplesAvailableNext, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_TYPE pcmType, + void *pcmBuf +#else - + int16_t *pcmBuf +#endif +); +static ivas_error IVAS_DEC_GetBufferedNumberOfSamples( IVAS_DEC_HANDLE hIvasDec, int16_t *nSamplesBuffered ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +static PCM_RESOLUTION pcm_type_API_to_internal( const IVAS_DEC_PCM_TYPE pcmType ); +static void *pcm_buffer_offset( void *buffer, const IVAS_DEC_PCM_TYPE pcmType, int32_t offset ); +static ivas_error set_pcm_buffer_to_zero( void *buffer, const IVAS_DEC_PCM_TYPE pcmType, int16_t nZeroSamples ); +#endif /*---------------------------------------------------------------------* * IVAS_DEC_Open() * @@ -137,6 +182,17 @@ ivas_error IVAS_DEC_Open( } hIvasDec = *phIvasDec; hIvasDec->hVoIP = NULL; +#ifdef API_5MS + hIvasDec->apaExecBuffer = NULL; + hIvasDec->hTimeScaler = NULL; + hIvasDec->tsm_scale = 100; + hIvasDec->needNewFrame = false; + hIvasDec->nTransportChannelsOld = 0; + hIvasDec->nSamplesAvailableNext = 0; + hIvasDec->nSamplesRendered = 0; + hIvasDec->nSamplesFrame = 0; + hIvasDec->hasBeenFedFrame = false; +#endif hIvasDec->hasBeenFedFirstGoodFrame = false; hIvasDec->hasDecodedFirstGoodFrame = false; hIvasDec->isInitialized = false; @@ -234,8 +290,17 @@ static void init_decoder_config( hDecoderConfig->orientation_tracking = HEAD_ORIENT_TRK_NONE; hDecoderConfig->Opt_non_diegetic_pan = 0; hDecoderConfig->non_diegetic_pan_gain = 0; - +#ifdef API_5MS + hDecoderConfig->Opt_tsm = 0; +#ifdef SPLIT_REND_WITH_HEAD_ROT + hDecoderConfig->Opt_Limiter = 1; +#endif +#ifdef API_5MS_BASELINE + hDecoderConfig->Opt_5ms = 0; +#endif +#else hDecoderConfig->voip_active = 0; +#endif hDecoderConfig->Opt_delay_comp = 0; @@ -273,6 +338,14 @@ void IVAS_DEC_Close( ( *phIvasDec )->st_ivas = NULL; } +#ifdef API_5MS + apa_exit( &( *phIvasDec )->hTimeScaler ); + + if ( ( *phIvasDec )->apaExecBuffer != NULL ) + { + free( ( *phIvasDec )->apaExecBuffer ); + } +#endif free( *phIvasDec ); *phIvasDec = NULL; phIvasDec = NULL; @@ -324,9 +397,15 @@ static IVAS_DEC_BS_FORMAT mapIvasFormat( *---------------------------------------------------------------------*/ ivas_error IVAS_DEC_Configure( - IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ - const uint32_t sampleRate, /* i : output sampling frequency */ - const AUDIO_CONFIG outputConfig, /* i : output configuration */ + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + const uint32_t sampleRate, /* i : output sampling frequency */ + const AUDIO_CONFIG outputConfig, /* i : output configuration */ +#ifdef API_5MS + const int16_t tsmEnabled, /* i : enable TSM */ +#ifdef API_5MS_BASELINE + const int16_t enable5ms, +#endif +#endif const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ @@ -403,10 +482,79 @@ ivas_error IVAS_DEC_Configure( hIvasDec->st_ivas->ivas_format = MONO_FORMAT; } +#ifdef API_5MS + hDecoderConfig->Opt_tsm = tsmEnabled; +#ifdef API_5MS_BASELINE + hDecoderConfig->Opt_5ms = enable5ms; +#endif + hIvasDec->nSamplesFrame = (uint16_t) ( hDecoderConfig->output_Fs / FRAMES_PER_SEC ); + hIvasDec->nSamplesAvailableNext = 0; + hIvasDec->nSamplesRendered = 0; + hIvasDec->tsm_scale = 100; + hIvasDec->tsm_max_scaling = 100; +#endif + + return error; +} + +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) +/*---------------------------------------------------------------------* + * IVAS_DEC_EnableSplitRendering( ) + * + * Intitialize Split rendering + *---------------------------------------------------------------------*/ + +ivas_error IVAS_DEC_EnableSplitRendering( + IVAS_DEC_HANDLE hIvasDec /* i/o: IVAS decoder handle */ +) +{ + DECODER_CONFIG_HANDLE hDecoderConfig; + ivas_error error; + + error = IVAS_ERR_OK; + + + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + hDecoderConfig = hIvasDec->st_ivas->hDecoderConfig; + + hDecoderConfig->Opt_Headrotation = 1; +#if 0 // def API_5MS_BASELINE + hDecoderConfig->Opt_5ms = false; +#endif + + + hDecoderConfig->Opt_Limiter = 0; + return error; } +/*---------------------------------------------------------------------* + * IVAS_DEC_Get5msFlag( ) + * + * Get the 5ms flag + *---------------------------------------------------------------------*/ + +ivas_error IVAS_DEC_Get5msFlag( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + int16_t *enable5ms /* o : 5ms flag */ +) +{ + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL || enable5ms == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + *enable5ms = (int16_t) hIvasDec->st_ivas->hDecoderConfig->Opt_5ms; + + return IVAS_ERR_OK; +} +#endif + /*---------------------------------------------------------------------* * IVAS_DEC_EnableVoIP( ) * @@ -417,9 +565,11 @@ ivas_error IVAS_DEC_Configure( ivas_error IVAS_DEC_EnableVoIP( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING const IVAS_DEC_VOIP_MODE voipMode, /* i : VoIP or variable speed */ const uint16_t speedFac, /* i : speed factor for variable speed */ +#endif #endif const int16_t jbmSafetyMargin, /* i : allowed delay reserve for JBM, in milliseconds */ const IVAS_DEC_INPUT_FORMAT inputFormat /* i : format of the input bitstream */ @@ -439,7 +589,14 @@ ivas_error IVAS_DEC_EnableVoIP( hDecoderConfig = hIvasDec->st_ivas->hDecoderConfig; hIvasDec->Opt_VOIP = 1; +#ifdef API_5MS + hDecoderConfig->Opt_tsm = 1; +#ifdef API_5MS_BASELINE + hDecoderConfig->Opt_5ms = 1; +#endif +#else hDecoderConfig->voip_active = 1; +#endif if ( hDecoderConfig->output_config != AUDIO_CONFIG_EXTERNAL ) { @@ -465,12 +622,15 @@ ivas_error IVAS_DEC_EnableVoIP( hIvasDec->hVoIP->lastDecodedWasActive = 0; hIvasDec->hVoIP->hCurrentDataUnit = NULL; +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING hIvasDec->hVoIP->voipMode = voipMode; hIvasDec->hVoIP->speedFac = speedFac; hIvasDec->hVoIP->needNewFrame = false; +#endif #endif hIvasDec->hVoIP->nSamplesFrame = (uint16_t) ( hDecoderConfig->output_Fs / FRAMES_PER_SEC ); +#ifndef API_5MS hIvasDec->hVoIP->nSamplesAvailableNext = 0; hIvasDec->hVoIP->rendererType = JBM_RENDERER_NONE; hIvasDec->hVoIP->hFifoOut = NULL; @@ -478,6 +638,7 @@ ivas_error IVAS_DEC_EnableVoIP( /* postpone init of the buffers until we know the real number of TCs*/ hIvasDec->hVoIP->apaExecBuffer = NULL; hIvasDec->hVoIP->nTransportChannelsOld = 0; +#endif #define WMC_TOOL_SKIP /* Bitstream conversion is not counted towards complexity and memory usage */ @@ -490,10 +651,12 @@ ivas_error IVAS_DEC_EnableVoIP( } /* initialize JBM */ +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING hIvasDec->hVoIP->hJBM = NULL; if ( hIvasDec->hVoIP->voipMode == IVAS_DEC_VOIP_MODE_VOIP ) { +#endif #endif if ( ( error = JB4_Create( &hIvasDec->hVoIP->hJBM ) != IVAS_ERR_OK ) != IVAS_ERR_OK ) { @@ -503,10 +666,13 @@ ivas_error IVAS_DEC_EnableVoIP( { return IVAS_ERR_FAILED_ALLOC; } +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING } #endif +#endif +#ifndef API_5MS /* postpone init of time scaler and output FIFO until we know the real number of TCs */ hIvasDec->hVoIP->hTimeScaler = NULL; #ifdef VARIABLE_SPEED_DECODING @@ -515,7 +681,7 @@ ivas_error IVAS_DEC_EnableVoIP( hIvasDec->hVoIP->needNewFrame = true; } #endif - +#endif return error; } @@ -559,9 +725,13 @@ ivas_error IVAS_DEC_FeedFrame_Serial( { hIvasDec->st_ivas->hDecoderConfig->ivas_total_brate = ACELP_8k00; } +#ifdef API_5MS + hIvasDec->isInitialized = true; +#endif } - +#ifndef API_5MS hIvasDec->isInitialized = true; +#endif } if ( !bfi ) /* TODO(mcjbm): Is this ok for bfi == 2 (partial frame)? Is there enough info to fully configure decoder? */ @@ -603,11 +773,18 @@ ivas_error IVAS_DEC_FeedFrame_Serial( st->use_partial_copy = 1; } +#ifdef API_5MS + hIvasDec->needNewFrame = false; + hIvasDec->hasBeenFedFrame = true; + hIvasDec->nSamplesRendered = 0; + hIvasDec->nSamplesAvailableNext = hIvasDec->nSamplesFrame; +#else #ifdef VARIABLE_SPEED_DECODING if ( hIvasDec->hVoIP != NULL && hIvasDec->hVoIP->voipMode == IVAS_DEC_VOIP_MODE_VARIABLE_SPEED ) { hIvasDec->hVoIP->needNewFrame = false; } +#endif #endif return error; @@ -620,11 +797,21 @@ ivas_error IVAS_DEC_FeedFrame_Serial( * Main function to decode to PCM data *---------------------------------------------------------------------*/ +#if !defined API_5MS || defined( API_5MS_BASELINE ) +#ifdef API_5MS_BASELINE +static ivas_error _GetSamples( +#else ivas_error IVAS_DEC_GetSamples( +#endif IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ - int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ - int16_t *nOutSamples /* o : number of samples per channel written to output buffer */ -#ifdef SPLIT_REND_WITH_HEAD_ROT +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const PCM_RESOLUTION pcm_resolution, + void *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif + int16_t *nOutSamples /* o : number of samples per channel written to output buffer */ +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && !defined( API_5MS ) , uint8_t *splitRendBitsBuf /* o : output split rendering bits */ #endif @@ -646,7 +833,15 @@ ivas_error IVAS_DEC_GetSamples( if ( hIvasDec->mode == IVAS_DEC_MODE_EVS ) { - if ( ( error = evs_dec_main( st_ivas, *nOutSamples, NULL, pcmBuf ) ) != IVAS_ERR_OK ) +#if defined DEBUGGING && defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + assert( pcm_resolution == PCM_INT16 ); +#endif + + if ( ( error = evs_dec_main( st_ivas, *nOutSamples, NULL, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + (int16_t *) +#endif + pcmBuf ) ) != IVAS_ERR_OK ) { return error; } @@ -654,8 +849,13 @@ ivas_error IVAS_DEC_GetSamples( else if ( hIvasDec->mode == IVAS_DEC_MODE_IVAS ) { /* run the main IVAS decoding routine */ - if ( ( error = ivas_dec( st_ivas, pcmBuf -#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_dec( st_ivas, + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_resolution, +#endif + pcmBuf +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && !defined( API_5MS ) , splitRendBitsBuf #endif @@ -674,6 +874,445 @@ ivas_error IVAS_DEC_GetSamples( return error; } +#endif +#ifdef API_5MS +ivas_error IVAS_DEC_GetSamples( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + int16_t nSamplesAsked, /* i: number of samples wanted by the caller */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const IVAS_DEC_PCM_TYPE pcmType, /* i : type for the decoded PCM resolution */ + void *pcmBuf, /* o : output synthesis signal */ +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif + int16_t *nOutSamples, /* o : number of samples per channel written to output buffer */ + bool *needNewFrame /* indication that the decoder needs a new frame */ +#if defined SPLIT_REND_WITH_HEAD_ROT && !defined API_5MS + , + IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits /* o : bitstream output for split rendering mode*/ +#endif +) +{ + ivas_error error; + int16_t nOutSamplesElse, result, nSamplesToRender; + uint16_t nSamplesRendered, nSamplesRendered_loop, l_ts, nTimeScalerOutSamples; + uint8_t nTransportChannels, nOutChannels; + error = IVAS_ERR_OK; + nSamplesRendered = 0; + nOutChannels = 0; + +#ifdef API_5MS_BASELINE + nSamplesRendered_loop = 0; + l_ts = 0; + nTransportChannels = 0; +#endif + + + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + if ( !hIvasDec->hasBeenFedFrame && hIvasDec->nSamplesAvailableNext == 0 ) + { + /* no frame was fed, do nothing but ask for a frame */ + *needNewFrame = true; + *nOutSamples = 0; + hIvasDec->needNewFrame = true; + return error; + } + + /* check if we are still at the beginning with bad frames, put out zeroes, keep track of subframes */ + if ( !hIvasDec->isInitialized && hIvasDec->st_ivas->bfi ) + { + hIvasDec->hasBeenFedFrame = false; + set_s( pcmBuf, 0, hIvasDec->st_ivas->hDecoderConfig->nchan_out * nSamplesAsked ); + hIvasDec->nSamplesRendered += nSamplesAsked; + *nOutSamples = nSamplesAsked; + hIvasDec->nSamplesAvailableNext -= nSamplesAsked; + if ( hIvasDec->nSamplesAvailableNext == 0 ) + { + hIvasDec->needNewFrame = true; + *needNewFrame = true; + } + } +#ifdef API_5MS_BASELINE + + /* only for 1st step 5ms API, split rendering still needs to go through the old decoding function */ + else + { + /* check if we need to run the setup function */ + if ( !hIvasDec->isInitialized || hIvasDec->hasBeenFedFrame ) + { + /* setup */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + int16_t enable5ms_old = hIvasDec->st_ivas->hDecoderConfig->Opt_5ms; +#endif + if ( ( error = IVAS_DEC_Setup( hIvasDec, &l_ts, &nTransportChannels, &nOutChannels, &nSamplesRendered_loop, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcmType, + pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ) +#else + pcmBuf + nSamplesRendered * nOutChannels +#endif + + ) ) != IVAS_ERR_OK ) + { + return error; + } +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + /* :TODO: change nSamplesAsked also if we are in 5ms 0dof split rendering... */ + if ( enable5ms_old != hIvasDec->st_ivas->hDecoderConfig->Opt_5ms ) + { + if ( enable5ms_old ) + { + nSamplesAsked *= MAX_PARAM_SPATIAL_SUBFRAMES; + } + else + { + nSamplesAsked /= MAX_PARAM_SPATIAL_SUBFRAMES; + } + } +#endif + } + if ( !hIvasDec->st_ivas->hDecoderConfig->Opt_5ms ) + { + if ( ( error = _GetSamples( hIvasDec, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_type_API_to_internal( pcmType ), +#endif + pcmBuf, nOutSamples ) ) != IVAS_ERR_OK ) + { + return error; + } +#ifdef DEBUGGING + assert( *nOutSamples == nSamplesAsked ); +#endif + hIvasDec->nSamplesAvailableNext = 0; + hIvasDec->nSamplesRendered = *nOutSamples; + nSamplesRendered = *nOutSamples; + hIvasDec->needNewFrame = true; + hIvasDec->hasBeenFedFrame = false; + *needNewFrame = true; + } +#endif + else + { + /* check if we need to run the setup function, tc decoding and feeding the renderer */ + if ( !hIvasDec->isInitialized || hIvasDec->hasBeenFedFrame ) + { + int16_t nResidualSamples, nSamplesTcsScaled; +#ifndef API_5MS_BASELINE + /* setup */ + if ( ( error = IVAS_DEC_Setup( hIvasDec, &l_ts, &nTransportChannels, &nOutChannels, &nSamplesRendered_loop, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcmType, + pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ) +#else + pcmBuf + nSamplesRendered * nOutChannels +#endif + + ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif + nSamplesRendered += nSamplesRendered_loop; + if ( nTransportChannels != hIvasDec->nTransportChannelsOld ) + { + IVAS_DEC_VoIP_reconfigure( hIvasDec, nTransportChannels, l_ts ); + } + /* decode TCs only */ + if ( ( error = IVAS_DEC_GetTcSamples( hIvasDec, hIvasDec->apaExecBuffer, &nOutSamplesElse ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm ) + { + if ( apa_set_scale( hIvasDec->hTimeScaler, hIvasDec->tsm_scale ) != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + result = apa_exec( hIvasDec->hTimeScaler, hIvasDec->apaExecBuffer, hIvasDec->nSamplesFrame * nTransportChannels, (uint16_t) hIvasDec->tsm_max_scaling, hIvasDec->apaExecBuffer, &nTimeScalerOutSamples ); + if ( result != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + assert( nTimeScalerOutSamples <= APA_BUF ); + } + else + { + nTimeScalerOutSamples = hIvasDec->nSamplesFrame * nTransportChannels; + } + nSamplesTcsScaled = nTimeScalerOutSamples / nTransportChannels; + + /* render IVAS frames */ + + + if ( ( error = IVAS_DEC_RendererFeedTcSamples( hIvasDec, nSamplesTcsScaled, &nResidualSamples, hIvasDec->apaExecBuffer ) ) != IVAS_ERR_OK ) + { + return error; + } + + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm ) + { + /* feed residual samples to TSM for the next call */ + if ( apa_set_renderer_residual_samples( hIvasDec->hTimeScaler, (uint16_t) nResidualSamples ) != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + } + hIvasDec->hasBeenFedFrame = false; + } + /* render IVAS frames directly to the output buffer */ + nSamplesToRender = nSamplesAsked - nSamplesRendered; + if ( ( error = IVAS_DEC_GetRenderedSamples( hIvasDec, nSamplesToRender, &nSamplesRendered_loop, &hIvasDec->nSamplesAvailableNext, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcmType, + pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ) +#else + pcmBuf + nSamplesRendered * nOutChannels +#endif + + ) ) != IVAS_ERR_OK ) + { + return error; + } + nSamplesRendered += nSamplesRendered_loop; + nSamplesToRender -= nSamplesRendered_loop; + if ( hIvasDec->nSamplesAvailableNext == 0 ) + { + *needNewFrame = true; + hIvasDec->needNewFrame = true; + } + else + { + *needNewFrame = false; + } + } +#ifdef API_5MS_BASELINE + } +#endif + + *nOutSamples = nSamplesRendered; + + return error; +} +#endif + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +ivas_error IVAS_DEC_GetSplitBinauralBitstream( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + void *pcmBuf_out, /* o : output synthesis signal for BINAURAL_SPLIT_PCM */ + IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits, /* o : bitstream output for split rendering mode */ + const int16_t nSamplesAsked, + int16_t *nOutSamples, /* o : number of samples per channel written to output buffer */ + bool *needNewFrame /* indication that the decoder needs a new frame */ +) +{ + Decoder_Struct *st_ivas; + AUDIO_CONFIG output_config; + int32_t output_Fs; + float *writePtr; + float *readPtr, *readEnd; + float *pOutput[BINAURAL_CHANNELS * MAX_HEAD_ROT_POSES]; + float output[BINAURAL_CHANNELS * MAX_HEAD_ROT_POSES][L_FRAME48k]; + float pcmBuf[BINAURAL_CHANNELS * MAX_HEAD_ROT_POSES * L_FRAME48k]; + int16_t numSamplesPerChannelCacheSize; + int16_t numSamplesPerChannelToDecode; + int16_t numSamplesPerChannelToSplitEncode; + int16_t i, j; + ivas_error error; + IVAS_DEC_SPLIT_REND_WRAPPER *hSplitBinRend; + int16_t max_band; + int16_t pcm_out; + int16_t td_input; + int16_t numPoses; + + error = IVAS_ERR_OK; + st_ivas = hIvasDec->st_ivas; + output_config = st_ivas->hDecoderConfig->output_config; + output_Fs = st_ivas->hDecoderConfig->output_Fs; + numSamplesPerChannelToDecode = (int16_t) ( output_Fs / FRAMES_PER_SEC ); + numSamplesPerChannelToDecode = nSamplesAsked; + *needNewFrame = FALSE; + hSplitBinRend = &st_ivas->hSplitBinRend; + if ( hSplitBinRend->numTdSamplesPerChannelCached == 0 ) + { + if ( ( error = ivas_set_split_rend_setup( hSplitBinRend, + &st_ivas->hRenderConfig->split_rend_config, + st_ivas->hCombinedOrientationData, + hSplitRendBits->bits_buf ) ) != IVAS_ERR_OK ) + { + return error; + } + } + numPoses = hSplitBinRend->splitrend.multiBinPoseData.num_poses; + + if ( hIvasDec->st_ivas->hRenderConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && + ( hIvasDec->st_ivas->hRenderConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE || + hIvasDec->st_ivas->hRenderConfig->split_rend_config.dof == 0 ) ) + { + numSamplesPerChannelToSplitEncode = (int16_t) ( output_Fs / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ); + numSamplesPerChannelCacheSize = numSamplesPerChannelToDecode - numSamplesPerChannelToSplitEncode; + hSplitRendBits->codec_frame_size_ms = 5; + if ( hSplitBinRend->tdDataOut == NULL ) + { + /* Allocate enough space to save all decoded samples that will not be split encoded directly after decoding */ + hSplitBinRend->tdDataOut = malloc( numSamplesPerChannelCacheSize * BINAURAL_CHANNELS * numPoses * sizeof( float ) ); + if ( hSplitBinRend->tdDataOut == NULL ) + { + return IVAS_ERR_FAILED_ALLOC; + } + } + } + else + { + numSamplesPerChannelToSplitEncode = (int16_t) ( output_Fs / FRAMES_PER_SEC ); + numSamplesPerChannelCacheSize = 0; + hSplitRendBits->codec_frame_size_ms = 20; + } + + if ( output_config != AUDIO_CONFIG_BINAURAL_SPLIT_CODED && output_config != AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) + { + return IVAS_ERR_WRONG_PARAMS; + } + + if ( hSplitBinRend->numSamplesCollected < numSamplesPerChannelToSplitEncode ) + { + /* Decode and render */ + error = IVAS_DEC_GetSamples( + hIvasDec, + numSamplesPerChannelToDecode, + IVAS_DEC_PCM_FLOAT, + pcmBuf, + nOutSamples, + needNewFrame ); + numSamplesPerChannelToDecode = *nOutSamples; + if ( error != IVAS_ERR_OK ) + { + return error; + } + if ( hIvasDec->st_ivas->hRenderConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && + ( hIvasDec->st_ivas->hRenderConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE || + hIvasDec->st_ivas->hRenderConfig->split_rend_config.dof == 0 ) ) + { + numSamplesPerChannelToSplitEncode = (int16_t) ( output_Fs / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ); + numSamplesPerChannelCacheSize = numSamplesPerChannelToDecode - numSamplesPerChannelToSplitEncode; + hSplitRendBits->codec_frame_size_ms = 5; + if ( hSplitBinRend->tdDataOut == NULL ) + { + /* Allocate enough space to save all decoded samples that will not be split encoded directly after decoding */ + hSplitBinRend->tdDataOut = malloc( numSamplesPerChannelCacheSize * BINAURAL_CHANNELS * numPoses * sizeof( float ) ); + if ( hSplitBinRend->tdDataOut == NULL ) + { + return IVAS_ERR_FAILED_ALLOC; + } + } + } + else + { + numSamplesPerChannelToSplitEncode = (int16_t) ( output_Fs / FRAMES_PER_SEC ); + numSamplesPerChannelCacheSize = 0; + hSplitRendBits->codec_frame_size_ms = 20; + } +#ifdef DEBUGGING + assert( numSamplesPerChannelToDecode == *nOutSamples ); +#endif + + /* copy to cache if cache is in use */ + if ( hSplitBinRend->tdDataOut != NULL ) + { + writePtr = hSplitBinRend->tdDataOut; + readPtr = pcmBuf + numSamplesPerChannelToSplitEncode * BINAURAL_CHANNELS * numPoses; + readEnd = pcmBuf + *nOutSamples * BINAURAL_CHANNELS * numPoses; + + while ( readPtr != readEnd ) + { + *writePtr++ = *readPtr++; + } + hSplitBinRend->numTdSamplesPerChannelCached = *nOutSamples - numSamplesPerChannelToSplitEncode; + } + hSplitBinRend->numSamplesCollected += *nOutSamples; + } + else if ( hSplitBinRend->tdDataOut != NULL ) + { + /* copy from cache */ + assert( hSplitBinRend->tdDataOut != NULL ); + + readPtr = hSplitBinRend->tdDataOut + ( numSamplesPerChannelCacheSize - hSplitBinRend->numTdSamplesPerChannelCached ) * BINAURAL_CHANNELS * numPoses; + readEnd = readPtr + numSamplesPerChannelToSplitEncode * BINAURAL_CHANNELS * numPoses; + writePtr = pcmBuf; + + while ( readPtr != readEnd ) + { + *writePtr++ = *readPtr++; + } + hSplitBinRend->numTdSamplesPerChannelCached -= numSamplesPerChannelToSplitEncode; + } + + if ( numSamplesPerChannelToSplitEncode == hSplitBinRend->numSamplesCollected ) + /* change buffer layout */ + { + for ( i = 0; i < numSamplesPerChannelToSplitEncode; ++i ) + { + for ( j = 0; j < BINAURAL_CHANNELS * numPoses; ++j ) + { + output[j][i] = pcmBuf[i * BINAURAL_CHANNELS * numPoses + j]; + } + } + for ( i = 0; i < BINAURAL_CHANNELS * MAX_HEAD_ROT_POSES; ++i ) + { + pOutput[i] = output[i]; + } + max_band = (int16_t) ( ( BINAURAL_MAXBANDS * output_Fs ) / 48000 ); + pcm_out = ( output_config == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0; + td_input = st_ivas->renderer_type != RENDERER_BINAURAL_FASTCONV && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC && st_ivas->renderer_type != RENDERER_BINAURAL_PARAMETRIC_ROOM && st_ivas->renderer_type != RENDERER_STEREO_PARAMETRIC; + + error = ivas_renderMultiBinToSplitBinaural( &hSplitBinRend->splitrend, + st_ivas->hHeadTrackData->Quaternion, + st_ivas->hRenderConfig->split_rend_config.splitRendBitRate, + st_ivas->hRenderConfig->split_rend_config.codec, +#ifdef API_5MS + st_ivas->hRenderConfig->split_rend_config.codec_frame_size_ms, +#endif + hSplitBinRend->hSplitRendBits, + hSplitBinRend->hMultiBinCldfbData->Cldfb_RealBuffer_Binaural, + hSplitBinRend->hMultiBinCldfbData->Cldfb_ImagBuffer_Binaural, + max_band, + pOutput, + 1, + td_input, + pcm_out ); + if ( error != IVAS_ERR_OK ) + { + return error; + } + + /* convert to int16 with limiting for BINAURAL_SPLIT_PCM */ + if ( pcm_out ) + { +#ifndef DISABLE_LIMITER + ivas_limiter_dec( st_ivas->hLimiter, pOutput, + st_ivas->hDecoderConfig->nchan_out, + numSamplesPerChannelToSplitEncode, st_ivas->BER_detect ); +#endif + +#ifdef DEBUGGING + st_ivas->noClipping += +#endif + ivas_syn_output( pOutput, numSamplesPerChannelToSplitEncode, + st_ivas->hDecoderConfig->nchan_out, + (int16_t *) pcmBuf_out ); + } + hSplitBinRend->numSamplesCollected = 0; + free( st_ivas->hSplitBinRend.hMultiBinCldfbData ); + } + + return error; +} +#endif /*---------------------------------------------------------------------* @@ -688,7 +1327,12 @@ static ivas_error IVAS_DEC_Setup( uint8_t *nTransportChannels, /* o : number of decoded transport PCM channels */ uint8_t *nOutChannels, /* o : number of decoded out channels (PCM or CLDFB) */ uint16_t *nSamplesRendered, /* o : number of samples flushed from the last frame */ - int16_t *data /* o : flushed samples */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const IVAS_DEC_PCM_TYPE pcmType, /* i : type for the decoded PCM resolution */ + void *data /* o : output synthesis signal */ +#else + int16_t *data /* o : output synthesis signal */ +#endif ) { ivas_error error; @@ -726,7 +1370,11 @@ static ivas_error IVAS_DEC_Setup( if ( st_ivas->bfi == 0 ) { - if ( ( error = ivas_dec_setup( st_ivas, nSamplesRendered, data ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_dec_setup( st_ivas, nSamplesRendered, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_type_API_to_internal( pcmType ), +#endif + data ) ) != IVAS_ERR_OK ) { return error; } @@ -839,7 +1487,12 @@ static ivas_error IVAS_DEC_GetRenderedSamples( const uint16_t nSamplesForRendering, /* i : number of TC samples wanted from the renderer */ uint16_t *nSamplesRendered, /* o : number of samples rendered */ uint16_t *nSamplesAvailableNext, /* o : number of samples still available in the renerer pipeline */ - int16_t *pcmBuf /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_TYPE pcmType, + void *pcmBuf +#else + int16_t *pcmBuf +#endif ) { Decoder_Struct *st_ivas; @@ -855,7 +1508,11 @@ static ivas_error IVAS_DEC_GetRenderedSamples( st_ivas = hIvasDec->st_ivas; /* run the main IVAS decoding routine */ - if ( ( error = ivas_jbm_dec_render( st_ivas, nSamplesForRendering, nSamplesRendered, nSamplesAvailableNext, pcmBuf ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_jbm_dec_render( st_ivas, nSamplesForRendering, nSamplesRendered, nSamplesAvailableNext, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcm_type_API_to_internal( pcmType ), +#endif + pcmBuf ) ) != IVAS_ERR_OK ) { return error; } @@ -1072,9 +1729,14 @@ ivas_error IVAS_DEC_GetMasaMetadata( *---------------------------------------------------------------------*/ ivas_error IVAS_DEC_FeedHeadTrackData( - IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ +#ifdef API_5MS + IVAS_QUATERNION orientation, /* i : head-tracking data, listener orientation */ + IVAS_VECTOR3 Pos /* i : listener position */ +#else IVAS_QUATERNION *orientation, /* i : head-tracking data, listener orientation */ IVAS_VECTOR3 *Pos /* i : listener position */ +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , const IVAS_SPLIT_REND_ROT_AXIS rot_axis @@ -1082,9 +1744,15 @@ ivas_error IVAS_DEC_FeedHeadTrackData( ) { HEAD_TRACK_DATA_HANDLE hHeadTrackData; +#ifndef API_5MS int16_t i; +#endif +#ifdef API_5MS + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) +#else if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL || orientation == NULL ) +#endif { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } @@ -1097,6 +1765,18 @@ ivas_error IVAS_DEC_FeedHeadTrackData( } /* Move head-tracking data to the decoder handle */ +#ifdef API_5MS + /* check for Euler angle signaling */ + if ( orientation.w == -3.0f ) + { + Euler2Quat( deg2rad( orientation.x ), deg2rad( orientation.y ), deg2rad( orientation.z ), &orientation ); + } + + ivas_orient_trk_Process( hHeadTrackData->OrientationTracker, orientation, FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES, &hHeadTrackData->Quaternion ); + hHeadTrackData->Pos.x = Pos.x; + hHeadTrackData->Pos.y = Pos.y; + hHeadTrackData->Pos.z = Pos.z; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { /* check for Euler angle signaling */ @@ -1111,12 +1791,15 @@ ivas_error IVAS_DEC_FeedHeadTrackData( hHeadTrackData->Pos[i].y = Pos[i].y; hHeadTrackData->Pos[i].z = Pos[i].z; } +#endif +#ifndef API_5MS #ifdef SPLIT_REND_WITH_HEAD_ROT hHeadTrackData->num_quaternions = 0; #else hIvasDec->st_ivas->hHeadTrackData->num_quaternions = 0; #endif +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT hHeadTrackData->sr_pose_pred_axis = rot_axis; @@ -1189,18 +1872,32 @@ ivas_error IVAS_DEC_FeedRefVectorData( *---------------------------------------------------------------------*/ ivas_error IVAS_DEC_FeedExternalOrientationData( - IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ +#ifdef API_5MS + IVAS_QUATERNION orientation, /* i : external orientation data */ + int8_t enableHeadRotation, /* i : flag to enable head rotation for this frame */ + int8_t enableExternalOrientation, /* i : flag to enable external orientation for this frame */ + int8_t enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ + int16_t numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#else IVAS_QUATERNION *orientation, /* i : external orientation data */ int8_t *enableHeadRotation, /* i : flag to enable head rotation for this frame */ int8_t *enableExternalOrientation, /* i : flag to enable external orientation for this frame */ int8_t *enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ int16_t *numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#endif ) { EXTERNAL_ORIENTATION_HANDLE hExternalOrientationData; +#ifndef API_5MS int16_t i; +#endif +#ifdef API_5MS + if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL ) +#else if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL || orientation == NULL ) +#endif { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } @@ -1213,6 +1910,14 @@ ivas_error IVAS_DEC_FeedExternalOrientationData( } /* Move external orientation data to the decoder handle (invert orientations) */ +#ifdef API_5MS + QuaternionInverse( orientation, &hExternalOrientationData->Quaternion ); + + hExternalOrientationData->enableHeadRotation = enableHeadRotation; + hExternalOrientationData->enableExternalOrientation = enableExternalOrientation; + hExternalOrientationData->enableRotationInterpolation = enableRotationInterpolation; + hExternalOrientationData->numFramesToTargetOrientation = numFramesToTargetOrientation; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { QuaternionInverse( orientation[i], &hExternalOrientationData->Quaternions[i] ); @@ -1222,6 +1927,7 @@ ivas_error IVAS_DEC_FeedExternalOrientationData( hExternalOrientationData->enableRotationInterpolation[i] = enableRotationInterpolation[i]; hExternalOrientationData->numFramesToTargetOrientation[i] = numFramesToTargetOrientation[i]; } +#endif return IVAS_ERR_OK; } @@ -1368,6 +2074,53 @@ ivas_error IVAS_DEC_GetHrtfParamBinHandle( return IVAS_ERR_OK; } +#ifdef API_5MS +static ivas_error copyRendererConfigStruct( RENDER_CONFIG_HANDLE hRCin, IVAS_RENDER_CONFIG_HANDLE hRCout ) +{ + if ( hRCin == NULL || hRCout == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + +#ifdef DEBUGGING + switch ( hRCin->renderer_type_override ) + { + case RENDER_TYPE_OVERRIDE_CREND: + hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_CREND; + break; + case RENDER_TYPE_OVERRIDE_FASTCONV: + hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_FASTCONV; + break; + default: + hRCout->renderer_type_override = IVAS_RENDER_TYPE_OVERRIDE_NONE; + break; + } +#endif + hRCout->room_acoustics.override = hRCin->roomAcoustics.override; + hRCout->room_acoustics.nBands = hRCin->roomAcoustics.nBands; + hRCout->room_acoustics.acousticPreDelay = hRCin->roomAcoustics.acousticPreDelay; + hRCout->room_acoustics.inputPreDelay = hRCin->roomAcoustics.inputPreDelay; + + mvr2r( hRCin->roomAcoustics.pFc_input, hRCout->room_acoustics.pFc_input, CLDFB_NO_CHANNELS_MAX ); + mvr2r( hRCin->roomAcoustics.pAcoustic_rt60, hRCout->room_acoustics.pAcoustic_rt60, CLDFB_NO_CHANNELS_MAX ); + mvr2r( hRCin->roomAcoustics.pAcoustic_dsr, hRCout->room_acoustics.pAcoustic_dsr, CLDFB_NO_CHANNELS_MAX ); + mvr2r( hRCin->directivity, hRCout->directivity, 3 ); +#ifdef SPLIT_REND_WITH_HEAD_ROT + hRCout->split_rend_config.splitRendBitRate = SPLIT_REND_768k; + hRCout->split_rend_config.dof = 3; + hRCout->split_rend_config.hq_mode = 0; + hRCout->split_rend_config.codec_delay_ms = 0; + hRCout->split_rend_config.codec_frame_size_ms = 0; /* 0 means "use default for selected codec" */ + hRCout->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; + hRCout->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB; + hRCout->split_rend_config.rendererSelection = hRCin->split_rend_config.rendererSelection; +#endif + hRCout->room_acoustics.use_er = hRCin->roomAcoustics.use_er; + hRCout->room_acoustics.lowComplexity = hRCin->roomAcoustics.lowComplexity; + + return IVAS_ERR_OK; +} +#endif /*---------------------------------------------------------------------* * IVAS_DEC_GetRenderConfig( ) @@ -1380,13 +2133,18 @@ ivas_error IVAS_DEC_GetRenderConfig( const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render configuration handle */ ) { +#ifndef API_5MS RENDER_CONFIG_HANDLE hRCin; +#endif if ( hIvasDec == NULL || hIvasDec->st_ivas == NULL || hIvasDec->st_ivas->hRenderConfig == NULL || hRCout == NULL ) { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } +#ifdef API_5MS + return copyRendererConfigStruct( hIvasDec->st_ivas->hRenderConfig, hRCout ); +#else hRCin = hIvasDec->st_ivas->hRenderConfig; #ifdef DEBUGGING switch ( hRCin->renderer_type_override ) @@ -1426,8 +2184,28 @@ ivas_error IVAS_DEC_GetRenderConfig( hRCout->room_acoustics.lowComplexity = hRCin->roomAcoustics.lowComplexity; return IVAS_ERR_OK; +#endif } +#ifdef API_5MS +/*! r: error code*/ +ivas_error IVAS_DEC_GetDefaultRenderConfig( + IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render config handle */ +) +{ + RENDER_CONFIG_DATA RCin; + RENDER_CONFIG_HANDLE hRCin = &RCin; + ivas_error error; + + if ( ( error = ivas_render_config_init_from_rom( &hRCin ) ) != IVAS_ERR_OK ) + { + return error; + } + + return copyRendererConfigStruct( hRCin, hRCout ); +} +#endif + /*---------------------------------------------------------------------* * IVAS_DEC_FeedRenderConfig( ) @@ -1734,33 +2512,236 @@ ivas_error IVAS_DEC_VoIP_FeedFrame( } } - return IVAS_ERR_OK; -} + return IVAS_ERR_OK; +} + +#if defined( VARIABLE_SPEED_DECODING ) || defined( API_5MS ) +/*---------------------------------------------------------------------* + * IVAS_DEC_VoIP_SetScale( ) + * + * Set the TSM scale + *---------------------------------------------------------------------*/ + +ivas_error IVAS_DEC_VoIP_SetScale( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ +#ifdef API_5MS + const int16_t maxScaling, +#endif + const int16_t scale /* i : TSM scale to set */ +) +{ + ivas_error error; + + error = IVAS_ERR_OK; + +#ifdef API_5MS + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm == false ) + { + return IVAS_ERR_TSM_NOT_ENABLED; + } + else + { + hIvasDec->tsm_scale = scale; + hIvasDec->tsm_max_scaling = maxScaling; + } +#else + hIvasDec->hVoIP->speedFac = scale; +#endif + + return error; +} +#endif + + +#ifdef API_5MS +/*---------------------------------------------------------------------* + * IVAS_DEC_VoIP_GetSamples( ) + * + * Main function to decode one frame in VoIP + *---------------------------------------------------------------------*/ + +ivas_error IVAS_DEC_VoIP_GetSamples( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + uint16_t nSamplesPerChannel, /* i : number of samples per channel requested to be written to output buffer */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_TYPE pcmType, + void *pcmBuf, +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif + const uint32_t systemTimestamp_ms /* i : current system timestamp */ +#ifdef SUPPORT_JBM_TRACEFILE + , + JbmTraceFileWriterFn jbmWriterFn, + void *jbmWriter +#endif + +) +{ + Decoder_Struct *st_ivas; + DECODER_CONFIG_HANDLE hDecoderConfig; + IVAS_DEC_VOIP *hVoIP; + uint32_t extBufferedTime_ms, scale, maxScaling; + JB4_DATAUNIT_HANDLE dataUnit; + uint16_t extBufferedSamples; + int16_t timeScalingDone; + int16_t result; + ivas_error error; + int16_t nSamplesRendered; + uint8_t nOutChannels; + + error = IVAS_ERR_OK; + + st_ivas = hIvasDec->st_ivas; + hDecoderConfig = st_ivas->hDecoderConfig; + hVoIP = hIvasDec->hVoIP; + timeScalingDone = 0; + + nOutChannels = (uint8_t) st_ivas->hDecoderConfig->nchan_out; + nSamplesRendered = 0; + + if ( nSamplesPerChannel == 0 ) + { + return IVAS_ERR_WRONG_PARAMS; + } + + /* make sure that the FIFO after decoder/scaler contains at least one sound card frame (i.e. 20ms) */ + while ( nSamplesRendered < nSamplesPerChannel ) + { + if ( hIvasDec->nSamplesAvailableNext == 0 ) + { + int16_t nSamplesBuffered; + nSamplesBuffered = 0; + if ( hIvasDec->hasBeenFedFirstGoodFrame ) + { + IVAS_DEC_GetBufferedNumberOfSamples( hIvasDec, &nSamplesBuffered ); + } + extBufferedSamples = nSamplesRendered + nSamplesBuffered; + + extBufferedTime_ms = extBufferedSamples * 1000 / hDecoderConfig->output_Fs; + + dataUnit = NULL; + + + /* pop one access unit from the jitter buffer */ + result = JB4_PopDataUnit( hVoIP->hJBM, systemTimestamp_ms, extBufferedTime_ms, &dataUnit, &scale, &maxScaling ); + if ( result != 0 ) + { + return IVAS_ERR_UNKNOWN; + } + -#ifdef VARIABLE_SPEED_DECODING -#ifdef DEBUGGING -/*---------------------------------------------------------------------* - * IVAS_DEC_VoIP_SetScale( ) - * - * Set the TSM scale - *---------------------------------------------------------------------*/ + maxScaling = maxScaling * hDecoderConfig->output_Fs / 1000; + /* avoid time scaling multiple times in one sound card slot */ + if ( scale != 100U ) + { + if ( timeScalingDone ) + { + scale = 100; + } + else + { + timeScalingDone = 1; + } + } -ivas_error IVAS_DEC_VoIP_SetScale( - IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ - const int16_t scale /* i : TSM scale to set */ -) -{ - ivas_error error; + /* limit scale to range supported by time scaler */ + if ( scale < APA_MIN_SCALE ) + { + scale = APA_MIN_SCALE; + } + else if ( scale > APA_MAX_SCALE ) + { + scale = APA_MAX_SCALE; + } - error = IVAS_ERR_OK; + IVAS_DEC_VoIP_SetScale( hIvasDec, (int16_t) maxScaling, (int16_t) scale ); - hIvasDec->hVoIP->speedFac = scale; + /* copy bitstream into decoder state */ + if ( dataUnit ) + { + hIvasDec->hVoIP->hCurrentDataUnit = dataUnit; - return error; -} + bsCompactToSerial( dataUnit->data, hIvasDec->hVoIP->bs_conversion_buf, dataUnit->dataSize ); + IVAS_DEC_FeedFrame_Serial( hIvasDec, hIvasDec->hVoIP->bs_conversion_buf, dataUnit->dataSize, 0 ); + } + else if ( hIvasDec->hasDecodedFirstGoodFrame ) + { + /* Decoder has been initialized with first good frame - do PLC */ + IVAS_DEC_FeedFrame_Serial( hIvasDec, hIvasDec->hVoIP->bs_conversion_buf, 0, 1 ); + } + +#ifdef SUPPORT_JBM_TRACEFILE + /* jbmWriterFn and jbmWriter may be NULL if tracefile writing was not requested on CLI */ + if ( jbmWriterFn != NULL && jbmWriter != NULL ) + { + /* write JBM trace data entry */ + store_JbmData( hVoIP, dataUnit, systemTimestamp_ms, extBufferedSamples, hDecoderConfig->output_Fs ); + if ( ( jbmWriterFn( &hVoIP->JbmTraceData, jbmWriter ) ) != IVAS_ERR_OK ) + { + fprintf( stderr, "\nError writing JBM Trace data to file\n" ); + return IVAS_ERR_UNKNOWN; + } + } +#endif + if ( dataUnit ) + { + if ( dataUnit->partial_frame != 0 ) + { + hVoIP->lastDecodedWasActive = 1; + } + else + { + hVoIP->lastDecodedWasActive = !dataUnit->silenceIndicator; + } + /* data unit memory is no longer used */ + JB4_FreeDataUnit( hVoIP->hJBM, dataUnit ); + } + if ( !hIvasDec->hasBeenFedFirstGoodFrame ) + { + hIvasDec->nSamplesAvailableNext = hIvasDec->nSamplesFrame; + hIvasDec->nSamplesRendered = 0; + } + } + /* decode */ + if ( !hIvasDec->hasBeenFedFirstGoodFrame ) + { + /* codec mode to use not known yet - simply output silence */ + /* directly set output zero */ + int16_t nSamplesToZero = min( nSamplesPerChannel, hIvasDec->nSamplesAvailableNext ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + set_pcm_buffer_to_zero( pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ), pcmType, nSamplesToZero * nOutChannels ); +#else + set_s( pcmBuf + nSamplesRendered * nOutChannels, 0, nSamplesToZero * nOutChannels ); #endif + nSamplesRendered += nSamplesToZero; + hIvasDec->nSamplesRendered += nSamplesToZero; + hIvasDec->nSamplesAvailableNext -= nSamplesToZero; + } + else + { + int16_t nSamplesToRender, nSamplesRendered_loop; + bool tmp; + nSamplesToRender = nSamplesPerChannel - nSamplesRendered; + + /* render IVAS frames directly to the output buffer */ + if ( ( error = IVAS_DEC_GetSamples( hIvasDec, nSamplesToRender, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcmType, + pcm_buffer_offset( pcmBuf, pcmType, nSamplesRendered * nOutChannels ), +#else + pcmBuf + nSamplesRendered * nOutChannels, #endif - + &nSamplesRendered_loop, &tmp ) ) != IVAS_ERR_OK ) + { + return error; + } + nSamplesRendered += nSamplesRendered_loop; + } + } + return error; +} +#else /*---------------------------------------------------------------------* * IVAS_DEC_VoIP_GetSamples( ) * @@ -2146,6 +3127,7 @@ ivas_error IVAS_DEC_VoIP_GetSamples( return error; } +#endif /*---------------------------------------------------------------------* * IVAS_DEC_VoIP_Flush( ) @@ -2153,25 +3135,41 @@ ivas_error IVAS_DEC_VoIP_GetSamples( * Function to flush remaining audio in VoIP *---------------------------------------------------------------------*/ -ivas_error IVAS_DEC_VoIP_Flush( +ivas_error IVAS_DEC_Flush( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ const int16_t nSamplesPerChannel, /* i : number of samples per channel requested to be written to output buffer */ - int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ - uint16_t *nSamplesAvailableNext, /* o : number of samples still available */ - int16_t *nSamplesFlushed /* o : number of samples flushed */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const IVAS_DEC_PCM_TYPE pcmType, + void *pcmBuf, +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif +#ifndef API_5MS + uint16_t *nSamplesAvailableNext, /* o : number of samples still available */ +#endif + int16_t *nSamplesFlushed /* o : number of samples flushed */ ) { ivas_error error; +#ifndef API_5MS IVAS_DEC_VOIP *hVoIP; int16_t rendererPcmBuf[( MAX_OUTPUT_CHANNELS * L_FRAME_MAX * APA_MAX_SCALE ) / 100]; +#endif uint16_t nSamplesToRender; uint16_t nSamplesFlushedLocal; error = IVAS_ERR_OK; +#ifndef API_5MS hVoIP = hIvasDec->hVoIP; +#endif +#ifdef API_5MS + *nSamplesFlushed = min( nSamplesPerChannel, hIvasDec->nSamplesAvailableNext ); +#else *nSamplesFlushed = min( nSamplesPerChannel, hVoIP->nSamplesAvailableNext ); +#endif +#ifndef API_5MS if ( hVoIP->rendererType == JBM_RENDERER_NONE ) { /* fetch a user-specified number of samples from FIFO */ @@ -2184,6 +3182,7 @@ ivas_error IVAS_DEC_VoIP_Flush( } else { + nSamplesToRender = (uint16_t) *nSamplesFlushed; /* render IVAS frames */ if ( ( error = IVAS_DEC_GetRenderedSamples( hIvasDec, nSamplesToRender, &nSamplesFlushedLocal, &hVoIP->nSamplesAvailableNext, rendererPcmBuf ) ) != IVAS_ERR_OK ) @@ -2204,7 +3203,19 @@ ivas_error IVAS_DEC_VoIP_Flush( *nSamplesAvailableNext = hVoIP->nSamplesAvailableNext; *nSamplesFlushed = (int16_t) nSamplesFlushedLocal; } +#else + nSamplesToRender = (uint16_t) *nSamplesFlushed; + /* render IVAS frames */ + if ( ( error = IVAS_DEC_GetRenderedSamples( hIvasDec, nSamplesToRender, &nSamplesFlushedLocal, &hIvasDec->nSamplesAvailableNext, +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + pcmType, +#endif + pcmBuf ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif return error; } @@ -2219,7 +3230,11 @@ bool IVAS_DEC_VoIP_IsEmpty( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ const int16_t nSamplesAsked ) { +#ifdef API_5MS + return ( ( JB4_bufferedDataUnits( hIvasDec->hVoIP->hJBM ) == 0 ) && ( hIvasDec->nSamplesAvailableNext < nSamplesAsked ) ); +#else return ( ( JB4_bufferedDataUnits( hIvasDec->hVoIP->hJBM ) == 0 ) && ( hIvasDec->hVoIP->nSamplesAvailableNext < nSamplesAsked ) ); +#endif } @@ -2258,6 +3273,7 @@ static void IVAS_DEC_Close_VoIP( { JB4_Destroy( &hVoIP->hJBM ); +#ifndef API_5MS apa_exit( &hVoIP->hTimeScaler ); pcmdsp_fifo_destroy( &hVoIP->hFifoOut ); @@ -2266,7 +3282,7 @@ static void IVAS_DEC_Close_VoIP( { free( hVoIP->apaExecBuffer ); } - +#endif if ( hVoIP->bs_conversion_buf != NULL ) { #define WMC_TOOL_SKIP @@ -2613,15 +3629,32 @@ static ivas_error printConfigInfo_dec( } } +#ifdef API_5MS + /*-----------------------------------------------------------------* + * Print TSM mode info + *-----------------------------------------------------------------*/ + if ( st_ivas->hDecoderConfig->Opt_tsm ) + { + fprintf( stdout, "TSM mode: ON\n" ); + } +#ifdef API_5MS_BASELINE + /*-----------------------------------------------------------------* + * Print 5ms API mode info + *-----------------------------------------------------------------*/ + if ( st_ivas->hDecoderConfig->Opt_5ms ) + { + fprintf( stdout, "API 5ms mode: ON\n" ); + } +#endif +#else /*-----------------------------------------------------------------* * Print VoIP mode info *-----------------------------------------------------------------*/ - if ( st_ivas->hDecoderConfig->voip_active ) { fprintf( stdout, "VoIP mode: ON\n" ); } - +#endif return IVAS_ERR_OK; } @@ -2795,7 +3828,20 @@ static ivas_error evs_dec_main( v_multc( output[0], mixer_left, output[0], nOutSamples ); } - if ( floatBuf != NULL ) + +#ifdef API_5MS + if ( !st_ivas->hDecoderConfig->Opt_tsm +#ifdef API_5MS_BASELINE + && st_ivas->hDecoderConfig->Opt_5ms +#endif + + ) + { + ivas_jbm_dec_copy_tc_no_tsm( st_ivas, p_output, nOutSamples ); + } + else +#endif + if ( floatBuf != NULL ) { /* BE workaround */ int16_t pcm_buf_local[L_FRAME48k * MAX_OUTPUT_CHANNELS_IN_DIEGETIC_PAN]; @@ -3023,11 +4069,11 @@ static ivas_error input_format_API_to_internal( static int16_t IVAS_DEC_VoIP_GetRenderGranularity( Decoder_Struct *st_ivas ) { - return st_ivas->hTcBuffer->n_samples_granularity; } +#ifndef API_5MS /*---------------------------------------------------------------------* * IVAS_DEC_VoIP_GetRendererConfig() * @@ -3051,7 +4097,7 @@ static JBM_RENDERER_TYPE IVAS_DEC_VoIP_GetRendererConfig( return rendererType; } - +#endif /*---------------------------------------------------------------------* * IVAS_DEC_VoIP_reconfigure() @@ -3065,58 +4111,100 @@ static ivas_error IVAS_DEC_VoIP_reconfigure( const uint16_t l_ts ) { +#ifndef API_5MS IVAS_DEC_VOIP *hVoIP; +#endif +#ifdef API_5MS + int16_t apa_buffer_size; +#endif + ivas_error error; +#ifndef API_5MS hVoIP = hIvasDec->hVoIP; +#endif +#ifdef API_5MS + apa_buffer_size = hIvasDec->nSamplesFrame; +#endif +#ifdef API_5MS + if ( hIvasDec->apaExecBuffer == NULL ) +#else if ( hIvasDec->hVoIP->hTimeScaler == NULL ) +#endif { - +#ifndef API_5MS uint16_t wss, css; float startQuality; +#endif DECODER_CONFIG_HANDLE hDecoderConfig; +#ifdef API_5MS + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm ) + { + uint16_t wss, css; + float startQuality; + + startQuality = 1.0f; + apa_buffer_size = APA_BUF_PER_CHANNEL; +#else #ifdef VARIABLE_SPEED_DECODING startQuality = hVoIP->voipMode == IVAS_DEC_VOIP_MODE_VARIABLE_SPEED ? -2.0f : 1.0f; #else startQuality = 1.0f; +#endif #endif - /* get current renderer type*/ - hVoIP->rendererType = IVAS_DEC_VoIP_GetRendererConfig( hIvasDec ); - hDecoderConfig = hIvasDec->st_ivas->hDecoderConfig; - if ( hDecoderConfig->output_Fs == 8000 ) - { - wss = 1; - css = 1; - } - else if ( hDecoderConfig->output_Fs == 16000 ) - { - wss = 2; - css = 1; - } - else if ( hDecoderConfig->output_Fs == 32000 ) - { - wss = 4; - css = 2; - } - else if ( hDecoderConfig->output_Fs == 48000 ) - { - wss = 6; - css = 3; - } - else - { - return IVAS_ERR_INIT_ERROR; - } + /* get current renderer type*/ +#ifndef API_5MS + hVoIP->rendererType = IVAS_DEC_VoIP_GetRendererConfig( hIvasDec ); +#endif + hDecoderConfig = hIvasDec->st_ivas->hDecoderConfig; - if ( ( hIvasDec->hVoIP->apaExecBuffer = malloc( sizeof( float ) * APA_BUF_PER_CHANNEL * nTransportChannels ) ) == NULL ) - { - return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Could not allocate VoIP handle" ); - } - set_zero( hIvasDec->hVoIP->apaExecBuffer, APA_BUF_PER_CHANNEL * nTransportChannels ); + if ( hDecoderConfig->output_Fs == 8000 ) + { + wss = 1; + css = 1; + } + else if ( hDecoderConfig->output_Fs == 16000 ) + { + wss = 2; + css = 1; + } + else if ( hDecoderConfig->output_Fs == 32000 ) + { + wss = 4; + css = 2; + } + else if ( hDecoderConfig->output_Fs == 48000 ) + { + wss = 6; + css = 3; + } + else + { + return IVAS_ERR_INIT_ERROR; + } +#ifndef API_5MS + if ( ( hIvasDec->hVoIP->apaExecBuffer = malloc( sizeof( float ) * APA_BUF_PER_CHANNEL * nTransportChannels ) ) == NULL ) + + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Could not allocate VoIP handle" ); + } + set_zero( hIvasDec->hVoIP->apaExecBuffer, APA_BUF_PER_CHANNEL * nTransportChannels ); +#endif +#ifdef API_5MS + if ( apa_init( &hIvasDec->hTimeScaler, + nTransportChannels ) != IVAS_ERR_OK || + apa_set_rate( hIvasDec->hTimeScaler, hDecoderConfig->output_Fs ) != 0 || + apa_set_complexity_options( hIvasDec->hTimeScaler, wss, css ) != 0 || + apa_set_quality( hIvasDec->hTimeScaler, startQuality, 4, 4 ) != 0 || + apa_set_renderer_granularity( hIvasDec->hTimeScaler, l_ts ) != 0 ) + { + return IVAS_ERR_INIT_ERROR; + } +#else if ( apa_init( &hIvasDec->hVoIP->hTimeScaler, nTransportChannels ) != IVAS_ERR_OK || apa_set_rate( hIvasDec->hVoIP->hTimeScaler, hDecoderConfig->output_Fs ) != 0 || @@ -3126,51 +4214,99 @@ static ivas_error IVAS_DEC_VoIP_reconfigure( { return IVAS_ERR_INIT_ERROR; } +#endif - if ( hVoIP->hFifoOut == NULL && hVoIP->rendererType == JBM_RENDERER_NONE ) - { - /* we still need the FIFO out buffer */ - if ( pcmdsp_fifo_create( &hIvasDec->hVoIP->hFifoOut ) != 0 || - pcmdsp_fifo_init( hIvasDec->hVoIP->hFifoOut, (uint16_t) ( hDecoderConfig->output_Fs * 4 / FRAMES_PER_SEC ) /* 4 frames */, hDecoderConfig->nchan_out, sizeof( int16_t ) ) != 0 ) +#ifndef API_5MS + if ( hVoIP->hFifoOut == NULL && hVoIP->rendererType == JBM_RENDERER_NONE ) { - return IVAS_ERR_INIT_ERROR; + /* we still need the FIFO out buffer */ + if ( pcmdsp_fifo_create( &hIvasDec->hVoIP->hFifoOut ) != 0 || + pcmdsp_fifo_init( hIvasDec->hVoIP->hFifoOut, (uint16_t) ( hDecoderConfig->output_Fs * 4 / FRAMES_PER_SEC ) /* 4 frames */, hDecoderConfig->nchan_out, sizeof( int16_t ) ) != 0 ) + { + return IVAS_ERR_INIT_ERROR; + } } - } #ifdef VARIABLE_SPEED_DECODING - else if ( hIvasDec->hVoIP->voipMode == IVAS_DEC_VOIP_MODE_VARIABLE_SPEED ) - { - if ( pcmdsp_fifo_create( &hIvasDec->hVoIP->hFifoOut ) != 0 || - pcmdsp_fifo_init( hIvasDec->hVoIP->hFifoOut, (uint16_t) ( hDecoderConfig->output_Fs * 4 / FRAMES_PER_SEC ) /* 4 frames */, hDecoderConfig->nchan_out, sizeof( int16_t ) ) != 0 ) + else if ( hIvasDec->hVoIP->voipMode == IVAS_DEC_VOIP_MODE_VARIABLE_SPEED ) { - return IVAS_ERR_INIT_ERROR; + if ( pcmdsp_fifo_create( &hIvasDec->hVoIP->hFifoOut ) != 0 || + pcmdsp_fifo_init( hIvasDec->hVoIP->hFifoOut, (uint16_t) ( hDecoderConfig->output_Fs * 4 / FRAMES_PER_SEC ) /* 4 frames */, hDecoderConfig->nchan_out, sizeof( int16_t ) ) != 0 ) + { + return IVAS_ERR_INIT_ERROR; + } } - } +#endif #endif - if ( hIvasDec->mode == IVAS_DEC_MODE_EVS ) - { + if ( hIvasDec->mode == IVAS_DEC_MODE_EVS ) + { +#ifdef API_5MS + if ( apa_set_evs_compat_mode( hIvasDec->hTimeScaler, true ) != 0 ) +#else if ( apa_set_evs_compat_mode( hIvasDec->hVoIP->hTimeScaler, true ) != 0 ) +#endif + { + return IVAS_ERR_INIT_ERROR; + } + } +#ifdef API_5MS + if ( ( hIvasDec->apaExecBuffer = malloc( sizeof( float ) * apa_buffer_size * nTransportChannels ) ) == NULL ) + { - return IVAS_ERR_INIT_ERROR; + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Could not allocate VoIP handle" ); } + + set_zero( hIvasDec->apaExecBuffer, apa_buffer_size * nTransportChannels ); + } +#endif +#ifndef API_5MS + if ( ( hVoIP->apaExecBuffer = malloc( sizeof( float ) * APA_BUF_PER_CHANNEL * nTransportChannels ) ) == NULL ) + + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Could not allocate VoIP handle" ); } + + set_zero( hVoIP->apaExecBuffer, APA_BUF_PER_CHANNEL * nTransportChannels ); +#endif } else { - if ( apa_reconfigure( hVoIP->hTimeScaler, nTransportChannels, l_ts ) != 0 ) +#ifdef API_5MS + if ( hIvasDec->st_ivas->hDecoderConfig->Opt_tsm ) { - return IVAS_ERR_INIT_ERROR; + if ( apa_reconfigure( hIvasDec->hTimeScaler, nTransportChannels, l_ts ) != 0 ) +#else + if ( apa_reconfigure( hVoIP->hTimeScaler, nTransportChannels, l_ts ) != 0 ) +#endif + { + return IVAS_ERR_INIT_ERROR; + } +#ifdef API_5MS + apa_buffer_size = APA_BUF_PER_CHANNEL; + free( hIvasDec->apaExecBuffer ); + if ( ( hIvasDec->apaExecBuffer = malloc( sizeof( float ) * apa_buffer_size * nTransportChannels ) ) == NULL ) + { + return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Could not allocate VoIP handle" ); + } + set_zero( hIvasDec->apaExecBuffer, apa_buffer_size * nTransportChannels ); } - +#endif /* realloc apa_exe_buffer */ +#ifndef API_5MS free( hIvasDec->hVoIP->apaExecBuffer ); if ( ( hIvasDec->hVoIP->apaExecBuffer = malloc( sizeof( float ) * APA_BUF_PER_CHANNEL * nTransportChannels ) ) == NULL ) { return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Could not allocate VoIP handle" ); } set_zero( hIvasDec->hVoIP->apaExecBuffer, APA_BUF_PER_CHANNEL * nTransportChannels ); +#endif } + +#ifdef API_5MS + hIvasDec->nTransportChannelsOld = nTransportChannels; +#else hIvasDec->hVoIP->nTransportChannelsOld = (uint8_t) nTransportChannels; +#endif error = IVAS_ERR_OK; @@ -3268,3 +4404,64 @@ ivas_error IVAS_DEC_GetCldfbSamples( return error; } #endif + +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +void *pcm_buffer_offset( void *buffer, const IVAS_DEC_PCM_TYPE pcmType, int32_t offset ) +{ + switch ( pcmType ) + { + case IVAS_DEC_PCM_FLOAT: + { + float *tmpBuf = (float *) buffer; + return (void *) ( tmpBuf + offset ); + } + break; + case IVAS_DEC_PCM_INT16: + { + int16_t *tmpBuf = (int16_t *) buffer; + return (void *) ( tmpBuf + offset ); + } + break; + default: + return NULL; + } +} + +ivas_error set_pcm_buffer_to_zero( void *buffer, const IVAS_DEC_PCM_TYPE pcmType, int16_t nZeroSamples ) +{ + ivas_error error; + error = IVAS_ERR_OK; + + switch ( pcmType ) + { + case IVAS_DEC_PCM_FLOAT: + set_zero( (float *) buffer, nZeroSamples ); + break; + case IVAS_DEC_PCM_INT16: + set_s( (int16_t *) buffer, 0, nZeroSamples ); + break; + default: + error = IVAS_ERR_INTERNAL; + } + return error; +} + +PCM_RESOLUTION pcm_type_API_to_internal( const IVAS_DEC_PCM_TYPE pcmType ) +{ + PCM_RESOLUTION pcm_resolution; + pcm_resolution = PCM_NOT_KNOW; + switch ( pcmType ) + { + case IVAS_DEC_PCM_FLOAT: + pcm_resolution = PCM_FLOAT32; + break; + case IVAS_DEC_PCM_INT16: + pcm_resolution = PCM_INT16; + break; + default: + pcm_resolution = PCM_NOT_KNOW; + } + return pcm_resolution; +} + +#endif diff --git a/lib_dec/lib_dec.h b/lib_dec/lib_dec.h index 623603b6c342f8c57425f573f9ff38eeffa5c36a..87a1cd6551a00a46f200ac9c90d43b1652190e2d 100644 --- a/lib_dec/lib_dec.h +++ b/lib_dec/lib_dec.h @@ -63,7 +63,7 @@ typedef enum _IVAS_DEC_COMPLEXITY_LEVEL IVAS_DEC_COMPLEXITY_LEVEL_THREE = 3 } IVAS_DEC_COMPLEXITY_LEVEL; - +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING typedef enum { @@ -71,6 +71,7 @@ typedef enum IVAS_DEC_VOIP_MODE_VARIABLE_SPEED = 1 } IVAS_DEC_VOIP_MODE; #endif +#endif #ifdef DEBUGGING typedef enum _IVAS_DEC_FORCED_REND_MODE @@ -82,6 +83,16 @@ typedef enum _IVAS_DEC_FORCED_REND_MODE } IVAS_DEC_FORCED_REND_MODE; #endif +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +typedef enum _IVAS_DEC_PCM_TYPE +{ + IVAS_DEC_PCM_INT16, + IVAS_DEC_PCM_FLOAT, + IVAS_DEC_PCM_INVALID +} IVAS_DEC_PCM_TYPE; +#endif + + /* bitstream formats that can be consumed */ typedef enum _IVAS_DEC_BS_FORMAT { @@ -122,11 +133,17 @@ ivas_error IVAS_DEC_Configure( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ const uint32_t sampleRate, /* i : output sampling frequency */ const AUDIO_CONFIG outputConfig, /* i : audio configuration */ +#ifdef API_5MS + const int16_t tsmEnabled, /* i : enable TSM */ +#ifdef API_5MS_BASELINE + const int16_t enable5ms, /* i : enable 5ms rendering path */ +#endif +#endif const int16_t customLsOutputEnabled, /* i : enable custom loudspeaker setup handle */ const int16_t hrtfReaderEnabled, /* i : enable HRTF binary file input */ const int16_t enableHeadRotation, /* i : enable head rotation for binaural output */ const int16_t enableExternalOrientation, /* i : enable external orientations */ - const HEAD_ORIENT_TRK_T orientation_tracking, /* i : head orientation tracking type */ + const HEAD_ORIENT_TRK_T orientation_tracking, /* i : head orientation tracking type */ const int16_t renderConfigEnabled, /* i : enable Renderer config. file for binaural output */ const int16_t Opt_non_diegetic_pan, /* i : diegetic or not */ const float non_diegetic_pan_gain, /* i : non diegetic panning gain */ @@ -150,15 +167,39 @@ ivas_error IVAS_DEC_FeedFrame_Serial( /*! r: decoder error code */ ivas_error IVAS_DEC_GetSamples( +#ifdef API_5MS + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + int16_t nSamplesAsked, /* i: number of samples wanted by the caller */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const IVAS_DEC_PCM_TYPE pcmType, /* i : type for the decoded PCM resolution */ + void *pcmBuf, /* o : output synthesis signal */ +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif + int16_t *nOutSamples, /* o : number of samples per channel written to output buffer */ + bool *needNewFrame /* indication that the decoder needs a new frame */ +#else IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ int16_t *nOutSamples /* o : number of samples per channel written to output buffer */ -#ifdef SPLIT_REND_WITH_HEAD_ROT +#endif +#if defined SPLIT_REND_WITH_HEAD_ROT && !defined API_5MS , uint8_t *splitRendBitsBuf /* o : output split rendering bits */ #endif ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +ivas_error IVAS_DEC_GetSplitBinauralBitstream( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + void *pcmBuf_out, /* o : output synthesis signal for BINAURAL_SPLIT_PCM */ + IVAS_SPLIT_REND_BITS_HANDLE hSplitRendBits, /* o : bitstream output for split rendering mode */ + const int16_t nSamplesAsked, + int16_t *nOutSamples, /* o : number of samples per channel written to output buffer */ + bool *needNewFrame /* indication that the decoder needs a new frame */ +); +#endif + #ifdef SPLIT_REND_WITH_HEAD_ROT /*! r: decoder error code */ ivas_error IVAS_DEC_GetSplitRendBits( @@ -194,8 +235,13 @@ ivas_error IVAS_DEC_GetMasaMetadata( /*! r: error code */ ivas_error IVAS_DEC_FeedHeadTrackData( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ - IVAS_QUATERNION *orientation, /* i : head-tracking data */ - IVAS_VECTOR3 *Pos /* i : listener position */ +#ifdef API_5MS + IVAS_QUATERNION orientation, /* i : head-tracking data, listener orientation */ + IVAS_VECTOR3 Pos /* i : listener position */ +#else + IVAS_QUATERNION *orientation, /* i : head-tracking data, listener orientation */ + IVAS_VECTOR3 *Pos /* i : listener position */ +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , IVAS_SPLIT_REND_ROT_AXIS rot_axis /*i : external control for rotation axis for split rendering */ @@ -217,11 +263,19 @@ ivas_error IVAS_DEC_FeedRefVectorData( /*! r: error code */ ivas_error IVAS_DEC_FeedExternalOrientationData( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ - IVAS_QUATERNION *orientation, /* i : external orientation data */ - int8_t *enableHeadRotation, /* i : flag to enable head rotation for this frame */ - int8_t *enableExternalOrientation, /* i : flag to enable external orientation for this frame */ - int8_t *enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ - int16_t *numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#ifdef API_5MS + IVAS_QUATERNION orientation, /* i : external orientation data */ + int8_t enableHeadRotation, /* i : flag to enable head rotation for this frame */ + int8_t enableExternalOrientation, /* i : flag to enable external orientation for this frame */ + int8_t enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ + int16_t numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#else + IVAS_QUATERNION *orientation, /* i : external orientation data */ + int8_t *enableHeadRotation, /* i : flag to enable head rotation for this frame */ + int8_t *enableExternalOrientation, /* i : flag to enable external orientation for this frame */ + int8_t *enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ + int16_t *numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#endif ); /*! r: error code */ @@ -235,6 +289,13 @@ ivas_error IVAS_DEC_VoIP_FeedFrame( const bool qBit /* i : Q bit for AMR-WB IO */ ); +#ifdef API_5MS +ivas_error IVAS_DEC_VoIP_SetScale( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + const int16_t maxScaling, /* i : maximum allowed TSM scale */ + const int16_t scale /* i : TSM scale to set */ +); +#else #ifdef VARIABLE_SPEED_DECODING #ifdef DEBUGGING /*! r: error code */ @@ -244,25 +305,41 @@ ivas_error IVAS_DEC_VoIP_SetScale( ); #endif #endif +#endif /*! r: error code */ ivas_error IVAS_DEC_VoIP_GetSamples( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ uint16_t nSamplesPerChannel, /* i : number of samples per channel requested to be written to output buffer */ - int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ - const uint32_t systemTimestamp_ms, /* i : current system timestamp */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + IVAS_DEC_PCM_TYPE pcmType, + void *pcmBuf, +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif + const uint32_t systemTimestamp_ms /* i : current system timestamp */ +#ifndef API_5MS + , uint16_t *sampleAvailableNext /* o : samples available for the next call */ +#endif #ifdef SUPPORT_JBM_TRACEFILE , JbmTraceFileWriterFn jbmWriterFn, void* jbmWriter #endif ); -ivas_error IVAS_DEC_VoIP_Flush( +ivas_error IVAS_DEC_Flush( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ const int16_t nSamplesPerChannel, /* i : number of samples per channel requested to be written to output buffer */ - int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ - uint16_t *nSamplesAvailableNext, /* o : number of samples still available */ +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT + const IVAS_DEC_PCM_TYPE pcmType, + void *pcmBuf, +#else + int16_t *pcmBuf, /* i/o: buffer for decoded PCM output. The memory must already be allocated and be able to hold the expected number of output samples, based on frame size and number of output channels */ +#endif +#ifndef API_5MS + uint16_t *nSamplesAvailableNext, /* o : number of samples still available */ +#endif int16_t *nSamplesFlushed /* o : number of samples flushed */ ); @@ -271,14 +348,31 @@ ivas_error IVAS_DEC_VoIP_Flush( /*! r: error code */ ivas_error IVAS_DEC_EnableVoIP( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ +#ifndef API_5MS #ifdef VARIABLE_SPEED_DECODING const IVAS_DEC_VOIP_MODE voipMode, /* i : VoIP or variable speed */ const uint16_t speedFac, /* i : speed factor for variable speed */ +#endif #endif const int16_t jbmSafetyMargin, /* i : allowed delay reserve for JBM, in milliseconds */ const IVAS_DEC_INPUT_FORMAT inputFormat /* i : format of the input bitstream */ ); +#ifdef API_5MS +#ifdef SPLIT_REND_WITH_HEAD_ROT + +/*! r: error code */ +ivas_error IVAS_DEC_EnableSplitRendering( + IVAS_DEC_HANDLE hIvasDec /* i/o: IVAS decoder handle */ +); + +ivas_error IVAS_DEC_Get5msFlag( + IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ + int16_t *enable5ms /* o : 5ms flag */ +); +#endif +#endif + #ifdef DEBUGGING bool IVAS_DEC_GetBerDetectFlag( IVAS_DEC_HANDLE hIvasDec /* i : IVAS decoder handle */ @@ -363,6 +457,13 @@ ivas_error IVAS_DEC_GetRenderConfig( const IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render config handle */ ); +#ifdef API_5MS +/*! r: error code*/ +ivas_error IVAS_DEC_GetDefaultRenderConfig( + IVAS_RENDER_CONFIG_HANDLE hRCout /* o : Render config handle */ +); +#endif + /*! r: error code*/ ivas_error IVAS_DEC_FeedRenderConfig( IVAS_DEC_HANDLE hIvasDec, /* i/o: IVAS decoder handle */ diff --git a/lib_enc/ivas_ism_metadata_enc.c b/lib_enc/ivas_ism_metadata_enc.c index a01c2187bab9bbb76c6af415c3592f96ecb13db6..1dabd090fc1deb5538c5d87a9c72bfe8589e8744 100644 --- a/lib_enc/ivas_ism_metadata_enc.c +++ b/lib_enc/ivas_ism_metadata_enc.c @@ -636,7 +636,7 @@ ivas_error ivas_ism_metadata_enc( ism_total_brate_ref = *ism_total_brate; brate_limit_flag = calculate_brate_limit_flag( ism_imp, nchan_ism ); - bits_ism = (int16_t) ( *ism_total_brate / FRAMES_PER_SECOND ); + bits_ism = (int16_t) ( *ism_total_brate / FRAMES_PER_SEC ); set_s( bits_element, bits_ism / nchan_ism, nchan_ism ); bits_element[nchan_ism - 1] += bits_ism % nchan_ism; bitbudget_to_brate( bits_element, element_brate, nchan_ism ); diff --git a/lib_rend/ivas_crend.c b/lib_rend/ivas_crend.c index 20ed1a066ea29d113608d30a82208ec1886a0ba9..48f82a6a30bf32620c79346f0e5145188e396520 100644 --- a/lib_rend/ivas_crend.c +++ b/lib_rend/ivas_crend.c @@ -1713,13 +1713,21 @@ ivas_error ivas_rend_crendProcess( EFAP_HANDLE hEFAPdata, float *output[], /* i/o: input/output audio channels */ const int32_t output_Fs +#ifdef API_5MS + , + const int16_t num_subframes /* i : number of subframes to render */ +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , const int16_t pos_idx #endif ) { +#ifdef API_5MS + int16_t i, subframe_idx, subframe_len; +#else int16_t i, subframe_idx, output_frame, subframe_len; +#endif int16_t nchan_out; float pcm_tmp[BINAURAL_CHANNELS][L_FRAME48k]; float *p_pcm_tmp[BINAURAL_CHANNELS]; @@ -1738,6 +1746,12 @@ ivas_error ivas_rend_crendProcess( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + if ( hCombinedOrientationData->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { if ( hCombinedOrientationData->enableCombinedOrientation[subframe_idx] != 0 ) @@ -1746,6 +1760,7 @@ ivas_error ivas_rend_crendProcess( break; } } +#endif } push_wmops( "ivas_rend_crendProcess" ); @@ -1757,15 +1772,23 @@ ivas_error ivas_rend_crendProcess( return error; } +#ifdef API_5MS + subframe_len = (int16_t) ( output_Fs / ( FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES ) ); +#else output_frame = (int16_t) ( output_Fs / FRAMES_PER_SEC ); subframe_len = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; +#endif for ( i = 0; i < BINAURAL_CHANNELS; i++ ) { p_pcm_tmp[i] = pcm_tmp[i]; } +#ifdef API_5MS + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) +#endif { /* Early Reflections */ if ( hCrend->reflections != NULL ) @@ -1831,7 +1854,11 @@ ivas_error ivas_rend_crendProcess( /* move to output */ for ( i = 0; i < nchan_out; i++ ) { +#ifdef API_5MS + mvr2r( pcm_tmp[i], output[i], num_subframes * subframe_len ); +#else mvr2r( pcm_tmp[i], output[i], output_frame ); +#endif } pop_wmops(); @@ -1880,6 +1907,12 @@ ivas_error ivas_rend_crendProcessSubframe( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + if ( hCombinedOrientationData->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { if ( hCombinedOrientationData->enableCombinedOrientation[subframe_idx] != 0 ) @@ -1888,6 +1921,7 @@ ivas_error ivas_rend_crendProcessSubframe( break; } } +#endif } push_wmops( "ivas_rend_crendProcessSubframe" ); @@ -2038,7 +2072,10 @@ ivas_error ivas_rend_crendProcessSplitBin( float *output[], const int32_t output_Fs ) { - int16_t i, j, sf; + int16_t i, j; +#ifndef API_5MS + int16_t sf; +#endif int16_t pos_idx, output_frame; ivas_error error; float gain_lfe; @@ -2067,6 +2104,16 @@ ivas_error ivas_rend_crendProcessSplitBin( combinedOrientationDataLocal = *pCombinedOrientationDataLocal; if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { +#ifdef API_5MS + combinedOrientationDataLocal.Quaternion = combinedOrientationDataLocal.Quaternion; + for ( i = 0; i < 3; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + combinedOrientationDataLocal.Rmat[i][j] = combinedOrientationDataLocal.Rmat[i][j]; + } + } +#else for ( sf = 1; sf < RENDERER_HEAD_POSITIONS_PER_FRAME; ++sf ) { combinedOrientationDataLocal.Quaternions[sf] = combinedOrientationDataLocal.Quaternions[0]; @@ -2078,6 +2125,7 @@ ivas_error ivas_rend_crendProcessSplitBin( } } } +#endif } /* copy LFE to tmpLfeBuffer and apply gain only once */ @@ -2095,6 +2143,18 @@ ivas_error ivas_rend_crendProcessSplitBin( for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; ++pos_idx ) { /* Update head positions */ +#ifdef API_5MS + IVAS_QUATERNION Quaternion_orig, Quaternion_abs; + Quaternion_orig = combinedOrientationDataLocal.Quaternion; + Quaternion_abs.w = -3.0f; + Quat2EulerDegree( combinedOrientationDataLocal.Quaternion, &Quaternion_abs.z, &Quaternion_abs.y, &Quaternion_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + Quaternion_abs.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternion_abs.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternion_abs.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; + combinedOrientationDataLocal.Quaternion = Quaternion_abs; + QuatToRotMat( combinedOrientationDataLocal.Quaternion, combinedOrientationDataLocal.Rmat ); +#else IVAS_QUATERNION Quaternions_orig[RENDERER_HEAD_POSITIONS_PER_FRAME], Quaternions_abs; for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) { @@ -2108,6 +2168,7 @@ ivas_error ivas_rend_crendProcessSplitBin( combinedOrientationDataLocal.Quaternions[i] = Quaternions_abs; QuatToRotMat( combinedOrientationDataLocal.Quaternions[i], combinedOrientationDataLocal.Rmat[i] ); } +#endif /* render inplace to first two channels of tmpInputBuffer */ pCombinedOrientationDataLocal = &combinedOrientationDataLocal; @@ -2116,7 +2177,19 @@ ivas_error ivas_rend_crendProcessSplitBin( { mvr2r( hCombinedOrientationData->Rmat_prev[pos_idx][i], pCombinedOrientationDataLocal->Rmat_prev[0][i], 3 ); } - if ( ( error = ivas_rend_crendProcess( pCrend, inConfig, outConfig, hDecoderConfig, pCombinedOrientationDataLocal, hIntSetup, hEFAPdata, p_tmpInputBuffer, output_Fs, pos_idx ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_rend_crendProcess( pCrend, + inConfig, + outConfig, + hDecoderConfig, + pCombinedOrientationDataLocal, + hIntSetup, + hEFAPdata, + p_tmpInputBuffer, + output_Fs, +#ifdef API_5MS + 4, +#endif + pos_idx ) ) != IVAS_ERR_OK ) { return error; } @@ -2142,10 +2215,14 @@ ivas_error ivas_rend_crendProcessSplitBin( } /* restore original headrotation data */ +#ifdef API_5MS + combinedOrientationDataLocal.Quaternion = Quaternion_orig; +#else for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) { combinedOrientationDataLocal.Quaternions[i] = Quaternions_orig[i]; } +#endif } /* copy split binaural rendered signals to final output */ diff --git a/lib_rend/ivas_dirac_dec_binaural_functions.c b/lib_rend/ivas_dirac_dec_binaural_functions.c index b378236c8991d74f65919a4c91110f590275d601..04993164046fe408263e8d9e92bbf64f29a65209 100644 --- a/lib_rend/ivas_dirac_dec_binaural_functions.c +++ b/lib_rend/ivas_dirac_dec_binaural_functions.c @@ -313,7 +313,15 @@ ivas_error ivas_dirac_dec_init_binaural_data( #endif /* allocate transport channels*/ - if ( st_ivas->hDecoderConfig->voip_active == 1 && st_ivas->hTcBuffer == NULL ) + if ( +#ifdef API_5MS +#ifdef API_5MS_BASELINE + st_ivas->hDecoderConfig->Opt_5ms && +#endif +#else + st_ivas->hDecoderConfig->voip_active == 1 && +#endif + st_ivas->hTcBuffer == NULL ) { int16_t nchan_to_allocate; int16_t n_samples_granularity; @@ -558,6 +566,10 @@ void ivas_dirac_dec_binaural( COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ const int16_t nchan_transport /* i : number of transport channels */ +#ifdef API_5MS + , + const int16_t num_subframes /* i : number of subframes to render */ +#endif ) { int16_t subframe; @@ -626,7 +638,11 @@ void ivas_dirac_dec_binaural( generate_masking_noise_lb_dirac( st->hFdCngDec->hFdCngCom, st_ivas->hTcBuffer->tc[nchan_transport], DEFAULT_JBM_CLDFB_TIMESLOTS, st->cna_dirac_flag && st->flag_cna ); } +#ifdef API_5MS + for ( subframe = 0; subframe < num_subframes; subframe++ ) +#else for ( subframe = 0; subframe < MAX_PARAM_SPATIAL_SUBFRAMES; subframe++ ) +#endif { int16_t n_samples_sf = slot_size * hSpatParamRendCom->subframe_nbslots[subframe]; @@ -852,7 +868,11 @@ static void ivas_dirac_dec_binaural_internal( { for ( j = 0; j < 3; j++ ) { +#ifdef API_5MS + Rmat[i][j] = hCombinedOrientationData->Rmat[i][j]; +#else Rmat[i][j] = hCombinedOrientationData->Rmat[subframe][i][j]; +#endif } } @@ -870,7 +890,12 @@ static void ivas_dirac_dec_binaural_internal( #ifndef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN ivas_dirac_dec_binaural_formulate_input_and_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, Rmat, subframe, - hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, st_ivas->hMasaIsmData ); +#ifdef API_5MS + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation > 0, +#else + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, +#endif + st_ivas->hMasaIsmData ); #endif if ( config_data.ivas_format == ISM_FORMAT ) @@ -891,7 +916,11 @@ static void ivas_dirac_dec_binaural_internal( ivas_dirac_dec_binaural_formulate_input_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Cldfb_RealBuffer_in, Cldfb_ImagBuffer_in, subframe, subFrameTotalEne, IIReneLimiter ); ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat, subframe, +#ifdef API_5MS + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation > 0, +#else hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, +#endif subFrameTotalEne, IIReneLimiter, st_ivas->hMasaIsmData ); #endif @@ -906,7 +935,11 @@ static void ivas_dirac_dec_binaural_internal( } ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat, subframe, +#ifdef API_5MS + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation > 0, +#else hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, +#endif nchanSeparateChannels, st_ivas->hMasaIsmData ); #ifdef SPLIT_REND_WITH_HEAD_ROT_PARAMBIN @@ -952,7 +985,11 @@ static void ivas_dirac_dec_binaural_internal( if ( hCombinedOrientationData ) { +#ifdef API_5MS + Quaternions_ref = &hCombinedOrientationData->Quaternion; +#else Quaternions_ref = &hCombinedOrientationData->Quaternions[0]; +#endif Quaternions_rot.w = -3.0f; /* signal to use Euler */ Quaternions_abs.w = -3.0f; /* signal to use Euler */ Quat2EulerDegree( *Quaternions_ref, &Quaternions_abs.z, &Quaternions_abs.y, &Quaternions_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ @@ -981,11 +1018,20 @@ static void ivas_dirac_dec_binaural_internal( mvr2r( st_ivas->hDiracDecBin[0]->ChCrossIm, hDiracDecBin->ChCrossIm, hSpatParamRendCom->num_freq_bands ); ivas_dirac_dec_binaural_formulate_target_covariance_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, Rmat_local, subframe, +#ifdef API_5MS + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation > 0, +#else hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, +#endif subFrameTotalEne, IIReneLimiter, st_ivas->hMasaIsmData ); ivas_dirac_dec_binaural_determine_processing_matrices( hDiracDecBin, hSpatParamRendCom, &config_data, max_band_decorr, Rmat_local, subframe, - hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, nchanSeparateChannels, st_ivas->hMasaIsmData ); +#ifdef API_5MS + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation > 0, +#else + hCombinedOrientationData && hCombinedOrientationData->enableCombinedOrientation[subframe] > 0, +#endif + nchanSeparateChannels, st_ivas->hMasaIsmData ); /* re-use reverb and decorr from main direction for the sides */ diff --git a/lib_rend/ivas_lcld_decoder.c b/lib_rend/ivas_lcld_decoder.c index 7e9734e763b20eacbea18a892cebc18476aad943..f776e2d5fdb95d472edb83411773f9d80f2e42fe 100644 --- a/lib_rend/ivas_lcld_decoder.c +++ b/lib_rend/ivas_lcld_decoder.c @@ -714,7 +714,15 @@ void DeleteLCLDDecoder( LCLDDecoder *psLCLDDecoder ) } #endif - DeletePredictionDecoder( psLCLDDecoder->psPredictionDecoder ); +#ifdef FIX_CQMFPREDDEC_FREE + if ( psLCLDDecoder->psPredictionDecoder != NULL ) + { +#endif + DeletePredictionDecoder( psLCLDDecoder->psPredictionDecoder ); +#ifdef FIX_CQMFPREDDEC_FREE + psLCLDDecoder->psPredictionDecoder = NULL; + } +#endif if ( psLCLDDecoder->psNoiseGen != NULL ) { diff --git a/lib_rend/ivas_objectRenderer.c b/lib_rend/ivas_objectRenderer.c index e1da3aaa414f81ccdafd3a2fc421331da50aa0a2..d44b484cac6441bcd347e120c504b559c2bb20d4 100644 --- a/lib_rend/ivas_objectRenderer.c +++ b/lib_rend/ivas_objectRenderer.c @@ -273,6 +273,10 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t ism_md_subframe_update, /* i : Number of subframes to delay ism metadata to sync with audio */ float *output[], /* i/o: SCE channels / Binaural synthesis */ const int16_t output_frame /* i : output frame length */ +#ifdef API_5MS + , + const int16_t num_subframes /* i : number of subframes to render */ +#endif ) { int16_t subframe_length; @@ -288,7 +292,11 @@ ivas_error ivas_td_binaural_renderer_unwrap( p_reverb_signal[ch] = reverb_signal[ch]; } +#ifdef API_5MS + subframe_length = output_frame / num_subframes; +#else subframe_length = output_frame / MAX_PARAM_SPATIAL_SUBFRAMES; +#endif c_indx = 0; for ( nS = 0; nS < num_src; nS++ ) @@ -301,7 +309,11 @@ ivas_error ivas_td_binaural_renderer_unwrap( } } +#ifdef API_5MS + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) +#endif { if ( subframe_idx == ism_md_subframe_update ) { @@ -309,8 +321,11 @@ ivas_error ivas_td_binaural_renderer_unwrap( TDREND_Update_object_positions( hBinRendererTd, num_src, ivas_format, hIsmMetaData ); } /* Update the listener's location/orientation */ +#ifdef API_5MS + TDREND_Update_listener_orientation( hBinRendererTd, ( enableCombinedOrientation != NULL ) ? *enableCombinedOrientation : 0, ( Quaternions != NULL ) ? Quaternions : NULL, ( Pos != NULL ) ? Pos : NULL ); +#else TDREND_Update_listener_orientation( hBinRendererTd, ( enableCombinedOrientation != NULL ) ? enableCombinedOrientation[subframe_idx] : 0, ( Quaternions != NULL ) ? &Quaternions[subframe_idx] : NULL, ( Pos != NULL ) ? &Pos[subframe_idx] : NULL ); - +#endif if ( hReverb != NULL ) { if ( ( error = ivas_reverb_process( hReverb, transport_config, 0, output, p_reverb_signal, subframe_idx ) ) != IVAS_ERR_OK ) @@ -635,8 +650,11 @@ ivas_error ivas_td_binaural_renderer_ext( const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ const REVERB_HANDLE hReverb, /* i : Reverberator handle */ const int16_t ism_md_subframe_update_ext, /* i : Metadata Delay in subframes to sync with audio delay */ - const int16_t output_frame, /* i : output frame length */ - float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ +#ifdef API_5MS + const int32_t output_Fs, /* i : output sampling rate */ +#endif + const int16_t output_frame, /* i : output frame length */ + float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ) { ISM_METADATA_FRAME hIsmMetaDataFrame; @@ -694,10 +712,21 @@ ivas_error ivas_td_binaural_renderer_ext( } if ( ( error = ivas_td_binaural_renderer_unwrap( hReverb, transport_config, pTDRend->hBinRendererTd, num_src, lfe_idx, ivas_format, hIsmMetaData, +#ifdef API_5MS + ( hCombinedOrientationData != NULL ) ? &( *hCombinedOrientationData )->enableCombinedOrientation : NULL, + ( hCombinedOrientationData != NULL ) ? &( *hCombinedOrientationData )->Quaternion : NULL, + ( hCombinedOrientationData != NULL ) ? &( *hCombinedOrientationData )->listenerPos : NULL, +#else ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->enableCombinedOrientation : NULL, ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->Quaternions : NULL, ( hCombinedOrientationData != NULL ) ? ( *hCombinedOrientationData )->listenerPos : NULL, - ism_md_subframe_update_ext, p_output, output_frame ) ) != IVAS_ERR_OK ) +#endif + ism_md_subframe_update_ext, p_output, output_frame +#ifdef API_5MS + , + (int16_t) ( ( output_frame * FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES ) / output_Fs ) +#endif + ) ) != IVAS_ERR_OK ) { return error; } @@ -726,7 +755,11 @@ ivas_error ObjRenderIvasFrame_splitBinaural( float tmpBinaural[MAX_HEAD_ROT_POSES * 2][L_FRAME48k]; float *p_tmpProcessing[MAX_OUTPUT_CHANNELS]; int16_t pos_idx; +#ifdef API_5MS + IVAS_QUATERNION originalHeadRot; +#else IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES]; +#endif MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; BINAURAL_TD_OBJECT_RENDERER_HANDLE tmpTdRendHandle; ivas_error error; @@ -758,10 +791,14 @@ ivas_error ObjRenderIvasFrame_splitBinaural( } /* Save current head positions */ +#ifdef API_5MS + originalHeadRot = st_ivas->hCombinedOrientationData->Quaternion; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { originalHeadRot[i] = st_ivas->hCombinedOrientationData->Quaternions[i]; } +#endif /* Copy input audio to a processing buffer. Cannot render in-place because binaurally rendered * audio would overwrite original material, which is still needed for rendering next head pose. */ @@ -775,6 +812,26 @@ ivas_error ObjRenderIvasFrame_splitBinaural( /* Update head positions */ if ( pos_idx != 0 ) { +#ifdef API_5MS + if ( originalHeadRot.w == -3.0f ) + { + st_ivas->hCombinedOrientationData->Quaternion.w = -3.0f; + st_ivas->hCombinedOrientationData->Quaternion.x = originalHeadRot.x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; + st_ivas->hCombinedOrientationData->Quaternion.y = originalHeadRot.y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; + st_ivas->hCombinedOrientationData->Quaternion.z = originalHeadRot.z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; + } + else + { + st_ivas->hCombinedOrientationData->Quaternion.w = -3.0f; + Quat2EulerDegree( originalHeadRot, /* TODO tmu : fix bug with ordering*/ + &st_ivas->hCombinedOrientationData->Quaternion.z, + &st_ivas->hCombinedOrientationData->Quaternion.y, + &st_ivas->hCombinedOrientationData->Quaternion.x ); + st_ivas->hCombinedOrientationData->Quaternion.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + st_ivas->hCombinedOrientationData->Quaternion.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + st_ivas->hCombinedOrientationData->Quaternion.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; + } +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { if ( originalHeadRot[i].w == -3.0f ) @@ -798,6 +855,7 @@ ivas_error ObjRenderIvasFrame_splitBinaural( st_ivas->hCombinedOrientationData->Quaternions[i].z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; } } +#endif } /* Handle the 1 ISM case where there is only one channel in the input buffer */ @@ -845,10 +903,14 @@ ivas_error ObjRenderIvasFrame_splitBinaural( } /* Restore original head rotation */ +#ifdef API_5MS + st_ivas->hCombinedOrientationData->Quaternion = originalHeadRot; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { st_ivas->hCombinedOrientationData->Quaternions[i] = originalHeadRot[i]; } +#endif pop_wmops(); diff --git a/lib_rend/ivas_prot_rend.h b/lib_rend/ivas_prot_rend.h index 3a56fd3993ff5c42fd83460031ad0c60492e206d..f97f7279ebe510588ce3f35078cc22c94f7ef705 100644 --- a/lib_rend/ivas_prot_rend.h +++ b/lib_rend/ivas_prot_rend.h @@ -162,6 +162,9 @@ void ivas_dirac_dec_binaural( COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i : combined orientation handle */ float *output_f[], /* i/o: synthesized core-coder transport channels/DirAC output */ const int16_t nchan_transport /* i : number of transport channels */ +#ifdef API_5MS + ,const int16_t num_subframes /* i : number of subframes to render */ +#endif ); void ivas_dirac_dec_binaural_render( @@ -560,6 +563,9 @@ ivas_error ivas_td_binaural_renderer_unwrap( const int16_t ism_md_subframe_update, float *output[], /* i/o: SCE channels / Binaural synthesis */ const int16_t output_frame /* i : output frame length */ +#ifdef API_5MS + ,const int16_t num_subframes /* i : number of subframes to render */ +#endif ); ivas_error ivas_td_binaural_renderer_ext( @@ -570,6 +576,9 @@ ivas_error ivas_td_binaural_renderer_ext( const IVAS_REND_AudioObjectPosition *currentPos, /* i : Object position */ const REVERB_HANDLE hReverb, /* i : Reverberator handle */ const int16_t ism_md_subframe_update_ext, /* i : Metadata Delay in subframes to sync with audio delay */ +#ifdef API_5MS + const int32_t output_Fs, /* i : output sampling rate */ +#endif const int16_t output_frame, /* i : output frame length */ float output[][L_FRAME48k] /* i/o: SCE channels / Binaural synthesis */ ); @@ -850,6 +859,9 @@ ivas_error ivas_rend_crendProcess( EFAP_HANDLE hEFAPdata, float *output[], /* i/o: input/output audio channels */ const int32_t output_Fs +#ifdef API_5MS + ,const int16_t num_subframes /* i : number of subframes to render */ +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , const int16_t pos_idx @@ -1453,9 +1465,16 @@ void ivas_renderSplitUpdateNoCorrectionPoseData( ivas_error ivas_renderMultiBinToSplitBinaural( SPLIT_REND_WRAPPER *hSplitBin, +#ifdef API_5MS + const IVAS_QUATERNION headPosition, +#else const IVAS_QUATERNION headPositions[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif const int32_t SplitRendBitRate, IVAS_SPLIT_REND_CODEC splitCodec, + #ifdef API_5MS + int16_t codec_frame_size_ms, + #endif ivas_split_rend_bits_t *pBits, float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], @@ -1468,7 +1487,11 @@ ivas_error ivas_renderMultiBinToSplitBinaural( void ivas_rend_CldfbSplitPreRendProcess( const BIN_HR_SPLIT_PRE_REND_HANDLE hBinHrSplitPreRend, +#ifdef API_5MS + const IVAS_QUATERNION headPosition, +#else const IVAS_QUATERNION headPositions[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], @@ -1480,9 +1503,15 @@ void ivas_rend_CldfbSplitPreRendProcess( void ivas_rend_CldfbSplitPostRendProcess( BIN_HR_SPLIT_POST_REND_HANDLE hBinHrSplitPostRend, MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, +#ifdef API_5MS + const IVAS_QUATERNION QuaternionPost, + float Cldfb_RealBuffer_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], + float Cldfb_ImagBuffer_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], +#else const IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES], float Cldfb_RealBuffer_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_ImagBuffer_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], +#endif float output[][L_FRAME48k], const int16_t is_cldfb_in ); @@ -1556,10 +1585,16 @@ void ivas_SplitRenderer_GetRotMd( void ivas_SplitRenderer_PostRenderer( BIN_HR_SPLIT_POST_REND_HANDLE hBinPostRenderer, /* i/o: binaural renderer handle */ MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, +#ifdef API_5MS + float Cldfb_RealBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */ + float Cldfb_ImagBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */ + const IVAS_QUATERNION Quaternion_act +#else float Cldfb_RealBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o: Reference/out Binaural signals */ float Cldfb_ImagBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o: Reference/out Binaural signals */ const IVAS_QUATERNION Quaternions_act[MAX_PARAM_SPATIAL_SUBFRAMES] -); +#endif + ); #endif @@ -1714,6 +1749,10 @@ void ivas_rend_CldfbMultiBinRendProcess( float Cldfb_Out_Real[MAX_HEAD_ROT_POSES*BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* o : Binaural signals */ float Cldfb_Out_Imag[MAX_HEAD_ROT_POSES*BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], const int16_t low_res_pre_rend_rot +#ifdef API_5MS + , + int16_t num_subframes +#endif ); ivas_error ivas_rend_openCldfb( @@ -1787,7 +1826,12 @@ int32_t ivas_split_rend_bitstream_read_int32( ); IVAS_QUATERNION ivas_split_rend_get_sf_rot_data( +#ifdef API_5MS + /* TODO(splitrend): clean up */ + const IVAS_QUATERNION headPositions[1], +#else const IVAS_QUATERNION headPositions[RENDERER_HEAD_POSITIONS_PER_FRAME], +#endif int16_t subframe_idx ); @@ -1808,6 +1852,9 @@ int32_t ivas_get_split_rend_md_target_brate( int32_t ivas_get_lc3plus_bitrate( const int32_t SplitRendBitRate, const IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode +#ifdef API_5MS +, int16_t split_prerender_frame_size_ms +#endif ); int8_t ivas_get_lc3plus_bitrate_id( @@ -1817,6 +1864,9 @@ int8_t ivas_get_lc3plus_bitrate_id( int32_t ivas_get_lc3plus_size_from_id( const int8_t SplitRendBitRateId, const IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode +#ifdef API_5MS +, int16_t split_prerender_frame_size_ms +#endif ); ivas_error ivas_split_rend_validate_config( @@ -1841,8 +1891,16 @@ void ivas_split_rend_get_quant_params( int16_t *num_complex_bands ); -void ivas_split_rend_choose_default_codec( +#ifdef API_5MS +ivas_error +#else +void +#endif +ivas_split_rend_choose_default_codec( IVAS_SPLIT_REND_CODEC *pCodec, /* i/o: pointer to codec setting */ +#ifdef API_5MS + int16_t *pCodec_frame_size_ms, /* i/o: pointer to codec frame size setting */ +#endif const int16_t isRenderingInTd, /* i : flag: is rendering done in TD? */ const int16_t pcm_out /* i : flag to indicate PCM output */ ); diff --git a/lib_rend/ivas_render_config.c b/lib_rend/ivas_render_config.c index 0d6d5f389bd72a6ae7ab70d164c5fea3c81d07f6..5f12c7521447ed3270108ec9380396abd7dbb0ea 100644 --- a/lib_rend/ivas_render_config.c +++ b/lib_rend/ivas_render_config.c @@ -136,6 +136,9 @@ ivas_error ivas_render_config_init_from_rom( ( *hRenderConfig )->split_rend_config.dof = 3; ( *hRenderConfig )->split_rend_config.hq_mode = 0; ( *hRenderConfig )->split_rend_config.codec_delay_ms = 0; +#ifdef API_5MS + ( *hRenderConfig )->split_rend_config.codec_frame_size_ms = 0; /* 0 means "use default for selected codec" */ +#endif ( *hRenderConfig )->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; ( *hRenderConfig )->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB; ( *hRenderConfig )->split_rend_config.rendererSelection = IVAS_SPLIT_REND_RENDERER_SELECTION_DEFAULT; diff --git a/lib_rend/ivas_rotation.c b/lib_rend/ivas_rotation.c index 388617a53224cf579934e17b916221d60b5318d2..a46c0a971924dfe663b678a837ee477440528fdb 100644 --- a/lib_rend/ivas_rotation.c +++ b/lib_rend/ivas_rotation.c @@ -48,15 +48,26 @@ * Local funtion declarations *-----------------------------------------------------------------------*/ -static ivas_error combine_external_and_head_orientations( IVAS_QUATERNION *headRotQuaternions, IVAS_VECTOR3 *listenerPos, +static ivas_error combine_external_and_head_orientations( + IVAS_QUATERNION *headRotQuaternions, + IVAS_VECTOR3 *listenerPos, #ifdef SPLIT_REND_WITH_HEAD_ROT - IVAS_SPLIT_REND_ROT_AXIS sr_pose_pred_axis, /* i : split rend pose prediction axis*/ + IVAS_SPLIT_REND_ROT_AXIS sr_pose_pred_axis, /* i : split rend pose prediction axis*/ +#endif +#ifndef API_5MS + int16_t numHeadRotQuaternions, #endif - int16_t numHeadRotQuaternions, - EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData ); + EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData ); -static void external_target_interpolation( EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, const int16_t i ); +static void external_target_interpolation( + EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData +#ifndef API_5MS + , + const int16_t i +#endif +); static bool are_orientations_same( const IVAS_QUATERNION *orientation1, const IVAS_QUATERNION *orientation2 ); @@ -81,7 +92,9 @@ ivas_error ivas_headTrack_open( } /* Initialization */ +#ifndef API_5MS ( *hHeadTrackData )->num_quaternions = 0; +#endif ( *hHeadTrackData )->lrSwitchInterpVal = 0.0f; ( *hHeadTrackData )->lrSwitchedCurrent = 0; ( *hHeadTrackData )->lrSwitchedNext = 0; @@ -384,14 +397,18 @@ void rotateFrame_shd( set_zero( SHrotmat[i], HEADROT_SHMAT_DIM ); } - /* calculate ambisonics rotation matrices for the previous and current frames */ #ifdef SPLIT_REND_WITH_HEAD_ROT SHrotmatgen( SHrotmat_prev, hCombinedOrientationData->Rmat_prev[0], shd_rot_max_order ); #else SHrotmatgen( SHrotmat_prev, hCombinedOrientationData->Rmat_prev, shd_rot_max_order ); #endif +#ifdef API_5MS + SHrotmatgen( SHrotmat, hCombinedOrientationData->Rmat, shd_rot_max_order ); +#else SHrotmatgen( SHrotmat, hCombinedOrientationData->Rmat[subframe_idx], shd_rot_max_order ); +#endif + for ( i = 0; i < subframe_len; i++ ) { @@ -443,11 +460,18 @@ void rotateFrame_shd( /* move Rmat to Rmat_prev */ for ( i = 0; i < 3; i++ ) { + mvr2r( +#ifdef API_5MS + hCombinedOrientationData->Rmat[i], +#else + hCombinedOrientationData->Rmat[subframe_idx][i], +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT - mvr2r( hCombinedOrientationData->Rmat[subframe_idx][i], hCombinedOrientationData->Rmat_prev[0][i], 3 ); + hCombinedOrientationData->Rmat_prev[0][i], #else - mvr2r( hCombinedOrientationData->Rmat[subframe_idx][i], hCombinedOrientationData->Rmat_prev[i], 3 ); + hCombinedOrientationData->Rmat_prev[i], #endif + 3 ); } return; @@ -540,7 +564,11 @@ void rotateFrame_sd( /* gains for current subframe rotation */ +#ifdef API_5MS + rotateAziEle( hTransSetup.ls_azimuth[ch_in_woLFE], hTransSetup.ls_elevation[ch_in_woLFE], &azimuth, &elevation, hCombinedOrientationData->Rmat, hTransSetup.is_planar_setup ); +#else rotateAziEle( hTransSetup.ls_azimuth[ch_in_woLFE], hTransSetup.ls_elevation[ch_in_woLFE], &azimuth, &elevation, hCombinedOrientationData->Rmat[subframe_idx], hTransSetup.is_planar_setup ); +#endif if ( hEFAPdata != NULL && ( hTransSetup.ls_azimuth[ch_in_woLFE] != azimuth || hTransSetup.ls_elevation[ch_in_woLFE] != elevation ) ) { efap_determine_gains( hEFAPdata, tmp_gains, azimuth, elevation, EFAP_MODE_EFAP ); @@ -577,11 +605,18 @@ void rotateFrame_sd( /* move Rmat to Rmat_prev */ for ( i = 0; i < 3; i++ ) { + mvr2r( +#ifdef API_5MS + hCombinedOrientationData->Rmat[i], +#else + hCombinedOrientationData->Rmat[subframe_idx][i], +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT - mvr2r( hCombinedOrientationData->Rmat[subframe_idx][i], hCombinedOrientationData->Rmat_prev[0][i], 3 ); + hCombinedOrientationData->Rmat_prev[0][i], #else - mvr2r( hCombinedOrientationData->Rmat[subframe_idx][i], hCombinedOrientationData->Rmat_prev[i], 3 ); + hCombinedOrientationData->Rmat_prev[i], #endif + 3 ); } /* copy to output */ @@ -810,7 +845,9 @@ ivas_error ivas_external_orientation_open( EXTERNAL_ORIENTATION_HANDLE *hExtOrientationData /* o : external orientation handle */ ) { +#ifndef API_5MS int16_t i; +#endif IVAS_QUATERNION identity; identity.w = 1.0f; @@ -823,6 +860,13 @@ ivas_error ivas_external_orientation_open( } /* Enable head rotation and disable external orientation as default */ +#ifdef API_5MS + ( *hExtOrientationData )->enableHeadRotation = 1; + ( *hExtOrientationData )->enableExternalOrientation = 0; + ( *hExtOrientationData )->enableRotationInterpolation = 0; + ( *hExtOrientationData )->numFramesToTargetOrientation = 0; + ( *hExtOrientationData )->Quaternion = identity; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { ( *hExtOrientationData )->enableHeadRotation[i] = 1; @@ -831,7 +875,7 @@ ivas_error ivas_external_orientation_open( ( *hExtOrientationData )->numFramesToTargetOrientation[i] = 0; ( *hExtOrientationData )->Quaternions[i] = identity; } - +#endif return IVAS_ERR_OK; } @@ -868,7 +912,10 @@ ivas_error ivas_combined_orientation_open( COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData /* o : combined orientation handle */ ) { - int16_t i, j; +#ifndef API_5MS + int16_t i; +#endif + int16_t j; IVAS_QUATERNION identity; IVAS_VECTOR3 origo; #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -888,7 +935,11 @@ ivas_error ivas_combined_orientation_open( /* Initialization */ ( *hCombinedOrientationData )->interpolationCoefficient = 1.0f; ( *hCombinedOrientationData )->interpolationIncrement = 1.0f; +#ifdef API_5MS + ( *hCombinedOrientationData )->maximumFramesToTargetOrientation = 2000; +#else ( *hCombinedOrientationData )->maximumFramesToTargetOrientation = 500; +#endif ( *hCombinedOrientationData )->lrSwitchedNext = 0; ( *hCombinedOrientationData )->lrSwitchedCurrent = 0; ( *hCombinedOrientationData )->lrSwitchInterpVal = 0.0f; @@ -896,6 +947,17 @@ ivas_error ivas_combined_orientation_open( ( *hCombinedOrientationData )->Quaternions_ext_interpolation_start = identity; ( *hCombinedOrientationData )->Quaternions_ext_interpolation_target = identity; +#ifdef API_5MS + ( *hCombinedOrientationData )->enableCombinedOrientation = 0; + ( *hCombinedOrientationData )->Quaternion = identity; + ( *hCombinedOrientationData )->listenerPos = origo; + + for ( j = 0; j < 3; j++ ) + { + set_zero( ( *hCombinedOrientationData )->Rmat[j], 3 ); + ( *hCombinedOrientationData )->Rmat[j][j] = 1.0f; + } +#else /* API_5MS */ /* Initialise orientations to identity */ for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { @@ -909,6 +971,7 @@ ivas_error ivas_combined_orientation_open( ( *hCombinedOrientationData )->Rmat[i][j][j] = 1.0f; } } +#endif /* API_5MS */ #ifdef SPLIT_REND_WITH_HEAD_ROT for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) @@ -919,6 +982,7 @@ ivas_error ivas_combined_orientation_open( ( *hCombinedOrientationData )->Rmat_prev[pos_idx][j][j] = 1.0f; } } + ( *hCombinedOrientationData )->sr_pose_pred_axis = DEFAULT_AXIS; #else for ( j = 0; j < 3; j++ ) { @@ -981,18 +1045,29 @@ ivas_error combine_external_and_head_orientations_dec( #ifdef SPLIT_REND_WITH_HEAD_ROT IVAS_SPLIT_REND_ROT_AXIS sr_pose_pred_axis; #endif +#ifdef API_5MS + IVAS_QUATERNION *pHeadRotQuaternion = NULL; +#else IVAS_QUATERNION *headRotQuaternions = NULL; +#endif IVAS_VECTOR3 *listenerPos = NULL; +#ifndef API_5MS int16_t numHeadRotQuaternions = 0; +#endif if ( hHeadTrackData != NULL ) { +#ifdef API_5MS + pHeadRotQuaternion = &hHeadTrackData->Quaternion; + listenerPos = &hHeadTrackData->Pos; +#else numHeadRotQuaternions = hHeadTrackData->num_quaternions; if ( hHeadTrackData->num_quaternions >= 0 ) { headRotQuaternions = hHeadTrackData->Quaternions; listenerPos = hHeadTrackData->Pos; } +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT sr_pose_pred_axis = hHeadTrackData->sr_pose_pred_axis; #endif @@ -1004,11 +1079,21 @@ ivas_error combine_external_and_head_orientations_dec( } #endif - return combine_external_and_head_orientations( headRotQuaternions, listenerPos, + return combine_external_and_head_orientations( +#ifdef API_5MS + pHeadRotQuaternion, +#else + headRotQuaternions, +#endif + listenerPos, #ifdef SPLIT_REND_WITH_HEAD_ROT - sr_pose_pred_axis, + sr_pose_pred_axis, #endif - numHeadRotQuaternions, hExtOrientationData, hCombinedOrientationData ); +#ifndef API_5MS + numHeadRotQuaternions, +#endif + hExtOrientationData, + hCombinedOrientationData ); } @@ -1029,8 +1114,10 @@ ivas_error combine_external_and_head_orientations_rend( #endif IVAS_QUATERNION *headRotQuaternions = NULL; IVAS_VECTOR3 *listenerPos = NULL; +#ifndef API_5MS int16_t numHeadRotQuaternions = 0; int16_t i; +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT sr_pose_pred_axis = DEFAULT_AXIS; @@ -1039,8 +1126,13 @@ ivas_error combine_external_and_head_orientations_rend( { if ( hHeadTrackData->headRotEnabled ) { +#ifdef API_5MS + headRotQuaternions = &hHeadTrackData->headPosition; + listenerPos = &hHeadTrackData->Pos; +#else headRotQuaternions = hHeadTrackData->headPositions; listenerPos = hHeadTrackData->Pos; +#endif } #ifdef SPLIT_REND_WITH_HEAD_ROT sr_pose_pred_axis = hHeadTrackData->sr_pose_pred_axis; @@ -1049,6 +1141,13 @@ ivas_error combine_external_and_head_orientations_rend( else if ( hExtOrientationData != NULL ) { /* Head rotation data not available, use the freezed value or disable */ +#ifdef API_5MS + if ( hExtOrientationData->enableHeadRotation != 2 ) + { + hExtOrientationData->enableHeadRotation = 0; + } + +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { if ( hExtOrientationData->enableHeadRotation[i] != 2 ) @@ -1056,13 +1155,20 @@ ivas_error combine_external_and_head_orientations_rend( hExtOrientationData->enableHeadRotation[i] = 0; } } +#endif } - return combine_external_and_head_orientations( headRotQuaternions, listenerPos, + return combine_external_and_head_orientations( + headRotQuaternions, + listenerPos, #ifdef SPLIT_REND_WITH_HEAD_ROT - sr_pose_pred_axis, + sr_pose_pred_axis, #endif - numHeadRotQuaternions, hExtOrientationData, hCombinedOrientationData ); +#ifndef API_5MS + numHeadRotQuaternions, +#endif + hExtOrientationData, + hCombinedOrientationData ); } @@ -1073,18 +1179,27 @@ ivas_error combine_external_and_head_orientations_rend( * NOTE that the external orientations are inversed. *------------------------------------------------------------------------*/ -static ivas_error combine_external_and_head_orientations( - IVAS_QUATERNION *headRotQuaternions, /* i : quaternions for head rotation */ - IVAS_VECTOR3 *listenerPos, /* i : listener position */ +ivas_error combine_external_and_head_orientations( +#ifdef API_5MS + IVAS_QUATERNION *headRotQuaternion, /* i : quaternion for head rotation */ +#else + IVAS_QUATERNION *headRotQuaternions, /* i : quaternions for head rotation */ +#endif + IVAS_VECTOR3 *listenerPos, /* i : listener position */ #ifdef SPLIT_REND_WITH_HEAD_ROT IVAS_SPLIT_REND_ROT_AXIS sr_pose_pred_axis, /* i : split rend pose prediction axis*/ #endif - int16_t numHeadRotQuaternions, /* i : number of head rotation quaternions */ - EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, /* i : external orientation handle */ - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData /* i/o: combined orientation handle */ +#ifndef API_5MS + int16_t numHeadRotQuaternions, /* i : number of head rotation quaternions */ +#endif + EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, /* i : external orientation handle */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData /* i/o: combined orientation handle */ ) { - int16_t i, j; +#ifndef API_5MS + int16_t i; +#endif + int16_t j; IVAS_QUATERNION identity; IVAS_VECTOR3 origo; @@ -1095,7 +1210,11 @@ static ivas_error combine_external_and_head_orientations( /* Form combined orientations or return if no data available */ if ( hCombinedOrientationData == NULL ) { +#ifdef API_5MS + if ( headRotQuaternion != NULL || hExtOrientationData != NULL ) +#else if ( headRotQuaternions != NULL || hExtOrientationData != NULL ) +#endif { return IVAS_ERR_UNEXPECTED_NULL_POINTER; } @@ -1104,7 +1223,11 @@ static ivas_error combine_external_and_head_orientations( return IVAS_ERR_OK; } } +#ifdef API_5MS + else if ( headRotQuaternion == NULL && hExtOrientationData == NULL ) +#else else if ( headRotQuaternions == NULL && hExtOrientationData == NULL ) +#endif { /* Reset the combined orientations and rotations */ hCombinedOrientationData->isInterpolationOngoing = FALSE; @@ -1112,6 +1235,18 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->interpolationIncrement = 1.0f; hCombinedOrientationData->Quaternions_ext_interpolation_start = identity; hCombinedOrientationData->Quaternions_ext_interpolation_target = identity; +#ifdef API_5MS + + hCombinedOrientationData->enableCombinedOrientation = 0; + hCombinedOrientationData->Quaternion = identity; + hCombinedOrientationData->listenerPos = origo; + + for ( j = 0; j < 3; j++ ) + { + set_zero( hCombinedOrientationData->Rmat[j], 3 ); + hCombinedOrientationData->Rmat[j][j] = 1.0f; + } +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { hCombinedOrientationData->enableCombinedOrientation[i] = 0; @@ -1124,10 +1259,18 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->Rmat[i][j][j] = 1.0f; } } +#endif } +#ifdef API_5MS + else if ( hExtOrientationData == NULL && headRotQuaternion != NULL ) +#else else if ( hExtOrientationData == NULL && headRotQuaternions != NULL ) +#endif { /* Head rotation only */ +#ifdef API_5MS + hCombinedOrientationData->Quaternion = *headRotQuaternion; +#else if ( numHeadRotQuaternions >= 0 ) { for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) @@ -1135,11 +1278,62 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->Quaternions[i] = headRotQuaternions[i]; } } +#endif } if ( hExtOrientationData != NULL ) { /* External orientations */ +#ifdef API_5MS + + /* Check for frozen external orientation */ + if ( hExtOrientationData->enableExternalOrientation == 2 ) + { + if ( hCombinedOrientationData->isExtOrientationFrozen != 1 ) + { + hCombinedOrientationData->Quaternion_frozen_ext = hExtOrientationData->Quaternion; + hCombinedOrientationData->isExtOrientationFrozen = 1; + } + } + else + { + hCombinedOrientationData->Quaternion_frozen_ext = identity; + hCombinedOrientationData->isExtOrientationFrozen = 0; + } + + if ( hExtOrientationData->enableRotationInterpolation == 1 && hExtOrientationData->enableExternalOrientation > 0 ) + { + if ( hCombinedOrientationData->isInterpolationOngoing == TRUE && hCombinedOrientationData->interpolationCoefficient <= 1.0f && are_orientations_same( &hCombinedOrientationData->Quaternions_ext_interpolation_target, &hExtOrientationData->Quaternion ) == true ) + { + /* Continue interpolation */ + QuaternionSlerp( hCombinedOrientationData->Quaternions_ext_interpolation_start, hCombinedOrientationData->Quaternions_ext_interpolation_target, hCombinedOrientationData->interpolationCoefficient, &hCombinedOrientationData->Quaternion ); + hCombinedOrientationData->interpolationCoefficient += hCombinedOrientationData->interpolationIncrement; + } + else + { + /* Stop interpolation or check for new interpolation */ + hCombinedOrientationData->isInterpolationOngoing = FALSE; + hCombinedOrientationData->interpolationCoefficient = 1.0f; + hCombinedOrientationData->interpolationIncrement = 1.0f; + external_target_interpolation( hExtOrientationData, hCombinedOrientationData ); + } + } + else + { + /* Interpolation disabled, use the current orientation values */ + + /* Use the most recent external orientation */ + if ( hExtOrientationData->enableExternalOrientation == 1 ) + { + hCombinedOrientationData->Quaternion = hExtOrientationData->Quaternion; + } + /* Use the freezed external orientation */ + else if ( hExtOrientationData->enableExternalOrientation == 2 ) + { + hCombinedOrientationData->Quaternion = hCombinedOrientationData->Quaternion_frozen_ext; + } + } +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { /* Check for frozen external orientation */ @@ -1190,11 +1384,64 @@ static ivas_error combine_external_and_head_orientations( } } } +#endif } +#ifdef API_5MS + if ( hExtOrientationData != NULL && headRotQuaternion != NULL ) +#else if ( hExtOrientationData != NULL && headRotQuaternions != NULL ) +#endif { /* Combine head and external orientations */ +#ifdef API_5MS + /* Check for frozen head rotation */ + if ( hExtOrientationData->enableHeadRotation == 2 ) + { + if ( hCombinedOrientationData->isHeadRotationFrozen != 1 ) + { + hCombinedOrientationData->Quaternion_frozen_head = *headRotQuaternion; + hCombinedOrientationData->isHeadRotationFrozen = 1; + } + } + else + { + hCombinedOrientationData->Quaternion_frozen_head = identity; + hCombinedOrientationData->isHeadRotationFrozen = 0; + } + + /* Use the most recent head rotation */ + if ( hExtOrientationData->enableHeadRotation == 1 ) + { + if ( hExtOrientationData->enableExternalOrientation > 0 ) + { + QuaternionProduct( hCombinedOrientationData->Quaternion, *headRotQuaternion, &hCombinedOrientationData->Quaternion ); + } + else + { + hCombinedOrientationData->Quaternion = *headRotQuaternion; + } + } + /* Use the freezed head rotation */ + else if ( hExtOrientationData->enableHeadRotation == 2 ) + { + if ( hExtOrientationData->enableExternalOrientation > 0 ) + { + QuaternionProduct( hCombinedOrientationData->Quaternion, hCombinedOrientationData->Quaternion_frozen_head, &hCombinedOrientationData->Quaternion ); + } + else + { + hCombinedOrientationData->Quaternion = hCombinedOrientationData->Quaternion_frozen_head; + } + } + + /* Reset the combined orientations to identity */ + if ( hExtOrientationData->enableHeadRotation == 0 && hExtOrientationData->enableExternalOrientation == 0 ) + { + hCombinedOrientationData->Quaternion = identity; + } + +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { /* Check for frozen head rotation */ @@ -1242,40 +1489,74 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->Quaternions[i] = identity; } } +#endif } +#ifdef API_5MS + if ( headRotQuaternion != NULL || hExtOrientationData != NULL ) +#else if ( headRotQuaternions != NULL || hExtOrientationData != NULL ) +#endif { /* Calculate the combined rotation matrix */ +#ifdef API_5MS + QuatToRotMat( hCombinedOrientationData->Quaternion, hCombinedOrientationData->Rmat ); +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { QuatToRotMat( hCombinedOrientationData->Quaternions[i], hCombinedOrientationData->Rmat[i] ); } +#endif } /* Save the current orientations */ if ( hExtOrientationData != NULL ) { +#ifdef API_5MS + if ( hExtOrientationData->enableExternalOrientation > 0 ) +#else if ( hExtOrientationData->enableExternalOrientation[MAX_PARAM_SPATIAL_SUBFRAMES - 1] > 0 ) +#endif { +#ifdef API_5MS + hCombinedOrientationData->Quaternion_prev_extOrientation = hExtOrientationData->Quaternion; +#else hCombinedOrientationData->Quaternion_prev_extOrientation = hExtOrientationData->Quaternions[MAX_PARAM_SPATIAL_SUBFRAMES - 1]; +#endif } else { hCombinedOrientationData->Quaternion_prev_extOrientation = identity; } } +#ifdef API_5MS + if ( headRotQuaternion != NULL ) +#else if ( headRotQuaternions != NULL ) +#endif { +#ifdef API_5MS + hCombinedOrientationData->listenerPos = *listenerPos; + +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { hCombinedOrientationData->listenerPos[i] = listenerPos[i]; } +#endif } /* Check if combined orientation is enabled */ +#ifdef API_5MS + if ( headRotQuaternion != NULL && hExtOrientationData == NULL ) +#else if ( headRotQuaternions != NULL && hExtOrientationData == NULL ) +#endif { +#ifdef API_5MS + hCombinedOrientationData->enableCombinedOrientation = 1; + +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { if ( numHeadRotQuaternions >= 0 ) @@ -1287,9 +1568,24 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->enableCombinedOrientation[i] = 0; } } +#endif } +#ifdef API_5MS + else if ( headRotQuaternion == NULL && hExtOrientationData != NULL ) +#else else if ( headRotQuaternions == NULL && hExtOrientationData != NULL ) +#endif { +#ifdef API_5MS + if ( hExtOrientationData->enableExternalOrientation > 0 ) + { + hCombinedOrientationData->enableCombinedOrientation = 1; + } + else + { + hCombinedOrientationData->enableCombinedOrientation = 0; + } +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { if ( hExtOrientationData->enableExternalOrientation[i] > 0 ) @@ -1301,9 +1597,24 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->enableCombinedOrientation[i] = 0; } } +#endif } +#ifdef API_5MS + else if ( headRotQuaternion != NULL && hExtOrientationData != NULL ) +#else else if ( headRotQuaternions != NULL && hExtOrientationData != NULL ) +#endif { +#ifdef API_5MS + if ( hExtOrientationData->enableExternalOrientation > 0 || hExtOrientationData->enableHeadRotation > 0 ) + { + hCombinedOrientationData->enableCombinedOrientation = 1; + } + else + { + hCombinedOrientationData->enableCombinedOrientation = 0; + } +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { if ( hExtOrientationData->enableExternalOrientation[i] > 0 || ( hExtOrientationData->enableHeadRotation[i] > 0 && numHeadRotQuaternions >= 0 ) ) @@ -1315,13 +1626,18 @@ static ivas_error combine_external_and_head_orientations( hCombinedOrientationData->enableCombinedOrientation[i] = 0; } } +#endif } else { +#ifdef API_5MS + hCombinedOrientationData->enableCombinedOrientation = 0; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { hCombinedOrientationData->enableCombinedOrientation[i] = 0; } +#endif } #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -1338,26 +1654,58 @@ static ivas_error combine_external_and_head_orientations( * *------------------------------------------------------------------------*/ -static void external_target_interpolation( - EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, /* i : external orientation handle */ - COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, /* i/o: combined orientation handle */ - const int16_t i ) +void external_target_interpolation( + EXTERNAL_ORIENTATION_HANDLE hExtOrientationData, /* i : external orientation handle */ + COMBINED_ORIENTATION_HANDLE hCombinedOrientationData /* i/o: combined orientation handle */ +#ifndef API_5MS + , + const int16_t i +#endif +) { /* Sanity check for number of frames */ +#ifdef API_5MS + hExtOrientationData->numFramesToTargetOrientation = min( hExtOrientationData->numFramesToTargetOrientation, hCombinedOrientationData->maximumFramesToTargetOrientation ); + hExtOrientationData->numFramesToTargetOrientation = max( hExtOrientationData->numFramesToTargetOrientation, 0 ); +#else hExtOrientationData->numFramesToTargetOrientation[i] = min( hExtOrientationData->numFramesToTargetOrientation[i], hCombinedOrientationData->maximumFramesToTargetOrientation ); hExtOrientationData->numFramesToTargetOrientation[i] = max( hExtOrientationData->numFramesToTargetOrientation[i], 0 ); +#endif /* Interpolate from the current orientation to the target orientation */ +#ifdef API_5MS + if ( hExtOrientationData->numFramesToTargetOrientation > 0 ) +#else if ( hExtOrientationData->numFramesToTargetOrientation[i] > 0 ) +#endif { +#ifdef API_5MS + if ( are_orientations_same( &hCombinedOrientationData->Quaternions_ext_interpolation_target, &hExtOrientationData->Quaternion ) == false ) +#else if ( are_orientations_same( &hCombinedOrientationData->Quaternions_ext_interpolation_target, &hExtOrientationData->Quaternions[i] ) == false ) +#endif { /* Target orientation is different from the previous target, update the values */ /* Set the received orientation as the target */ +#ifdef API_5MS + hCombinedOrientationData->Quaternions_ext_interpolation_target = hExtOrientationData->Quaternion; +#else hCombinedOrientationData->Quaternions_ext_interpolation_target = hExtOrientationData->Quaternions[i]; +#endif - /* Use the most recent external orientation as the starting orientation */ +/* Use the most recent external orientation as the starting orientation */ +#ifdef API_5MS + if ( hExtOrientationData->enableExternalOrientation == 1 ) + { + hCombinedOrientationData->Quaternions_ext_interpolation_start = hCombinedOrientationData->Quaternion_prev_extOrientation; + } + else if ( hExtOrientationData->enableExternalOrientation == 2 ) + { + hCombinedOrientationData->Quaternions_ext_interpolation_start = hCombinedOrientationData->Quaternion_frozen_ext; + } + +#else if ( hExtOrientationData->enableExternalOrientation[i] == 1 ) { if ( i > 0 ) @@ -1373,15 +1721,24 @@ static void external_target_interpolation( { hCombinedOrientationData->Quaternions_ext_interpolation_start = hCombinedOrientationData->Quaternion_frozen_ext; } +#endif /* Calculate the interpolation increment and coefficient */ +#ifdef API_5MS + hCombinedOrientationData->interpolationIncrement = 1.0f / ( (float) hExtOrientationData->numFramesToTargetOrientation ); +#else hCombinedOrientationData->interpolationIncrement = 1.0f / ( (float) hExtOrientationData->numFramesToTargetOrientation[i] * (float) MAX_PARAM_SPATIAL_SUBFRAMES ); +#endif hCombinedOrientationData->interpolationCoefficient = hCombinedOrientationData->interpolationIncrement; } /* Interpolate */ hCombinedOrientationData->isInterpolationOngoing = TRUE; +#ifdef API_5MS + QuaternionSlerp( hCombinedOrientationData->Quaternions_ext_interpolation_start, hCombinedOrientationData->Quaternions_ext_interpolation_target, hCombinedOrientationData->interpolationCoefficient, &hCombinedOrientationData->Quaternion ); +#else QuaternionSlerp( hCombinedOrientationData->Quaternions_ext_interpolation_start, hCombinedOrientationData->Quaternions_ext_interpolation_target, hCombinedOrientationData->interpolationCoefficient, &hCombinedOrientationData->Quaternions[i] ); +#endif hCombinedOrientationData->interpolationCoefficient += hCombinedOrientationData->interpolationIncrement; } else @@ -1390,7 +1747,11 @@ static void external_target_interpolation( hCombinedOrientationData->isInterpolationOngoing = FALSE; hCombinedOrientationData->interpolationCoefficient = 1.0f; hCombinedOrientationData->interpolationIncrement = 1.0f; +#ifdef API_5MS + hCombinedOrientationData->Quaternion = hExtOrientationData->Quaternion; +#else hCombinedOrientationData->Quaternions[i] = hExtOrientationData->Quaternions[i]; +#endif } return; diff --git a/lib_rend/ivas_splitRendererPost.c b/lib_rend/ivas_splitRendererPost.c index 7566007834e099d0d521072e8ae4ce14e245a732..9a3d6593d17ad50cd822bbf8e4e3addc19b0e3ef 100644 --- a/lib_rend/ivas_splitRendererPost.c +++ b/lib_rend/ivas_splitRendererPost.c @@ -1589,12 +1589,22 @@ static void interpolate_rend_md( void ivas_SplitRenderer_PostRenderer( BIN_HR_SPLIT_POST_REND_HANDLE hBinPostRenderer, /* i/o: binaural renderer handle */ MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, +#ifdef API_5MS + float Cldfb_RealBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */ + float Cldfb_ImagBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */ + const IVAS_QUATERNION Quaternion_act +#else float Cldfb_RealBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */ float Cldfb_ImagBuffer_Ref_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], /* i/o : Reference/out Binaural signals */ - const IVAS_QUATERNION Quaternions_act[MAX_PARAM_SPATIAL_SUBFRAMES] ) + const IVAS_QUATERNION Quaternions_act[MAX_PARAM_SPATIAL_SUBFRAMES] +#endif +) { int16_t pos_idx, b, brange[2], ch_idx1; - int16_t num_md_bands, slot_idx, b2, sf_idx, index_slot, num_subframes, num_slots, sf_idx_md; + int16_t num_md_bands, slot_idx, b2, index_slot, num_slots, sf_idx_md; +#ifndef API_5MS + int16_t sf_idx, num_subframes; +#endif float pred_out_re[BINAURAL_CHANNELS], pred_out_im[BINAURAL_CHANNELS], tmp_re, tmp_im, gd_int; #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG BIN_HR_SPLIT_REND_MD rot_md_act[MAX_HEAD_ROT_POSES][MAX_SPLIT_REND_MD_BANDS]; @@ -1618,19 +1628,40 @@ void ivas_SplitRenderer_PostRenderer( push_wmops( "ivas_SplitRenderer_PostRenderer" ); +#ifndef API_5MS num_subframes = MAX_PARAM_SPATIAL_SUBFRAMES; +#endif num_slots = MAX_PARAM_SPATIAL_SUBFRAMES; +#ifndef API_5MS for ( sf_idx = 0; sf_idx < num_subframes; sf_idx++ ) { +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG pos_idx = MAX_HEAD_ROT_POSES - 1; #else - pos_idx = 0; + pos_idx = 0; #endif - sf_idx_md = ( hBinPostRenderer->low_Res == 0 ) ? sf_idx : 0; - get_interpolation_vars( pMultiBinPoseData, &hBinPostRenderer->QuaternionsPre[sf_idx_md], &Quaternions_act[sf_idx], interp_yaw_pose_idx, interp_pitch_pose_idx, interp_roll_pose_idx, &interp_yaw_fact, &interp_pitch_fact, &interp_roll_fact ); +#ifdef API_5MS + /* TODO FhG2Dolby: needs verificatiion with 5ms framing */ + sf_idx_md = 0; +#else + sf_idx_md = ( hBinPostRenderer->low_Res == 0 ) ? sf_idx : 0; +#endif + get_interpolation_vars( pMultiBinPoseData, + &hBinPostRenderer->QuaternionsPre[sf_idx_md], +#ifdef API_5MS + &Quaternion_act, +#else + &Quaternions_act[sf_idx], +#endif + interp_yaw_pose_idx, + interp_pitch_pose_idx, + interp_roll_pose_idx, + &interp_yaw_fact, + &interp_pitch_fact, + &interp_roll_fact ); for ( b = 0; b < num_md_bands; b++ ) { for ( ch_idx1 = 0; ch_idx1 < BINAURAL_CHANNELS; ch_idx1++ ) @@ -1695,12 +1726,16 @@ void ivas_SplitRenderer_PostRenderer( #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) #else - pos_idx = 0; + pos_idx = 0; #endif { for ( slot_idx = 0; slot_idx < num_slots; slot_idx++ ) { - index_slot = sf_idx * num_slots + slot_idx; +#ifdef API_5MS + index_slot = slot_idx; /* TODO: can be cleaned up */ +#else + index_slot = sf_idx * num_slots + slot_idx; +#endif fade = ( (float) slot_idx + 1.0f ) / MAX_PARAM_SPATIAL_SUBFRAMES; fade = min( fade, 1.0f ); for ( b = 0; b < num_md_bands; b++ ) @@ -1763,8 +1798,8 @@ void ivas_SplitRenderer_PostRenderer( Cldfb_RealBuffer_Recons_Binaural[pos_idx][ch_idx1][index_slot][b2] = pred_out_re[ch_idx1]; Cldfb_ImagBuffer_Recons_Binaural[pos_idx][ch_idx1][index_slot][b2] = pred_out_im[ch_idx1]; #else - Cldfb_RealBuffer_Ref_Binaural[ch_idx1][index_slot][b2] = pred_out_re[ch_idx1]; - Cldfb_ImagBuffer_Ref_Binaural[ch_idx1][index_slot][b2] = pred_out_im[ch_idx1]; + Cldfb_RealBuffer_Ref_Binaural[ch_idx1][index_slot][b2] = pred_out_re[ch_idx1]; + Cldfb_ImagBuffer_Ref_Binaural[ch_idx1][index_slot][b2] = pred_out_im[ch_idx1]; #endif } } @@ -1775,7 +1810,7 @@ void ivas_SplitRenderer_PostRenderer( #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG for ( pos_idx = 0; pos_idx < MAX_HEAD_ROT_POSES; pos_idx++ ) #else - pos_idx = 0; + pos_idx = 0; #endif { for ( b = 0; b < num_md_bands; b++ ) @@ -1827,7 +1862,9 @@ void ivas_SplitRenderer_PostRenderer( } } #endif +#ifndef API_5MS } +#endif pop_wmops(); return; @@ -1843,18 +1880,33 @@ void ivas_SplitRenderer_PostRenderer( static void ivas_rend_CldfbSplitPostRendProcessTdIn( BIN_HR_SPLIT_POST_REND_HANDLE hBinHrSplitPostRend, MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, +#ifdef API_5MS + const IVAS_QUATERNION QuaternionPost, +#else const IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif float output[][L_FRAME48k] ) { int16_t ch_idx, slot_idx, num_cldfb_bands; +#ifdef API_5MS + float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; +#else float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; +#endif num_cldfb_bands = hBinHrSplitPostRend->cldfbSyn[0]->no_channels; /* Implement CLDFB analysis */ for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) { - for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX; slot_idx++ ) + for ( slot_idx = 0; +#ifdef API_5MS + slot_idx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; +#else + slot_idx < CLDFB_NO_COL_MAX; +#endif + slot_idx++ ) { cldfbAnalysis_ts( &( output[ch_idx][num_cldfb_bands * slot_idx] ), Cldfb_RealBuffer_Binaural[ch_idx][slot_idx], @@ -1864,21 +1916,48 @@ static void ivas_rend_CldfbSplitPostRendProcessTdIn( } } - ivas_SplitRenderer_PostRenderer( hBinHrSplitPostRend, pMultiBinPoseData, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, QuaternionsPost ); + ivas_SplitRenderer_PostRenderer( + hBinHrSplitPostRend, + pMultiBinPoseData, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, +#ifdef API_5MS + QuaternionPost +#else + QuaternionsPost +#endif + ); /* Implement CLDFB synthesis */ for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) { +#ifdef API_5MS + float *RealBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; +#else float *RealBuffer[CLDFB_NO_COL_MAX]; float *ImagBuffer[CLDFB_NO_COL_MAX]; +#endif - for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX; slot_idx++ ) + for ( slot_idx = 0; +#ifdef API_5MS + slot_idx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; +#else + slot_idx < CLDFB_NO_COL_MAX; +#endif + slot_idx++ ) { RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[ch_idx][slot_idx]; ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[ch_idx][slot_idx]; } - cldfbSynthesis( RealBuffer, ImagBuffer, &( output[ch_idx][0] ), num_cldfb_bands * CLDFB_NO_COL_MAX, hBinHrSplitPostRend->cldfbSyn[ch_idx] ); + cldfbSynthesis( RealBuffer, ImagBuffer, &( output[ch_idx][0] ), +#ifdef API_5MS + num_cldfb_bands * CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES, +#else + num_cldfb_bands * CLDFB_NO_COL_MAX, +#endif + hBinHrSplitPostRend->cldfbSyn[ch_idx] ); } return; @@ -1894,9 +1973,15 @@ static void ivas_rend_CldfbSplitPostRendProcessTdIn( void ivas_rend_CldfbSplitPostRendProcess( BIN_HR_SPLIT_POST_REND_HANDLE hBinHrSplitPostRend, MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, +#ifdef API_5MS + const IVAS_QUATERNION QuaternionPost, + float Cldfb_RealBuffer_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], + float Cldfb_ImagBuffer_Binaural[][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX], +#else const IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES], float Cldfb_RealBuffer_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_ImagBuffer_Binaural[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], +#endif float output[][L_FRAME48k], const int16_t is_cldfb_in ) { @@ -1908,28 +1993,57 @@ void ivas_rend_CldfbSplitPostRendProcess( if ( is_cldfb_in == 0 ) { - ivas_rend_CldfbSplitPostRendProcessTdIn( hBinHrSplitPostRend, pMultiBinPoseData, QuaternionsPost, output ); + ivas_rend_CldfbSplitPostRendProcessTdIn( hBinHrSplitPostRend, pMultiBinPoseData, +#ifdef API_5MS + QuaternionPost, +#else + QuaternionsPost, +#endif + output ); pop_wmops(); return; } - ivas_SplitRenderer_PostRenderer( hBinHrSplitPostRend, pMultiBinPoseData, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, QuaternionsPost ); + ivas_SplitRenderer_PostRenderer( + hBinHrSplitPostRend, + pMultiBinPoseData, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, +#ifdef API_5MS + QuaternionPost +#else + QuaternionsPost +#endif + ); /* Implement CLDFB synthesis */ for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) { +#ifdef API_5MS + float *RealBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + + for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) +#else float *RealBuffer[CLDFB_NO_COL_MAX]; float *ImagBuffer[CLDFB_NO_COL_MAX]; for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX; slot_idx++ ) +#endif { RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[ch_idx][slot_idx]; ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[ch_idx][slot_idx]; } - cldfbSynthesis( RealBuffer, ImagBuffer, &( output[ch_idx][0] ), num_cldfb_bands * CLDFB_NO_COL_MAX, hBinHrSplitPostRend->cldfbSyn[ch_idx] ); + cldfbSynthesis( RealBuffer, ImagBuffer, &( output[ch_idx][0] ), num_cldfb_bands * CLDFB_NO_COL_MAX +#ifdef API_5MS + / MAX_PARAM_SPATIAL_SUBFRAMES +#endif + , + hBinHrSplitPostRend->cldfbSyn[ch_idx] ); } + pop_wmops(); return; diff --git a/lib_rend/ivas_splitRendererPre.c b/lib_rend/ivas_splitRendererPre.c index e630837acaa17cafda84d111dda40788ff75d3c6..0780ed009497a8c54e1b3d26fdc3ce810852f833 100644 --- a/lib_rend/ivas_splitRendererPre.c +++ b/lib_rend/ivas_splitRendererPre.c @@ -1173,7 +1173,11 @@ static void ivas_SplitRenderer_code_md_huff( static void ivas_SplitRenderer_quant_code( const BIN_HR_SPLIT_PRE_REND_HANDLE hBinHrSplitPreRend, +#ifdef API_5MS + const IVAS_QUATERNION headPosition, +#else const IVAS_QUATERNION headPositions[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, ivas_split_rend_bits_t *pBits, const int16_t low_res_pre_rend_rot, @@ -1212,8 +1216,12 @@ static void ivas_SplitRenderer_quant_code( { int16_t angle; IVAS_QUATERNION head_pos_euler; - +#ifdef API_5MS + /* FhG@Dolby: please review, this can be likely optimised */ + Quat2EulerDegree( headPosition, &head_pos_euler.z, &head_pos_euler.y, &head_pos_euler.x ); +#else Quat2EulerDegree( headPositions[sf_idx], &head_pos_euler.z, &head_pos_euler.y, &head_pos_euler.x ); +#endif angle = (int16_t) roundf( head_pos_euler.x ); angle += 180; ivas_split_rend_bitstream_write_int32( pBits, angle, IVAS_SPLIT_REND_HEAD_POSE_BITS ); @@ -1550,7 +1558,11 @@ void ivas_SplitRenderer_GetRotMd( void ivas_rend_CldfbSplitPreRendProcess( const BIN_HR_SPLIT_PRE_REND_HANDLE hBinHrSplitPreRend, +#ifdef API_5MS + const IVAS_QUATERNION headPosition, +#else const IVAS_QUATERNION headPositions[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData, float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], @@ -1562,7 +1574,17 @@ void ivas_rend_CldfbSplitPreRendProcess( ivas_SplitRenderer_GetRotMd( hBinHrSplitPreRend, pMultiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, low_res_pre_rend_rot ); - ivas_SplitRenderer_quant_code( hBinHrSplitPreRend, headPositions, pMultiBinPoseData, pBits, low_res_pre_rend_rot, target_md_bits ); + ivas_SplitRenderer_quant_code( + hBinHrSplitPreRend, +#ifdef API_5MS + headPosition, +#else + headPositions, +#endif + pMultiBinPoseData, + pBits, + low_res_pre_rend_rot, + target_md_bits ); #ifdef SPLIT_POSE_CORRECTION_DEBUG float tmpCrendBuffer[2][L_FRAME48k], quant_val, step, minv, maxv; @@ -1789,7 +1811,9 @@ ivas_error ivas_set_split_rend_setup( COMBINED_ORIENTATION_HANDLE hCombinedOrientationData, uint8_t *splitRendBitsBuf ) { +#ifndef API_5MS int16_t sf, i, j; +#endif hSplitBinRend->hSplitRendBits->bits_buf = splitRendBitsBuf; hSplitBinRend->hSplitRendBits->bits_written = 0; @@ -1802,6 +1826,7 @@ ivas_error ivas_set_split_rend_setup( ivas_renderSplitGetMultiBinPoseData( hSplitBinConfig, &hSplitBinRend->splitrend.multiBinPoseData, hCombinedOrientationData->sr_pose_pred_axis ); +#ifndef API_5MS if ( hCombinedOrientationData != NULL && hSplitBinRend->splitrend.multiBinPoseData.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { for ( sf = 1; sf < MAX_PARAM_SPATIAL_SUBFRAMES; sf++ ) @@ -1817,6 +1842,7 @@ ivas_error ivas_set_split_rend_setup( } } } +#endif return IVAS_ERR_OK; } @@ -1870,13 +1896,24 @@ static ivas_error split_renderer_open_lc3plus( int16_t i, delayBufferLength; LC3PLUS_CONFIG config; +#ifdef API_5MS + config.lc3plus_frame_duration_us = pSplitRendConfig->codec_frame_size_ms * 1000; + config.ivas_frame_duration_us = ( pSplitRendConfig->dof == 0 ) ? config.lc3plus_frame_duration_us : 20000; +#else config.lc3plus_frame_duration_us = 5000; config.ivas_frame_duration_us = 20000; +#endif config.samplerate = OutSampleRate; config.channels = BINAURAL_CHANNELS; - error = IVAS_LC3PLUS_ENC_Open( config, ivas_get_lc3plus_bitrate( pSplitRendConfig->splitRendBitRate, pSplitRendConfig->poseCorrectionMode ), &hSplitRendWrapper->hLc3plusEnc ); + error = IVAS_LC3PLUS_ENC_Open( config, ivas_get_lc3plus_bitrate( pSplitRendConfig->splitRendBitRate, pSplitRendConfig->poseCorrectionMode +#ifdef API_5MS + , + (int16_t) ( config.ivas_frame_duration_us / 1000 ) +#endif + ), + &hSplitRendWrapper->hLc3plusEnc ); if ( error != IVAS_ERR_OK ) { return error; @@ -1892,7 +1929,7 @@ static ivas_error split_renderer_open_lc3plus( /* Alocate buffers for delay compensation */ if ( pSplitRendConfig->codec == IVAS_SPLIT_REND_CODEC_LC3PLUS ) { - delayBufferLength = (int16_t) ( OutSampleRate / (int32_t) FRAMES_PER_SECOND + hSplitRendWrapper->lc3plusDelaySamples ); + delayBufferLength = (int16_t) ( OutSampleRate / (int32_t) FRAMES_PER_SEC + hSplitRendWrapper->lc3plusDelaySamples ); for ( i = 0; i < hSplitRendWrapper->multiBinPoseData.num_poses * BINAURAL_CHANNELS; ++i ) { if ( ( hSplitRendWrapper->lc3plusDelayBuffers[i] = malloc( delayBufferLength * sizeof( float ) ) ) == NULL ) @@ -2187,7 +2224,9 @@ static ivas_error splitRendLc3plusEncodeAndWrite( pBits->bits_written += 8 * lc3plusBitstreamSize; pBits->codec = IVAS_SPLIT_REND_CODEC_LC3PLUS; pBits->pose_correction = hSplitBin->multiBinPoseData.poseCorrectionMode; - +#ifdef API_5MS + pBits->codec_frame_size_ms = (int16_t) ( hSplitBin->hLc3plusEnc->config.lc3plus_frame_duration_us / 1000 ); +#endif return IVAS_ERR_OK; } @@ -2200,8 +2239,15 @@ static ivas_error splitRendLc3plusEncodeAndWrite( static ivas_error ivas_renderMultiTDBinToSplitBinaural( SPLIT_REND_WRAPPER *hSplitBin, +#ifdef API_5MS + const IVAS_QUATERNION headPosition, +#else const IVAS_QUATERNION headPositions[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif const int32_t SplitRendBitRate, +#ifdef API_5MS + const int16_t codec_frame_size_ms, +#endif ivas_split_rend_bits_t *pBits, const int16_t max_bands, float *in[], @@ -2226,7 +2272,7 @@ static ivas_error ivas_renderMultiTDBinToSplitBinaural( if ( useLc3plus ) { - int16_t frame_size = (int16_t) ( hSplitBin->hLc3plusEnc->config.samplerate / (int32_t) FRAMES_PER_SECOND ); + int16_t frame_size = (int16_t) ( hSplitBin->hLc3plusEnc->config.samplerate / (int32_t) FRAMES_PER_SEC ); for ( i = 0; i < num_poses * BINAURAL_CHANNELS; ++i ) { @@ -2247,6 +2293,12 @@ static ivas_error ivas_renderMultiTDBinToSplitBinaural( actual_md_bits = pBits->bits_written; if ( ( hSplitBin->multiBinPoseData.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) || ( !useLc3plus && !pcm_out ) ) { +#ifdef API_5MS + if ( !useLc3plus && codec_frame_size_ms != 20 && !pcm_out ) + { + return IVAS_ERROR( IVAS_ERR_INVALID_INPUT_BUFFER_SIZE, "Unsupported framing for LCLD codec!" ); + } +#endif num_cldfb_bands = hSplitBin->hCldfbHandles->cldfbAna[0]->no_channels; /* CLDFB Analysis*/ @@ -2285,8 +2337,19 @@ static ivas_error ivas_renderMultiTDBinToSplitBinaural( target_md_bits = ivas_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out ) * L_FRAME48k / 48000; actual_md_bits = pBits->bits_written; - - ivas_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPositions, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, target_md_bits, low_res_pre_rend_rot ); + ivas_rend_CldfbSplitPreRendProcess( + hSplitBin->hBinHrSplitPreRend, +#ifdef API_5MS + headPosition, +#else + headPositions, +#endif + &hSplitBin->multiBinPoseData, + Cldfb_In_BinReal, + Cldfb_In_BinImag, + pBits, + target_md_bits, + low_res_pre_rend_rot ); } if ( pcm_out == 0 ) @@ -2299,8 +2362,15 @@ static ivas_error ivas_renderMultiTDBinToSplitBinaural( available_bits = SplitRendBitRate * L_FRAME48k / 48000; actual_md_bits = pBits->bits_written - actual_md_bits; available_bits -= actual_md_bits; - - ivas_splitBinLCLDEncProcess( hSplitBin->hSplitBinLCLDEnc, Cldfb_In_BinReal, Cldfb_In_BinImag, available_bits, pBits ); +#ifdef API_5MS + pBits->codec_frame_size_ms = 20; +#endif + ivas_splitBinLCLDEncProcess( + hSplitBin->hSplitBinLCLDEnc, + Cldfb_In_BinReal, + Cldfb_In_BinImag, + available_bits, + pBits ); } else { @@ -2317,7 +2387,19 @@ static ivas_error ivas_renderMultiTDBinToSplitBinaural( } /*zero pad*/ - bit_len = SplitRendBitRate / FRAMES_PER_SEC; +#ifdef API_5MS + if ( pcm_out ) + { +#endif + bit_len = SplitRendBitRate / FRAMES_PER_SEC; +#ifdef API_5MS + } + else + { + bit_len = SplitRendBitRate * codec_frame_size_ms / 1000; + } +#endif + while ( pBits->bits_written < bit_len ) { ivas_split_rend_bitstream_write_int32( pBits, 0L, 1 ); @@ -2399,9 +2481,16 @@ static void lc3plusTimeAlignCldfbPoseCorr( ivas_error ivas_renderMultiBinToSplitBinaural( SPLIT_REND_WRAPPER *hSplitBin, +#ifdef API_5MS + const IVAS_QUATERNION headPosition, +#else const IVAS_QUATERNION headPositions[MAX_PARAM_SPATIAL_SUBFRAMES], +#endif const int32_t SplitRendBitRate, IVAS_SPLIT_REND_CODEC splitCodec, +#ifdef API_5MS + int16_t codec_frame_size_ms, +#endif ivas_split_rend_bits_t *pBits, float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], @@ -2426,14 +2515,31 @@ ivas_error ivas_renderMultiBinToSplitBinaural( /* Needs to be done at runtime. If this was in another API function, * there would be no guarantee that the user did not change * the split rendering config before calling the main rendering function */ +#ifdef API_5MS + if ( ( error = ivas_split_rend_choose_default_codec( &splitCodec, &codec_frame_size_ms, td_input, pcm_out ) ) != IVAS_ERR_OK ) + { + return error; + } +#else ivas_split_rend_choose_default_codec( &splitCodec, td_input, pcm_out ); +#endif if ( td_input ) { /*TD input*/ /*if CLDFB handles have been allocated then assume valid multi binaural input in out[][] buffer and perform CLDFB analysis*/ - error = ivas_renderMultiTDBinToSplitBinaural( hSplitBin, headPositions, SplitRendBitRate, pBits, max_bands, output, low_res_pre_rend_rot, pcm_out ); - + error = ivas_renderMultiTDBinToSplitBinaural( hSplitBin, +#ifdef API_5MS + headPosition, +#else + headPositions, +#endif + SplitRendBitRate, +#ifdef API_5MS + codec_frame_size_ms, +#endif + pBits, max_bands, output, + low_res_pre_rend_rot, pcm_out ); pop_wmops(); return error; } @@ -2450,8 +2556,19 @@ ivas_error ivas_renderMultiBinToSplitBinaural( target_md_bits = ivas_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out ) * L_FRAME48k / 48000; actual_md_bits = pBits->bits_written; - - ivas_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPositions, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, target_md_bits, low_res_pre_rend_rot ); + ivas_rend_CldfbSplitPreRendProcess( + hSplitBin->hBinHrSplitPreRend, +#ifdef API_5MS + headPosition, +#else + headPositions, +#endif + &hSplitBin->multiBinPoseData, + Cldfb_In_BinReal, + Cldfb_In_BinImag, + pBits, + target_md_bits, + low_res_pre_rend_rot ); } if ( pcm_out == 0 ) @@ -2520,9 +2637,20 @@ ivas_error ivas_renderMultiBinToSplitBinaural( pBits->codec = IVAS_SPLIT_REND_CODEC_NONE; } - /*zero pad*/ - /*TODO: do this inside the LCLD ENC codec */ - bit_len = SplitRendBitRate / FRAMES_PER_SEC; +/*zero pad*/ +/*TODO: do this inside the LCLD ENC codec */ +#ifdef API_5MS + if ( pcm_out ) + { +#endif + bit_len = SplitRendBitRate / FRAMES_PER_SEC; +#ifdef API_5MS + } + else + { + bit_len = SplitRendBitRate * codec_frame_size_ms / 1000; + } +#endif while ( pBits->bits_written < bit_len ) { diff --git a/lib_rend/ivas_splitRenderer_utils.c b/lib_rend/ivas_splitRenderer_utils.c index 22d2b2183452f1a2e6cb0a0c16026d6f65e54623..edd765f97229a764e41032ce46538deda63aaada 100644 --- a/lib_rend/ivas_splitRenderer_utils.c +++ b/lib_rend/ivas_splitRenderer_utils.c @@ -455,6 +455,7 @@ void ivas_split_rend_bitstream_write_int32( return; } +#ifndef API_5MS /*------------------------------------------------------------------------- * Function ivas_split_rend_get_sf_rot_data() @@ -472,6 +473,7 @@ IVAS_QUATERNION ivas_split_rend_get_sf_rot_data( return headPositions[idx]; } +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT_DEBUG @@ -612,13 +614,22 @@ int32_t ivas_get_lcld_bitrate( * *------------------------------------------------------------------------*/ -int32_t ivas_get_lc3plus_bitrate( - const int32_t SplitRendBitRate, - IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode ) +int32_t ivas_get_lc3plus_bitrate( const int32_t SplitRendBitRate, IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode +#ifdef API_5MS + , + int16_t split_prerender_frame_size_ms +#endif +) { if ( poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { - int32_t inBandMdBps = (int32_t) ( 8 * FRAMES_PER_SECOND ); + int32_t inBandMdBps = (int32_t) ( 8 * +#ifdef API_5MS + 1000 / split_prerender_frame_size_ms +#else + FRAMES_PER_SEC +#endif + ); return ivas_get_lcld_bitrate( SplitRendBitRate, poseCorrectionMode ) - inBandMdBps; } if ( poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) @@ -673,14 +684,19 @@ int8_t ivas_get_lc3plus_bitrate_id( /*------------------------------------------------------------------------- - * Function ivas_mat_mult_2by2_complex() + * Function ivas_get_lc3plus_size_from_id() * * *------------------------------------------------------------------------*/ int32_t ivas_get_lc3plus_size_from_id( const int8_t SplitRendBitRateId, - const IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode ) + const IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrectionMode +#ifdef API_5MS + , + int16_t split_prerender_frame_size_ms +#endif +) { int32_t bitrate; @@ -718,10 +734,21 @@ int32_t ivas_get_lc3plus_size_from_id( } } - bitrate = ivas_get_lc3plus_bitrate( bitrate, poseCorrectionMode ); + bitrate = ivas_get_lc3plus_bitrate( bitrate, poseCorrectionMode +#ifdef API_5MS + , + split_prerender_frame_size_ms +#endif + ); /* Return size in bytes */ - return (int32_t) ( bitrate / FRAMES_PER_SECOND / 8 ); + return (int32_t) ( bitrate +#ifdef API_5MS + * split_prerender_frame_size_ms / 1000 +#else + / FRAMES_PER_SEC +#endif + / 8 ); } @@ -754,6 +781,21 @@ ivas_error ivas_split_rend_validate_config( return IVAS_ERROR( IVAS_ERR_INVALID_SPLIT_REND_CONFIG, "0 DOF and pose correction NONE must only ever be set together" ); } +#ifdef API_5MS + if ( pSplitRendConfig->codec_frame_size_ms != 0 ) /* 0 means "default for current codec", will be set to actual value at a later stage */ + { + if ( pSplitRendConfig->codec == IVAS_SPLIT_REND_CODEC_LCLD && pSplitRendConfig->codec_frame_size_ms != 20 ) + { + return IVAS_ERROR( IVAS_ERR_INVALID_SPLIT_REND_CONFIG, "Invalid framing for LCLD codec" ); + } + if ( pSplitRendConfig->codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && + ( pSplitRendConfig->codec_frame_size_ms != 5 && pSplitRendConfig->codec_frame_size_ms != 10 ) ) + { + return IVAS_ERROR( IVAS_ERR_INVALID_SPLIT_REND_CONFIG, "Invalid framing for LC3plus codec" ); + } + } +#endif + /* Validate bitrate */ if ( is_pcm_out == 0 ) { @@ -1087,8 +1129,16 @@ void ivas_init_multi_bin_pose_data( * *------------------------------------------------------------------------*/ -void ivas_split_rend_choose_default_codec( +#ifdef API_5MS +ivas_error +#else +void +#endif +ivas_split_rend_choose_default_codec( IVAS_SPLIT_REND_CODEC *pCodec, +#ifdef API_5MS + int16_t *pCodec_frame_size_ms, +#endif const int16_t isRenderingInTd, const int16_t pcm_out ) { @@ -1103,7 +1153,26 @@ void ivas_split_rend_choose_default_codec( { *pCodec = IVAS_SPLIT_REND_CODEC_NONE; } +#ifdef API_5MS + if ( *pCodec_frame_size_ms == 0 ) /* codec frame size hasn't been set yet - use default for current configuration */ + { + switch ( *pCodec ) + { + case IVAS_SPLIT_REND_CODEC_LCLD: + *pCodec_frame_size_ms = 20; + break; + case IVAS_SPLIT_REND_CODEC_LC3PLUS: + case IVAS_SPLIT_REND_CODEC_NONE: + *pCodec_frame_size_ms = 5; + break; + default: + return IVAS_ERROR( IVAS_ERR_INTERNAL_FATAL, "Unknown split codec value" ); + } + } + return IVAS_ERR_OK; +#else return; +#endif } #endif diff --git a/lib_rend/ivas_stat_rend.h b/lib_rend/ivas_stat_rend.h index 2f110efe97bc1dd23945cc1510bc0ac938e91554..9ac434a60226c4f81b0bae35abe4d6a8bd6a0e1e 100644 --- a/lib_rend/ivas_stat_rend.h +++ b/lib_rend/ivas_stat_rend.h @@ -621,9 +621,16 @@ typedef struct ivas_orient_trk_state_t typedef struct { int8_t headRotEnabled; +#ifdef API_5MS + IVAS_QUATERNION headPosition; + IVAS_VECTOR3 Pos; + float crossfade_5ms[L_FRAME48k / MAX_PARAM_SPATIAL_SUBFRAMES]; + float crossfade_20ms[L_FRAME48k]; +#else IVAS_QUATERNION headPositions[RENDERER_HEAD_POSITIONS_PER_FRAME]; IVAS_VECTOR3 Pos[RENDERER_HEAD_POSITIONS_PER_FRAME]; float crossfade[L_FRAME48k / RENDERER_HEAD_POSITIONS_PER_FRAME]; +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT IVAS_SPLIT_REND_ROT_AXIS sr_pose_pred_axis; #endif @@ -633,9 +640,14 @@ typedef struct typedef struct ivas_binaural_head_track_struct { +#ifdef API_5MS + IVAS_QUATERNION Quaternion; + IVAS_VECTOR3 Pos; +#else int16_t num_quaternions; IVAS_QUATERNION Quaternions[MAX_PARAM_SPATIAL_SUBFRAMES]; IVAS_VECTOR3 Pos[MAX_PARAM_SPATIAL_SUBFRAMES]; +#endif float Rmat[3][3]; float Rmat_prev[3][3]; @@ -661,12 +673,19 @@ typedef struct ivas_binaural_head_track_struct typedef struct ivas_external_orientation_struct { +#ifdef API_5MS + int8_t enableHeadRotation; /* 0 - disable, 1 - enable, 2 - freeze to previous rotation */ + int8_t enableExternalOrientation; /* 0 - disable, 1 - enable, 2 - freeze to previous orientation */ + int8_t enableRotationInterpolation; /* 0 - disable, 1 - enable */ + int16_t numFramesToTargetOrientation; /* Number of frames until target orientation is reached */ + IVAS_QUATERNION Quaternion; /* External orientation as a quaternion */ +#else int8_t enableHeadRotation[MAX_PARAM_SPATIAL_SUBFRAMES]; /* 0 - disable, 1 - enable, 2 - freeze to previous rotation */ int8_t enableExternalOrientation[MAX_PARAM_SPATIAL_SUBFRAMES]; /* 0 - disable, 1 - enable, 2 - freeze to previous orientation */ int8_t enableRotationInterpolation[MAX_PARAM_SPATIAL_SUBFRAMES]; /* 0 - disable, 1 - enable */ int16_t numFramesToTargetOrientation[MAX_PARAM_SPATIAL_SUBFRAMES]; /* Number of frames until target orientation is reached */ IVAS_QUATERNION Quaternions[MAX_PARAM_SPATIAL_SUBFRAMES]; /* External orientation in quaternions */ - +#endif } EXTERNAL_ORIENTATION_DATA, *EXTERNAL_ORIENTATION_HANDLE; /*----------------------------------------------------------------------------------* @@ -675,7 +694,11 @@ typedef struct ivas_external_orientation_struct typedef struct ivas_combined_orientation_struct { +#ifdef API_5MS + int16_t enableCombinedOrientation; +#else int16_t enableCombinedOrientation[MAX_PARAM_SPATIAL_SUBFRAMES]; +#endif float interpolationCoefficient; float interpolationIncrement; int16_t maximumFramesToTargetOrientation; @@ -683,11 +706,19 @@ typedef struct ivas_combined_orientation_struct uint8_t lrSwitchedCurrent; float lrSwitchInterpVal; bool isInterpolationOngoing; +#ifdef API_5MS + IVAS_QUATERNION Quaternion; +#else IVAS_QUATERNION Quaternions[MAX_PARAM_SPATIAL_SUBFRAMES]; +#endif IVAS_QUATERNION Quaternion_prev_extOrientation; IVAS_QUATERNION Quaternions_ext_interpolation_start; IVAS_QUATERNION Quaternions_ext_interpolation_target; +#ifdef API_5MS + float Rmat[3][3]; +#else float Rmat[MAX_PARAM_SPATIAL_SUBFRAMES][3][3]; +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT float Rmat_prev[MAX_HEAD_ROT_POSES][3][3]; #else @@ -696,7 +727,11 @@ typedef struct ivas_combined_orientation_struct float chEneIIR[2][MASA_FREQUENCY_BANDS]; /* independent of the format. MASA bands are suitable for the task and readily available in ROM. */ float procChEneIIR[2][MASA_FREQUENCY_BANDS]; int16_t shd_rot_max_order; +#ifdef API_5MS + IVAS_VECTOR3 listenerPos; +#else IVAS_VECTOR3 listenerPos[MAX_PARAM_SPATIAL_SUBFRAMES]; +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT IVAS_SPLIT_REND_ROT_AXIS sr_pose_pred_axis; #endif diff --git a/lib_rend/lib_rend.c b/lib_rend/lib_rend.c index 9380ac42605ff93159e8d4c1d9a2046565fb555a..79d6f3c988fe89e0a729bc63c53ad40da0e985bd 100644 --- a/lib_rend/lib_rend.c +++ b/lib_rend/lib_rend.c @@ -60,6 +60,9 @@ #define MAX_BUFFER_LENGTH ( MAX_BUFFER_LENGTH_PER_CHANNEL * MAX_INPUT_CHANNELS ) #define MAX_CLDFB_BUFFER_LENGTH ( MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL * MAX_INPUT_CHANNELS ) #define MAX_BIN_BUFFER_LENGTH ( MAX_BUFFER_LENGTH_PER_CHANNEL * BINAURAL_CHANNELS ) +#ifdef API_5MS +#define MAX_CLDFB_BIN_BUFFER_LENGTH ( MAX_CLDFB_BUFFER_LENGTH_PER_CHANNEL * BINAURAL_CHANNELS ) +#endif #else #define MAX_BUFFER_LENGTH ( MAX_BUFFER_LENGTH_PER_CHANNEL * MAX_INPUT_CHANNELS ) #endif @@ -67,7 +70,11 @@ #define MAX_BIN_DELAY_SAMPLES 50 /* Maximum supported rendering latency for binaural IRs */ /* Frame size required when rendering to binaural */ +#ifdef API_5MS +#define BINAURAL_RENDERING_FRAME_SIZE_MS 5 +#else #define BINAURAL_RENDERING_FRAME_SIZE_MS 20 +#endif /*-------------------------------------------------------------------* @@ -210,6 +217,10 @@ typedef struct input_base base; SPLIT_POST_REND_WRAPPER splitPostRendWrapper; float *bufferData; +#ifdef API_5MS + int16_t numCachedSamples; /* Number of decoded samples in bufferData that have not yet been played out */ + IVAS_REND_BitstreamBuffer *hBits; +#endif } input_split_post_rend; #endif @@ -343,6 +354,9 @@ static void convertBitsBufferToInternalBitsBuff( hBits->buf_len = outBits.config.bufLenInBytes; hBits->codec = outBits.config.codec; hBits->pose_correction = outBits.config.poseCorrection; +#ifdef API_5MS + hBits->codec_frame_size_ms = outBits.config.codec_frame_size_ms; +#endif return; } @@ -357,6 +371,9 @@ static void convertInternalBitsBuffToBitsBuffer( hOutBits->config.bufLenInBytes = bits.buf_len; hOutBits->config.codec = bits.codec; hOutBits->config.poseCorrection = bits.pose_correction; +#ifdef API_5MS + hOutBits->config.codec_frame_size_ms = bits.codec_frame_size_ms; +#endif return; } @@ -895,7 +912,11 @@ static ivas_error getNumNonLfeChannelsInSpeakerLayout( static ivas_error getMcConfigValues( AUDIO_CONFIG inConfig, +#ifdef API_5MS + const LSSETUP_CUSTOM_STRUCT *pInCustomLs, +#else LSSETUP_CUSTOM_STRUCT inCustomLs, +#endif const float **azimuth, const float **elevation, int16_t *lfe_idx, @@ -908,6 +929,22 @@ static ivas_error getMcConfigValues( switch ( inConfig ) { case AUDIO_CONFIG_LS_CUSTOM: +#ifdef API_5MS + *azimuth = (const float *) &pInCustomLs->ls_azimuth; + *elevation = (const float *) &pInCustomLs->ls_elevation; + if ( pInCustomLs->num_lfe > 0 ) + { + *lfe_idx = pInCustomLs->lfe_idx[0]; + } + for ( i = 0; i < pInCustomLs->num_spk; i++ ) + { + if ( pInCustomLs->ls_elevation[i] != 0 ) + { + *is_planar = 0; + break; + } + } +#else *azimuth = (const float *) &inCustomLs.ls_azimuth; *elevation = (const float *) &inCustomLs.ls_elevation; if ( inCustomLs.num_lfe > 0 ) @@ -922,6 +959,7 @@ static ivas_error getMcConfigValues( break; } } +#endif break; case AUDIO_CONFIG_MONO: case AUDIO_CONFIG_STEREO: @@ -1084,6 +1122,25 @@ static ivas_error initHeadRotation( /* Head rotation is enabled by default */ hIvasRend->headRotData.headRotEnabled = 1; +#ifdef API_5MS + /* Initialize 5ms crossfade */ + crossfade_len = L_FRAME48k / MAX_PARAM_SPATIAL_SUBFRAMES; + tmp = 1.f / ( crossfade_len - 1 ); + for ( i = 0; i < crossfade_len; i++ ) + { + hIvasRend->headRotData.crossfade_5ms[i] = i * tmp; + } + /* Initialize 20ms crossfade */ + crossfade_len = L_FRAME48k; + tmp = 1.f / ( crossfade_len - 1 ); + for ( i = 0; i < crossfade_len; i++ ) + { + hIvasRend->headRotData.crossfade_20ms[i] = i * tmp; + } + + /* Initialize with unit quaternion */ + hIvasRend->headRotData.headPosition = quaternionInit(); +#else /* Initialize 5ms crossfade */ crossfade_len = L_FRAME48k / RENDERER_HEAD_POSITIONS_PER_FRAME; tmp = 1.f / ( crossfade_len - 1 ); @@ -1097,6 +1154,7 @@ static ivas_error initHeadRotation( { hIvasRend->headRotData.headPositions[i] = quaternionInit(); } +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT hIvasRend->headRotData.sr_pose_pred_axis = DEFAULT_AXIS; #endif @@ -1365,6 +1423,22 @@ static ivas_error setRendInputActiveIsm( { return error; } + + /* Open TD renderer wrappers */ + for ( i = 0; i < MAX_HEAD_ROT_POSES - 1; ++i ) + { + if ( ( error = ivas_td_binaural_open_ext( &inputIsm->splitTdRendWrappers[i], + inConfig, + hRendCfg, + NULL, + *inputIsm->base.ctx.pOutSampleRate ) ) != IVAS_ERR_OK ) + { + return error; + } + + /* Assert same delay as main TD renderer */ + assert( inputIsm->splitTdRendWrappers[i].binaural_latency_ns == inputIsm->tdRendWrapper.binaural_latency_ns ); + } #endif } else if ( outConfig == AUDIO_CONFIG_MASA1 || outConfig == AUDIO_CONFIG_MASA2 ) @@ -2720,12 +2794,27 @@ static ivas_error setRendInputActiveSplitPostRend( rendCtx = inputSplitPostRend->base.ctx; outConfig = *rendCtx.pOutConfig; - if ( ( error = allocateInputBaseBufferData( &inputSplitPostRend->bufferData, MAX_BIN_BUFFER_LENGTH ) ) != IVAS_ERR_OK ) + if ( ( error = allocateInputBaseBufferData( &inputSplitPostRend->bufferData, +#ifdef API_5MS + MAX_CLDFB_BIN_BUFFER_LENGTH +#else + MAX_BIN_BUFFER_LENGTH +#endif + ) ) != IVAS_ERR_OK ) { return error; } - - initRendInputBase( &inputSplitPostRend->base, inConfig, id, rendCtx, inputSplitPostRend->bufferData, MAX_BIN_BUFFER_LENGTH ); + initRendInputBase( &inputSplitPostRend->base, inConfig, id, rendCtx, + inputSplitPostRend->bufferData, +#ifdef API_5MS + MAX_CLDFB_BIN_BUFFER_LENGTH +#else + MAX_BIN_BUFFER_LENGTH +#endif + ); +#ifdef API_5MS + inputSplitPostRend->numCachedSamples = 0; +#endif if ( ( error = updateSplitPostRendPanGains( inputSplitPostRend, outConfig, hRendCfg ) ) != IVAS_ERR_OK ) { @@ -3159,8 +3248,14 @@ static DecoderDummy *initDecoderDummy( decDummy->hDecoderConfig->output_Fs = sampleRate; decDummy->hDecoderConfig->nchan_out = numOutChannels; decDummy->hDecoderConfig->Opt_Headrotation = 0; +#ifdef API_5MS + decDummy->hDecoderConfig->Opt_tsm = 0; +#ifdef API_5MS_BASELINE + decDummy->hDecoderConfig->Opt_5ms = 0; +#endif +#else decDummy->hDecoderConfig->voip_active = 0; - +#endif decDummy->hBinRenderer = NULL; #ifdef SPLIT_REND_WITH_HEAD_ROT decDummy->hSplitBinRend.hSplitRendBits = NULL; @@ -3219,6 +3314,12 @@ static DecoderDummy *initDecoderDummy( set_zero( decDummy->hHeadTrackData->chEneIIR[1], MASA_FREQUENCY_BANDS ); set_zero( decDummy->hHeadTrackData->procChEneIIR[0], MASA_FREQUENCY_BANDS ); set_zero( decDummy->hHeadTrackData->procChEneIIR[1], MASA_FREQUENCY_BANDS ); +#ifdef API_5MS + decDummy->hHeadTrackData->Quaternion.w = 1.0f; + decDummy->hHeadTrackData->Quaternion.x = 0.0f; + decDummy->hHeadTrackData->Quaternion.y = 0.0f; + decDummy->hHeadTrackData->Quaternion.z = 0.0f; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; i++ ) { decDummy->hHeadTrackData->Quaternions[i].w = 1.0f; @@ -3227,6 +3328,7 @@ static DecoderDummy *initDecoderDummy( decDummy->hHeadTrackData->Quaternions[i].z = 0.0f; } decDummy->hHeadTrackData->num_quaternions = 0; +#endif decDummy->hHeadTrackData->lrSwitchInterpVal = 0.0f; decDummy->hHeadTrackData->lrSwitchedCurrent = 0; decDummy->hHeadTrackData->lrSwitchedNext = 0; @@ -3402,8 +3504,16 @@ static void freeDecoderDummy( pDecDummy->hoa_dec_mtx = NULL; } +#ifdef LIB_REND_FIX_HRTFPARAMBIN_MEMLEAK /* Parametric binaural renderer HRTF structure */ - free( pDecDummy->hHrtfParambin ); + if ( pDecDummy->hHrtfParambin != NULL ) + { +#endif + free( pDecDummy->hHrtfParambin ); +#ifdef LIB_REND_FIX_HRTFPARAMBIN_MEMLEAK + pDecDummy->hHrtfParambin = NULL; + } +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT /* Split binaural renderr handle */ @@ -4133,14 +4243,33 @@ ivas_error IVAS_REND_AddInput( } #ifdef SPLIT_REND_WITH_HEAD_ROT - if ( hIvasRend->splitRendEncBuffer.data == NULL && hIvasRend->hRendererConfig != NULL ) + if ( +#ifdef API_5MS + ( hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) && +#endif + hIvasRend->splitRendEncBuffer.data == NULL && hIvasRend->hRendererConfig != NULL ) { int16_t cldfb_in; cldfb_in = getCldfbRendFlag( hIvasRend, getAudioConfigType( inConfig ) ); #ifdef FIX_658_SPLIT_REND_MASA - ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, ( cldfb_in == 0 ), hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ); +#ifdef API_5MS + error = +#endif + ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, +#ifdef API_5MS + &hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, +#endif + ( cldfb_in == 0 ), + hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ); +#ifdef API_5MS + if ( error != IVAS_ERR_OK ) + { + return error; + } +#endif + #endif if ( ( error = initSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, hIvasRend->headRotData, hIvasRend->sampleRateOut, hIvasRend->outputConfig, cldfb_in ) ) != IVAS_ERR_OK ) @@ -4672,7 +4801,7 @@ ivas_error IVAS_REND_FeedInputAudio( if ( inputAudio.config.numSamplesPerChannel <= 0 || MAX_BUFFER_LENGTH_PER_CHANNEL < inputAudio.config.numSamplesPerChannel ) #endif { - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Buffer size outside of supported range" ); } if ( inputAudio.config.numChannels <= 0 || MAX_INPUT_CHANNELS < inputAudio.config.numChannels ) @@ -4682,14 +4811,15 @@ ivas_error IVAS_REND_FeedInputAudio( #ifdef SPLIT_REND_WITH_HEAD_ROT if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && + hIvasRend->outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_CODED && + hIvasRend->outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_PCM && ( inputAudio.config.numSamplesPerChannel * 1000 / cldfb2tdSampleFact ) != BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->sampleRateOut ) #else if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && inputAudio.config.numSamplesPerChannel * 1000 != BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->sampleRateOut ) #endif { - /* Binaural rendering requires specific frame size */ - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Binaural rendering requires specific frame size" ); } if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) @@ -4926,6 +5056,9 @@ int16_t IVAS_REND_GetRenderConfig( hRCout->split_rend_config.dof = 3; hRCout->split_rend_config.hq_mode = 0; hRCout->split_rend_config.codec_delay_ms = 0; +#ifdef API_5MS + hRCout->split_rend_config.codec_frame_size_ms = 0; /* 0 means "use default for selected codec" */ +#endif hRCout->split_rend_config.codec = IVAS_SPLIT_REND_CODEC_DEFAULT; hRCout->split_rend_config.poseCorrectionMode = IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB; hRCout->split_rend_config.rendererSelection = hRCin->split_rend_config.rendererSelection; @@ -5015,10 +5148,19 @@ int16_t IVAS_REND_FeedRenderConfig( int16_t cldfb_in; cldfb_in = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); closeSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer ); - +#ifdef API_5MS + if ( ( error = ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, + &hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, + ( cldfb_in == 0 ), + hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) != IVAS_ERR_OK ) + { + return error; + } +#else ivas_split_rend_choose_default_codec( &hIvasRend->hRendererConfig->split_rend_config.codec, ( cldfb_in == 0 ), hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ); +#endif if ( ( error = initSplitRend( &hIvasRend->splitRendWrapper, &hIvasRend->splitRendEncBuffer, &hIvasRend->hRendererConfig->split_rend_config, @@ -5035,6 +5177,41 @@ int16_t IVAS_REND_FeedRenderConfig( return IVAS_ERR_OK; } +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +/*-------------------------------------------------------------------* + * IVAS_REND_FeedSplitBinauralBitstream() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_FeedSplitBinauralBitstream( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + IVAS_REND_BitstreamBuffer *hBits /* i : buffer for input bitstream */ +) +{ + ivas_error error; + input_base *inputBase; + input_split_post_rend *inputSplitPostRend; + + /* Validate function arguments */ + if ( hIvasRend == NULL || hBits == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + if ( ( error = getInputById( hIvasRend, inputId, (void **) &inputBase ) ) != IVAS_ERR_OK ) + { + return error; + } + + inputSplitPostRend = (input_split_post_rend *) inputBase; + inputSplitPostRend->hBits = hBits; + + return IVAS_ERR_OK; +} +#endif + /*-------------------------------------------------------------------* * IVAS_REND_SetHeadRotation() @@ -5043,16 +5220,23 @@ int16_t IVAS_REND_FeedRenderConfig( *-------------------------------------------------------------------*/ ivas_error IVAS_REND_SetHeadRotation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ +#ifdef API_5MS + const IVAS_QUATERNION headRot, /* i : head orientations for next rendering call */ + const IVAS_VECTOR3 Pos /* i : listener positions for next rendering call */ +#else const IVAS_QUATERNION headRot[RENDERER_HEAD_POSITIONS_PER_FRAME], /* i : head orientations for next rendering call */ const IVAS_VECTOR3 Pos[RENDERER_HEAD_POSITIONS_PER_FRAME] /* i : listener positions for next rendering call */ +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , IVAS_SPLIT_REND_ROT_AXIS rot_axis #endif ) { +#ifndef API_5MS int16_t i; +#endif IVAS_QUATERNION rotQuat; /* Validate function arguments */ @@ -5067,6 +5251,20 @@ ivas_error IVAS_REND_SetHeadRotation( return IVAS_ERR_INVALID_OUTPUT_FORMAT; } +#ifdef API_5MS + /* check for Euler angle signaling */ + if ( headRot.w == -3.0f ) + { + Euler2Quat( deg2rad( headRot.x ), deg2rad( headRot.y ), deg2rad( headRot.z ), &rotQuat ); + } + else + { + rotQuat = headRot; + } + + ivas_orient_trk_Process( hIvasRend->headRotData.hOrientationTracker, rotQuat, FRAMES_PER_SEC * MAX_PARAM_SPATIAL_SUBFRAMES, &hIvasRend->headRotData.headPosition ); + hIvasRend->headRotData.Pos = Pos; +#else if ( headRot == NULL ) { #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -5123,6 +5321,7 @@ ivas_error IVAS_REND_SetHeadRotation( hIvasRend->headRotData.Pos[i] = Pos[i]; } } +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT hIvasRend->headRotData.sr_pose_pred_axis = rot_axis; @@ -5131,6 +5330,21 @@ ivas_error IVAS_REND_SetHeadRotation( return IVAS_ERR_OK; } +#ifdef API_5MS +ivas_error IVAS_REND_DisableHeadRotation( + IVAS_REND_HANDLE hIvasRend /* i/o: Renderer handle */ +) +{ + /* Validate function arguments */ + if ( hIvasRend == NULL ) + { + return IVAS_ERR_UNEXPECTED_NULL_POINTER; + } + + hIvasRend->headRotData.headRotEnabled = 0; + return IVAS_ERR_OK; +} +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT /*-------------------------------------------------------------------* @@ -5276,15 +5490,24 @@ ivas_error IVAS_REND_SetReferenceVector( *---------------------------------------------------------------------*/ ivas_error IVAS_REND_SetExternalOrientation( - IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_QUATERNION *orientation, /* i : external orientation data */ + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_QUATERNION *orientation, /* i : external orientation data */ +#ifdef API_5MS + int8_t enableHeadRotation, /* i : flag to enable head rotation for this frame */ + int8_t enableExternalOrientation, /* i : flag to enable external orientation for this frame */ + int8_t enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ + int16_t numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#else int8_t *enableHeadRotation, /* i : flag to enable head rotation for this frame */ int8_t *enableExternalOrientation, /* i : flag to enable external orientation for this frame */ int8_t *enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ int16_t *numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#endif ) { +#ifndef API_5MS int16_t i; +#endif /* Validate function arguments */ if ( hIvasRend == NULL || hIvasRend->hExternalOrientationData == NULL ) @@ -5294,22 +5517,45 @@ ivas_error IVAS_REND_SetExternalOrientation( if ( orientation == NULL ) { +#ifdef API_5MS + hIvasRend->hExternalOrientationData->enableExternalOrientation = 0; +#else for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; ++i ) { hIvasRend->hExternalOrientationData->enableExternalOrientation[i] = 0; } +#endif } else { +#ifdef API_5MS + QuaternionInverse( *orientation, &hIvasRend->hExternalOrientationData->Quaternion ); + + hIvasRend->hExternalOrientationData->enableHeadRotation = enableHeadRotation; + hIvasRend->hExternalOrientationData->enableExternalOrientation = enableExternalOrientation; + hIvasRend->hExternalOrientationData->enableRotationInterpolation = enableRotationInterpolation; + hIvasRend->hExternalOrientationData->numFramesToTargetOrientation = numFramesToTargetOrientation; +#else for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; ++i ) { +#ifdef API_5MS + /* just fix compilation issues, ToDo: change ext renderer also to 5ms */ + QuaternionInverse( orientation[i], &hIvasRend->hExternalOrientationData->Quaternion ); + + hIvasRend->hExternalOrientationData->enableHeadRotation = enableHeadRotation[i]; + hIvasRend->hExternalOrientationData->enableExternalOrientation = enableExternalOrientation[i]; + hIvasRend->hExternalOrientationData->enableRotationInterpolation = enableRotationInterpolation[i]; + hIvasRend->hExternalOrientationData->numFramesToTargetOrientation = numFramesToTargetOrientation[i]; +#else QuaternionInverse( orientation[i], &hIvasRend->hExternalOrientationData->Quaternions[i] ); hIvasRend->hExternalOrientationData->enableHeadRotation[i] = enableHeadRotation[i]; hIvasRend->hExternalOrientationData->enableExternalOrientation[i] = enableExternalOrientation[i]; hIvasRend->hExternalOrientationData->enableRotationInterpolation[i] = enableRotationInterpolation[i]; hIvasRend->hExternalOrientationData->numFramesToTargetOrientation[i] = numFramesToTargetOrientation[i]; +#endif } +#endif } return IVAS_ERR_OK; @@ -5346,7 +5592,9 @@ ivas_error IVAS_REND_GetCombinedOrientation( IVAS_QUATERNION *pOrientation /* i/o: Quaternion pointer processed orientation */ ) { +#ifndef API_5MS int16_t i; +#endif if ( hIvasRend == NULL || pOrientation == NULL ) { @@ -5355,10 +5603,19 @@ ivas_error IVAS_REND_GetCombinedOrientation( if ( hIvasRend->hCombinedOrientationData != NULL ) { +#ifdef API_5MS + *pOrientation = hIvasRend->hCombinedOrientationData->Quaternion; +#else for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; ++i ) { +#ifdef API_5MS + /* fix compilation only,ToDo adapt ext renderer to 5ms , this functions looks stale anyway */ + pOrientation[i] = hIvasRend->hCombinedOrientationData->Quaternion; +#else pOrientation[i] = hIvasRend->hCombinedOrientationData->Quaternions[i]; +#endif } +#endif } return IVAS_ERR_OK; @@ -5455,11 +5712,34 @@ static void renderBufferChannel( return; } +#ifdef API_5MS +static ivas_error chooseCrossfade( const IVAS_REND_HeadRotData *headRotData, int16_t numSamplesPerChannel, const float **pCrossfade ) +{ + if ( numSamplesPerChannel == L_FRAME48k ) + { + *pCrossfade = headRotData->crossfade_20ms; + } + else if ( numSamplesPerChannel == L_FRAME48k / MAX_PARAM_SPATIAL_SUBFRAMES ) + { + *pCrossfade = headRotData->crossfade_5ms; + } + else + { + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Only 5 and 20 ms framing is supported" ); + } + *pCrossfade = headRotData->crossfade_5ms; /* TODO cleanup to one crossfade only */ + return IVAS_ERR_OK; +} +#endif static ivas_error rotateFrameMc( - IVAS_REND_AudioBuffer inAudio, /* i : Input Audio buffer */ - AUDIO_CONFIG inConfig, /* i : Input Audio config */ - LSSETUP_CUSTOM_STRUCT inCustomLs, /* i : Input Custom LS setup */ + IVAS_REND_AudioBuffer inAudio, /* i : Input Audio buffer */ + AUDIO_CONFIG inConfig, /* i : Input Audio config */ +#ifdef API_5MS + const LSSETUP_CUSTOM_STRUCT *pInCustomLs, /* i : Input Custom LS setup */ +#else + LSSETUP_CUSTOM_STRUCT inCustomLs, /* i : Input Custom LS setup */ +#endif const IVAS_REND_HeadRotData *headRotData, /* i : Head rotation data */ const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData, /* i : Combined head and external orientations */ rotation_gains gains_prev, /* i/o: Previous frame rotation gains */ @@ -5469,7 +5749,12 @@ static ivas_error rotateFrameMc( { int16_t i; int16_t j; +#ifdef API_5MS + const float *crossfade; + int16_t subframe_idx, subframe_len, num_subframes; +#else int16_t subframe_idx, subframe_len; +#endif int16_t azimuth, elevation; int16_t is_planar_setup, lfe_idx; int16_t nchan; @@ -5484,6 +5769,13 @@ static ivas_error rotateFrameMc( push_wmops( "rotateFrameMc" ); +#ifdef API_5MS + if ( ( error = chooseCrossfade( headRotData, inAudio.config.numSamplesPerChannel, &crossfade ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif + if ( inConfig != AUDIO_CONFIG_LS_CUSTOM ) { if ( ( error = getAudioConfigNumChannels( inConfig, &nchan ) ) != IVAS_ERR_OK ) @@ -5493,10 +5785,20 @@ static ivas_error rotateFrameMc( } else { +#ifdef API_5MS + nchan = pInCustomLs->num_spk + pInCustomLs->num_lfe; +#else nchan = inCustomLs.num_spk + inCustomLs.num_lfe; +#endif } - if ( ( error = getMcConfigValues( inConfig, inCustomLs, &ls_azimuth, &ls_elevation, &lfe_idx, &is_planar_setup ) ) != IVAS_ERR_OK ) + if ( ( error = getMcConfigValues( inConfig, +#ifdef API_5MS + pInCustomLs, +#else + inCustomLs, +#endif + &ls_azimuth, &ls_elevation, &lfe_idx, &is_planar_setup ) ) != IVAS_ERR_OK ) { return error; } @@ -5508,17 +5810,30 @@ static ivas_error rotateFrameMc( gains[ch_in][ch_in] = 1.f; } +#ifdef API_5MS + /* subframe loop */ + num_subframes = inAudio.config.numSamplesPerChannel / ( L_FRAME48k / RENDERER_SUBFRAMES_PER_FRAME ); + subframe_len = inAudio.config.numSamplesPerChannel / num_subframes; + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + { +#else /* subframe loop */ subframe_len = inAudio.config.numSamplesPerChannel / RENDERER_HEAD_POSITIONS_PER_FRAME; for ( subframe_idx = 0; subframe_idx < RENDERER_HEAD_POSITIONS_PER_FRAME; subframe_idx++ ) { +#endif for ( i = 0; i < 3; i++ ) { if ( hCombinedOrientationData != NULL ) { for ( j = 0; j < 3; j++ ) { +#ifdef API_5MS + /* fix compiling only, ToDo adapt ext renderer to 5ms */ + Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[i][j]; +#else Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; +#endif } } else @@ -5574,8 +5889,13 @@ static ivas_error rotateFrameMc( for ( i = 0; i < subframe_len; i++ ) { *writePtr++ += +#ifdef API_5MS /* :TODO: cleanup to one crossfade only */ + ( *readPtr ) * ( ( 1 - crossfade[i] ) * gains_prev[ch_in][ch_out] ) + + ( *readPtr ) * ( crossfade[i] * gains[ch_in][ch_out] ); +#else ( *readPtr ) * ( ( 1 - headRotData->crossfade[i] ) * gains_prev[ch_in][ch_out] ) + ( *readPtr ) * ( headRotData->crossfade[i] * gains[ch_in][ch_out] ); +#endif readPtr++; } } @@ -5587,7 +5907,6 @@ static ivas_error rotateFrameMc( mvr2r( gains[i], gains_prev[i], nchan ); } } - pop_wmops(); return IVAS_ERR_OK; @@ -5606,7 +5925,14 @@ static ivas_error rotateFrameSba( int16_t i, l, n, m; int16_t m1, m2; int16_t shd_rot_max_order; +#ifdef API_5MS + const float *crossfade; + int16_t num_subframes; +#endif int16_t subframe_idx, subframe_len; +#ifdef API_5MS + float *readPtr; +#endif float *writePtr; rotation_matrix Rmat; float tmpRot[2 * HEADROT_ORDER + 1]; @@ -5615,17 +5941,33 @@ static ivas_error rotateFrameSba( int16_t idx; float val, cf, oneminuscf; + push_wmops( "rotateFrameSba" ); +#ifdef API_5MS + if ( ( error = chooseCrossfade( headRotData, inAudio.config.numSamplesPerChannel, &crossfade ) ) != IVAS_ERR_OK ) + { + return error; + } +#endif + if ( ( error = getAmbisonicsOrder( inConfig, &shd_rot_max_order ) ) != IVAS_ERR_OK ) { return error; } +#ifdef API_5MS + /* subframe loop */ + num_subframes = inAudio.config.numSamplesPerChannel / ( L_FRAME48k / RENDERER_SUBFRAMES_PER_FRAME ); + subframe_len = inAudio.config.numSamplesPerChannel / num_subframes; + for ( subframe_idx = 0; subframe_idx < num_subframes; subframe_idx++ ) + { +#else /* subframe loop */ subframe_len = inAudio.config.numSamplesPerChannel / RENDERER_HEAD_POSITIONS_PER_FRAME; for ( subframe_idx = 0; subframe_idx < RENDERER_HEAD_POSITIONS_PER_FRAME; subframe_idx++ ) { +#endif /* initialize rotation matrices with zeros */ for ( i = 0; i < HEADROT_SHMAT_DIM; i++ ) { @@ -5638,7 +5980,12 @@ static ivas_error rotateFrameSba( { for ( l = 0; l < 3; l++ ) { +#ifdef API_5MS + /* fix compilation only, ToDo adapt ext renderer to 5ms */ + Rmat[i][l] = ( *hCombinedOrientationData )->Rmat[i][l]; +#else Rmat[i][l] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][l]; +#endif } } else @@ -5651,12 +5998,16 @@ static ivas_error rotateFrameSba( /* calculate ambisonics rotation matrices for the previous and current frames */ SHrotmatgen( gains, Rmat, shd_rot_max_order ); - for ( i = 0; i < subframe_len; i++ ) { idx = subframe_idx * subframe_len + i; +#ifdef API_5MS + cf = crossfade[i]; +#else cf = headRotData->crossfade[i]; +#endif oneminuscf = 1 - cf; + /* As the rotation matrix becomes block diagonal in a SH basis, we can*/ /* apply each angular-momentum block individually to save complexity. */ @@ -5675,6 +6026,7 @@ static ivas_error rotateFrameSba( val = inAudio.data[m * inAudio.config.numSamplesPerChannel + idx]; /* crossfade with previous rotation gains */ tmpRot[n - m1] += ( cf * gains[n][m] * val + oneminuscf * gains_prev[n][m] * val ); + // merge end } } /* write back the result */ @@ -5729,8 +6081,18 @@ static ivas_error renderIsmToBinaural( ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); copyBufferTo2dArray( ismInput->base.inputBuffer, tmpTDRendBuffer ); - if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, - ismInput->hReverb, ism_md_subframe_update_ext, outAudio.config.numSamplesPerChannel, tmpTDRendBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, + ismInput->base.inConfig, + NULL, + ismInput->base.ctx.pCombinedOrientationData, + &ismInput->currentPos, + ismInput->hReverb, + ism_md_subframe_update_ext, +#ifdef API_5MS + *ismInput->base.ctx.pOutSampleRate, +#endif + outAudio.config.numSamplesPerChannel, + tmpTDRendBuffer ) ) != IVAS_ERR_OK ) { return error; } @@ -5742,6 +6104,30 @@ static ivas_error renderIsmToBinaural( return IVAS_ERR_OK; } +#ifdef API_5MS +static int16_t getNumSubframesInBuffer( const IVAS_REND_AudioBuffer *buffer, int32_t sampleRate ) +{ +#ifdef SPLIT_REND_WITH_HEAD_ROT + int16_t cldfb2tdSampleFact; + + cldfb2tdSampleFact = buffer->config.is_cldfb ? 2 : 1; +#endif + +#ifdef DEBUGGING +#ifdef SPLIT_REND_WITH_HEAD_ROT + assert( buffer->config.numSamplesPerChannel % ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES * cldfb2tdSampleFact ) == 0 ); +#else + assert( buffer->config.numSamplesPerChannel % ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) == 0 ); +#endif +#endif + +#ifdef SPLIT_REND_WITH_HEAD_ROT + return (int16_t) ( buffer->config.numSamplesPerChannel / ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES * cldfb2tdSampleFact ) ); +#else + return (int16_t) ( buffer->config.numSamplesPerChannel / ( sampleRate / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); +#endif +} +#endif static ivas_error renderIsmToBinauralRoom( input_ism *ismInput, @@ -5749,7 +6135,9 @@ static ivas_error renderIsmToBinauralRoom( { int16_t i; int16_t azi_rot, ele_rot; +#ifndef API_5MS int16_t subframe_idx, subframe_len; +#endif int16_t tmp; rotation_matrix Rmat; float tmpRendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; @@ -5776,31 +6164,53 @@ static ivas_error renderIsmToBinauralRoom( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + /* fix compilation only, ToDo adapt ext renderer to 5ms */ + if ( ( *hCombinedOrientationData )->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { + if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) { combinedOrientationEnabled = 1; break; } } +#endif } if ( combinedOrientationEnabled ) { +#ifndef API_5MS subframe_len = ismInput->base.inputBuffer.config.numSamplesPerChannel / RENDERER_HEAD_POSITIONS_PER_FRAME; // for ( subframe_idx = 0; subframe_idx < RENDERER_HEAD_POSITIONS_PER_FRAME; subframe_idx++ ) for ( subframe_idx = 0; subframe_idx < 1; subframe_idx++ ) { +#endif for ( i = 0; i < 3; i++ ) { - if ( hCombinedOrientationData != NULL && ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] ) +#ifdef API_5MS + /* at least this looks like it is adapted to 5ms already since it "loops" only over the first subframe, ToDo needs to be checked */ + if ( hCombinedOrientationData != NULL && ( *hCombinedOrientationData )->enableCombinedOrientation ) { for ( j = 0; j < 3; j++ ) { - Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; + Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[i][j]; } } +#else + if ( hCombinedOrientationData != NULL && ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] ) + { + for ( j = 0; j < 3; j++ ) + { + Rmat[i][j] = ( *hCombinedOrientationData )->Rmat[subframe_idx][i][j]; + } + } +#endif else { /* Set to identity */ @@ -5808,8 +6218,10 @@ static ivas_error renderIsmToBinauralRoom( Rmat[i][i] = 1.0f; } } +#ifndef API_5MS } (void) subframe_len; /* avoid warning */ +#endif } /* TODO tmu : see issue #518 */ @@ -5870,8 +6282,12 @@ static ivas_error renderIsmToBinauralRoom( if ( ( error = ivas_rend_crendProcess( ismInput->crendWrapper, AUDIO_CONFIG_7_1_4, AUDIO_CONFIG_BINAURAL_ROOM_IR, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *ismInput->base.ctx.pOutSampleRate -#ifdef SPLIT_REND_WITH_HEAD_ROT +#ifdef API_5MS , + getNumSubframesInBuffer( &outAudio, *ismInput->base.ctx.pOutSampleRate ) +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT + , 0 #endif ) ) != IVAS_ERR_OK ) @@ -5902,8 +6318,18 @@ static ivas_error renderIsmToBinauralReverb( ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); copyBufferTo2dArray( ismInput->base.inputBuffer, tmpRendBuffer ); - if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, - ismInput->hReverb, ism_md_subframe_update_ext, outAudio.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, + ismInput->base.inConfig, + NULL, + ismInput->base.ctx.pCombinedOrientationData, + &ismInput->currentPos, + ismInput->hReverb, + ism_md_subframe_update_ext, +#ifdef API_5MS + *ismInput->base.ctx.pOutSampleRate, +#endif + outAudio.config.numSamplesPerChannel, + tmpRendBuffer ) ) != IVAS_ERR_OK ) { return error; } @@ -6035,13 +6461,19 @@ static ivas_error renderIsmToSplitBinaural( input_ism *ismInput, const IVAS_REND_AudioBuffer outAudio ) { +#ifndef API_5MS int32_t i; +#endif ivas_error error; float tmpProcessing[MAX_NUM_OBJECTS][L_FRAME48k]; int16_t pos_idx; const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; const SPLIT_REND_WRAPPER *pSplitRendWrapper; +#ifdef API_5MS + IVAS_QUATERNION originalHeadRot; +#else IVAS_QUATERNION originalHeadRot[MAX_PARAM_SPATIAL_SUBFRAMES]; +#endif float tmpBinaural[MAX_HEAD_ROT_POSES * BINAURAL_CHANNELS][L_FRAME48k]; int16_t output_frame = ismInput->base.inputBuffer.config.numSamplesPerChannel; COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; @@ -6055,42 +6487,29 @@ static ivas_error renderIsmToSplitBinaural( /* Metadata Delay to sync with audio delay converted from ms to 5ms (1000/50/4) subframe index */ ism_md_subframe_update_ext = (int16_t) round( ismInput->ism_metadata_delay_ms / ( 1000 / FRAMES_PER_SEC / MAX_PARAM_SPATIAL_SUBFRAMES ) ); - /* If not yet allocated, open additional instances of TD renderer */ - for ( i = 0; i < pMultiBinPoseData->num_poses - 1; ++i ) - { - if ( ismInput->splitTdRendWrappers[i].hBinRendererTd != NULL ) - { - continue; - } - - /* ToDo: Could re-use already existing HRTF (ismInput->tdRendWrapper.hHrtfTD), but this complicates internal memory handling in TD renderer */ - ismInput->splitTdRendWrappers[i].hHrtfTD = NULL; - - /* Open TD renderer wrapper */ - if ( ( error = ivas_td_binaural_open_ext( &ismInput->splitTdRendWrappers[i], ismInput->base.inConfig, *ismInput->base.ctx.hhRendererConfig, NULL, *ismInput->base.ctx.pOutSampleRate ) ) != IVAS_ERR_OK ) - { - return error; - } - - /* Assert same delay as main TD renderer */ - assert( ismInput->splitTdRendWrappers[i].binaural_latency_ns == ismInput->tdRendWrapper.binaural_latency_ns ); - } - pCombinedOrientationData = *ismInput->base.ctx.pCombinedOrientationData; if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { +#ifdef API_5MS + pCombinedOrientationData->Quaternion = pCombinedOrientationData->Quaternion; +#else for ( i = 1; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { pCombinedOrientationData->Quaternions[i] = pCombinedOrientationData->Quaternions[0]; } +#endif } /* Save current head positions */ +#ifdef API_5MS + originalHeadRot = pCombinedOrientationData->Quaternion; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { originalHeadRot[i] = pCombinedOrientationData->Quaternions[i]; } +#endif /* Copy input audio to a processing buffer. */ copyBufferTo2dArray( ismInput->base.inputBuffer, tmpProcessing ); @@ -6100,6 +6519,26 @@ static ivas_error renderIsmToSplitBinaural( /* Update head positions */ if ( pos_idx != 0 ) { +#ifdef API_5MS + if ( originalHeadRot.w == -3.0f ) + { + pCombinedOrientationData->Quaternion.w = -3.0f; + pCombinedOrientationData->Quaternion.x = originalHeadRot.x + pMultiBinPoseData->relative_head_poses[pos_idx][0]; + pCombinedOrientationData->Quaternion.y = originalHeadRot.y + pMultiBinPoseData->relative_head_poses[pos_idx][1]; + pCombinedOrientationData->Quaternion.z = originalHeadRot.z + pMultiBinPoseData->relative_head_poses[pos_idx][2]; + } + else + { + pCombinedOrientationData->Quaternion.w = -3.0f; + Quat2EulerDegree( originalHeadRot, + &pCombinedOrientationData->Quaternion.z, + &pCombinedOrientationData->Quaternion.y, + &pCombinedOrientationData->Quaternion.x ); + pCombinedOrientationData->Quaternion.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + pCombinedOrientationData->Quaternion.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + pCombinedOrientationData->Quaternion.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; + } +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { if ( originalHeadRot[i].w == -3.0f ) @@ -6121,24 +6560,25 @@ static ivas_error renderIsmToSplitBinaural( pCombinedOrientationData->Quaternions[i].z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; } } +#endif } /* Render */ - if ( pos_idx == 0 ) + error = ivas_td_binaural_renderer_ext( ( pos_idx == 0 ) ? &ismInput->tdRendWrapper : &ismInput->splitTdRendWrappers[pos_idx - 1], + ismInput->base.inConfig, + NULL, + ismInput->base.ctx.pCombinedOrientationData, + &ismInput->currentPos, + NULL, + ism_md_subframe_update_ext, +#ifdef API_5MS + *ismInput->base.ctx.pOutSampleRate, +#endif + output_frame, + tmpProcessing ); + if ( error != IVAS_ERR_OK ) { - if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->tdRendWrapper, ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, NULL, - ism_md_subframe_update_ext, output_frame, tmpProcessing ) ) != IVAS_ERR_OK ) - { - return error; - } - } - else - { - if ( ( error = ivas_td_binaural_renderer_ext( &ismInput->splitTdRendWrappers[pos_idx - 1], ismInput->base.inConfig, NULL, ismInput->base.ctx.pCombinedOrientationData, &ismInput->currentPos, NULL, - ism_md_subframe_update_ext, output_frame, tmpProcessing ) ) != IVAS_ERR_OK ) - { - return error; - } + return error; } /* Copy rendered audio to tmp storage buffer. Copying directly to output would @@ -6151,10 +6591,14 @@ static ivas_error renderIsmToSplitBinaural( } /* Restore original head rotation */ +#ifdef API_5MS + pCombinedOrientationData->Quaternion = originalHeadRot; +#else for ( i = 0; i < MAX_PARAM_SPATIAL_SUBFRAMES; ++i ) { pCombinedOrientationData->Quaternions[i] = originalHeadRot[i]; } +#endif accumulate2dArrayToBuffer( tmpBinaural, &outAudio ); pop_wmops(); @@ -6198,8 +6642,7 @@ static ivas_error renderInputIsm( if ( ismInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) { - /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); } ismInput->base.numNewSamplesPerChannel = 0; @@ -6384,7 +6827,9 @@ static ivas_error renderMcToBinaural( IVAS_REND_AudioBuffer tmpRotBuffer; const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; int8_t combinedOrientationEnabled; +#ifndef API_5MS int16_t subframe_idx; +#endif float *p_tmpRendBuffer[MAX_OUTPUT_CHANNELS]; int16_t i; @@ -6401,6 +6846,14 @@ static ivas_error renderMcToBinaural( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + /* fix compiling only, ToDo ext renderer needs to adapted to 5ms */ + if ( ( *hCombinedOrientationData )->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } + +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) @@ -6409,14 +6862,22 @@ static ivas_error renderMcToBinaural( break; } } +#endif } if ( ( inConfig == AUDIO_CONFIG_LS_CUSTOM ) || ( combinedOrientationEnabled && ( inConfig == AUDIO_CONFIG_5_1 || inConfig == AUDIO_CONFIG_7_1 ) ) ) { copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); - if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, mcInput->base.ctx.pCombinedOrientationData, - NULL, mcInput->hReverb, 0, mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_td_binaural_renderer_ext( &mcInput->tdRendWrapper, mcInput->base.inConfig, &mcInput->customLsInput, + mcInput->base.ctx.pCombinedOrientationData, + NULL, + mcInput->hReverb, + 0, +#ifdef API_5MS + *mcInput->base.ctx.pOutSampleRate, +#endif + mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { return error; } @@ -6430,7 +6891,13 @@ static ivas_error renderMcToBinaural( tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, +#ifdef API_5MS + &mcInput->customLsInput, +#else + mcInput->customLsInput, +#endif + mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, #ifdef SPLIT_REND_WITH_HEAD_ROT mcInput->rot_gains_prev[0], @@ -6452,12 +6919,15 @@ static ivas_error renderMcToBinaural( /* call CREND */ if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate -#ifdef SPLIT_REND_WITH_HEAD_ROT +#ifdef API_5MS , + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT + , 0 #endif ) ) != IVAS_ERR_OK ) - { return error; } @@ -6494,7 +6964,9 @@ static ivas_error renderMcToBinauralRoom( int16_t i; const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; int8_t combinedOrientationEnabled; +#ifndef API_5MS int16_t subframe_idx; +#endif for ( i = 0; i < MAX_OUTPUT_CHANNELS; i++ ) { @@ -6509,6 +6981,13 @@ static ivas_error renderMcToBinauralRoom( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + /* fix compiling only, ToDo ext renderer needs to adapted to 5ms */ + if ( ( *hCombinedOrientationData )->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) @@ -6517,6 +6996,7 @@ static ivas_error renderMcToBinauralRoom( break; } } +#endif } if ( ( mcInput->hReverb != NULL && outConfig == AUDIO_CONFIG_BINAURAL_ROOM_REVERB ) && ( ( inConfig == AUDIO_CONFIG_LS_CUSTOM ) || ( combinedOrientationEnabled && ( inConfig == AUDIO_CONFIG_5_1 || inConfig == AUDIO_CONFIG_7_1 ) ) ) ) @@ -6527,6 +7007,9 @@ static ivas_error renderMcToBinauralRoom( mcInput->base.ctx.pCombinedOrientationData, NULL, mcInput->hReverb, 0, +#ifdef API_5MS + *mcInput->base.ctx.pOutSampleRate, +#endif mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { return error; @@ -6541,7 +7024,13 @@ static ivas_error renderMcToBinauralRoom( tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, +#ifdef API_5MS + &mcInput->customLsInput, +#else + mcInput->customLsInput, +#endif + mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, #ifdef SPLIT_REND_WITH_HEAD_ROT mcInput->rot_gains_prev[0], @@ -6562,9 +7051,14 @@ static ivas_error renderMcToBinauralRoom( } /* call CREND */ - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate -#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, + NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate +#ifdef API_5MS , + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT + , 0 #endif ) ) != IVAS_ERR_OK ) @@ -6605,7 +7099,9 @@ static ivas_error renderMcCustomLsToBinauralRoom( float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; int8_t combinedOrientationEnabled; +#ifndef API_5MS int16_t subframe_idx; +#endif push_wmops( "renderMcCustomLsToBinauralRoom" ); @@ -6620,6 +7116,13 @@ static ivas_error renderMcCustomLsToBinauralRoom( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + /* fix compilin only, ToDo adapt renderer to 5ms */ + if ( ( *hCombinedOrientationData )->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) @@ -6628,6 +7131,7 @@ static ivas_error renderMcCustomLsToBinauralRoom( break; } } +#endif } /* apply rotation */ @@ -6637,7 +7141,13 @@ static ivas_error renderMcCustomLsToBinauralRoom( tmpRotBuffer.data = malloc( tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels * sizeof( float ) ); set_zero( tmpRotBuffer.data, tmpRotBuffer.config.numSamplesPerChannel * tmpRotBuffer.config.numChannels ); - if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, mcInput->customLsInput, mcInput->base.ctx.pHeadRotData, + if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, +#ifdef API_5MS + &mcInput->customLsInput, +#else + mcInput->customLsInput, +#endif + mcInput->base.ctx.pHeadRotData, mcInput->base.ctx.pCombinedOrientationData, #ifdef SPLIT_REND_WITH_HEAD_ROT mcInput->rot_gains_prev[0], @@ -6672,8 +7182,12 @@ static ivas_error renderMcCustomLsToBinauralRoom( /* call CREND */ if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, AUDIO_CONFIG_7_1_4, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *mcInput->base.ctx.pOutSampleRate -#ifdef SPLIT_REND_WITH_HEAD_ROT +#ifdef API_5MS , + getNumSubframesInBuffer( &outAudio, *mcInput->base.ctx.pOutSampleRate ) +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT + , 0 #endif ) ) != IVAS_ERR_OK ) @@ -6774,7 +7288,10 @@ static ivas_error renderMcToSplitBinaural( const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio ) { - int16_t i, j, sf, pos_idx; + int16_t i, j, pos_idx; +#ifndef API_5MS + int16_t sf; +#endif int16_t output_frame; ivas_error error; const MULTI_BIN_REND_POSE_DATA *pMultiBinPoseData; @@ -6805,6 +7322,16 @@ static ivas_error renderMcToSplitBinaural( combinedOrientationDataLocal = *pCombinedOrientationDataLocal; if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { +#ifdef API_5MS + combinedOrientationDataLocal.Quaternion = combinedOrientationDataLocal.Quaternion; + for ( i = 0; i < 3; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + combinedOrientationDataLocal.Rmat[i][j] = combinedOrientationDataLocal.Rmat[i][j]; + } + } +#else for ( sf = 1; sf < RENDERER_HEAD_POSITIONS_PER_FRAME; ++sf ) { combinedOrientationDataLocal.Quaternions[sf] = combinedOrientationDataLocal.Quaternions[0]; @@ -6816,6 +7343,7 @@ static ivas_error renderMcToSplitBinaural( } } } +#endif } /* temporary buffer for rotation in source format for CREND */ @@ -6828,6 +7356,18 @@ static ivas_error renderMcToSplitBinaural( for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) { /* Update head positions */ +#ifdef API_5MS + IVAS_QUATERNION Quaternion_orig, Quaternion_abs; + Quaternion_orig = combinedOrientationDataLocal.Quaternion; + Quaternion_abs.w = -3.0f; + Quat2EulerDegree( combinedOrientationDataLocal.Quaternion, &Quaternion_abs.z, &Quaternion_abs.y, &Quaternion_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + Quaternion_abs.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternion_abs.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternion_abs.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; + combinedOrientationDataLocal.Quaternion = Quaternion_abs; + QuatToRotMat( combinedOrientationDataLocal.Quaternion, combinedOrientationDataLocal.Rmat ); +#else IVAS_QUATERNION Quaternions_orig[RENDERER_HEAD_POSITIONS_PER_FRAME], Quaternions_abs; for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) { @@ -6841,6 +7381,7 @@ static ivas_error renderMcToSplitBinaural( combinedOrientationDataLocal.Quaternions[i] = Quaternions_abs; QuatToRotMat( combinedOrientationDataLocal.Quaternions[i], combinedOrientationDataLocal.Rmat[i] ); } +#endif if ( inConfig == AUDIO_CONFIG_LS_CUSTOM || inConfig == AUDIO_CONFIG_5_1 || inConfig == AUDIO_CONFIG_7_1 ) { @@ -6855,6 +7396,9 @@ static ivas_error renderMcToSplitBinaural( copyBufferTo2dArray( mcInput->base.inputBuffer, tmpRendBuffer ); /* Render */ +#ifdef FIX_RENDMC_LOCAL_ORIENTATION + pCombinedOrientationDataLocal = &combinedOrientationDataLocal; +#endif if ( ( error = ivas_td_binaural_renderer_ext( ( pos_idx == 0 ) ? &mcInput->tdRendWrapper : &mcInput->splitTdRendWrappers[pos_idx - 1], mcInput->base.inConfig, &mcInput->customLsInput, @@ -6862,6 +7406,9 @@ static ivas_error renderMcToSplitBinaural( NULL, mcInput->hReverb, 0, /* Ism Audio Metadata Delay Sync in ms for External Renderer */ +#ifdef API_5MS + *mcInput->base.ctx.pOutSampleRate, +#endif mcInput->base.inputBuffer.config.numSamplesPerChannel, tmpRendBuffer ) ) != IVAS_ERR_OK ) { @@ -6890,7 +7437,11 @@ static ivas_error renderMcToSplitBinaural( pCombinedOrientationDataLocal = &combinedOrientationDataLocal; if ( ( error = rotateFrameMc( mcInput->base.inputBuffer, mcInput->base.inConfig, +#ifdef API_5MS + &mcInput->customLsInput, +#else mcInput->customLsInput, +#endif mcInput->base.ctx.pHeadRotData, &pCombinedOrientationDataLocal, #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -6907,7 +7458,19 @@ static ivas_error renderMcToSplitBinaural( copyBufferTo2dArray( tmpRotBuffer, tmpRendBuffer ); /* call CREND (rotation already performed) */ - if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, mcInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpRendBuffer, *mcInput->base.ctx.pOutSampleRate, pos_idx ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_rend_crendProcess( mcInput->crendWrapper, + mcInput->base.inConfig, + outConfig, + NULL, + NULL, + NULL, + NULL, + p_tmpRendBuffer, + *mcInput->base.ctx.pOutSampleRate, +#ifdef API_5MS + getNumSubframesInBuffer( &mcInput->base.inputBuffer, *mcInput->base.ctx.pOutSampleRate ), +#endif + pos_idx ) ) != IVAS_ERR_OK ) { return error; } @@ -6919,10 +7482,14 @@ static ivas_error renderMcToSplitBinaural( } /* restore original headrotation data */ +#ifdef API_5MS + combinedOrientationDataLocal.Quaternion = Quaternion_orig; +#else for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) { combinedOrientationDataLocal.Quaternions[i] = Quaternions_orig[i]; } +#endif } if ( inConfig != AUDIO_CONFIG_LS_CUSTOM && inConfig != AUDIO_CONFIG_5_1 && inConfig != AUDIO_CONFIG_7_1 ) @@ -6959,8 +7526,7 @@ static ivas_error renderInputMc( if ( mcInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) { - /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); } mcInput->base.numNewSamplesPerChannel = 0; @@ -7107,7 +7673,12 @@ static ivas_error splitBinLc3plusDecode( } /* Read LC3plus bitstream size info */ lc3plusBitrateId = ivas_split_rend_bitstream_read_int32( bits, 8 ); - lc3plusBitstreamSize = ivas_get_lc3plus_size_from_id( (int8_t) lc3plusBitrateId, pose_correction ); + lc3plusBitstreamSize = ivas_get_lc3plus_size_from_id( (int8_t) lc3plusBitrateId, pose_correction +#ifdef API_5MS + , + (int16_t) ( hSplitBin->hLc3plusDec->config.ivas_frame_duration_us / 1000 ) +#endif + ); for ( int16_t i = 0; i < BINAURAL_CHANNELS * hSplitBin->multiBinPoseData.num_poses; ++i ) { @@ -7128,19 +7699,35 @@ static ivas_error splitBinLc3plusDecode( static ivas_error renderSplitBinauralWithPostRot( input_split_post_rend *splitBinInput, IVAS_REND_AudioBuffer outAudio, +#ifndef API_5MS IVAS_REND_BitstreamBuffer *hBits, +#endif const int16_t SplitRendBFI ) { float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; ivas_error error; - IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES]; +#ifdef API_5MS + IVAS_QUATERNION QuaternionPost; + float Cldfb_RealBuffer_Binaural_5ms[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; + float Cldfb_ImagBuffer_Binaural_5ms[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES][CLDFB_NO_CHANNELS_MAX]; +#else + IVAS_QUATERNION QuaternionsPost[MAX_PARAM_SPATIAL_SUBFRAMES]; /* TODO(splitrend): remove subframes */ int16_t sf_idx; +#endif ivas_split_rend_bits_t bits; float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; COMBINED_ORIENTATION_HANDLE pCombinedOrientationData; SPLIT_POST_REND_WRAPPER *hSplitBin; int8_t isPostRendInputCldfb; +#ifdef API_5MS + int16_t chnlIdx, slotIdx, smplIdx; + int16_t preRendFrameSize_ms; + int16_t outBufNumSamplesPerChannel, outBufNumColPerChannel; + int16_t numSamplesPerChannelCacheSize, numColPerChannelCacheSize; + float *readPtr, *writePtr; +#endif + isPostRendInputCldfb = 0; @@ -7150,7 +7737,11 @@ static ivas_error renderSplitBinauralWithPostRot( pCombinedOrientationData = *splitBinInput->base.ctx.pCombinedOrientationData; hSplitBin = &splitBinInput->splitPostRendWrapper; +#ifdef API_5MS + convertBitsBufferToInternalBitsBuff( *splitBinInput->hBits, &bits ); +#else convertBitsBufferToInternalBitsBuff( *hBits, &bits ); +#endif if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD && splitBinInput->splitPostRendWrapper.hSplitBinLCLDDec == NULL ) { @@ -7162,8 +7753,13 @@ static ivas_error renderSplitBinauralWithPostRot( else if ( bits.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && splitBinInput->splitPostRendWrapper.hLc3plusDec == NULL ) { LC3PLUS_CONFIG config; +#ifdef API_5MS + config.lc3plus_frame_duration_us = bits.codec_frame_size_ms * 1000; + config.ivas_frame_duration_us = bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ? config.lc3plus_frame_duration_us : 20000; +#else config.lc3plus_frame_duration_us = 5000; config.ivas_frame_duration_us = 20000; +#endif config.channels = BINAURAL_CHANNELS; config.samplerate = *splitBinInput->base.ctx.pOutSampleRate; @@ -7177,10 +7773,14 @@ static ivas_error renderSplitBinauralWithPostRot( } } +#ifdef API_5MS + QuaternionPost = pCombinedOrientationData->Quaternion; +#else for ( sf_idx = 0; sf_idx < MAX_PARAM_SPATIAL_SUBFRAMES; sf_idx++ ) { QuaternionsPost[sf_idx] = ivas_split_rend_get_sf_rot_data( pCombinedOrientationData->Quaternions, sf_idx ); } +#endif if ( !SplitRendBFI ) { @@ -7207,18 +7807,124 @@ static ivas_error renderSplitBinauralWithPostRot( /* decode audio */ if ( splitBinInput->base.inConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) { +#ifdef API_5MS if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) { - ivas_splitBinLCLDDecProcess( hSplitBin->hSplitBinLCLDDec, &bits, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, SplitRendBFI ); isPostRendInputCldfb = 1; } + + preRendFrameSize_ms = bits.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS ? (int16_t) ( hSplitBin->hLc3plusDec->config.ivas_frame_duration_us ) / 1000 : 20; + + outBufNumSamplesPerChannel = outAudio.config.numSamplesPerChannel; + numSamplesPerChannelCacheSize = (int16_t) ( *splitBinInput->base.ctx.pOutSampleRate * ( preRendFrameSize_ms - bits.codec_frame_size_ms ) / 1000 ); + + outBufNumColPerChannel = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; + numColPerChannelCacheSize = CLDFB_NO_COL_MAX - outBufNumColPerChannel; + + if ( splitBinInput->numCachedSamples == 0 ) + { +#endif + if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) + { + ivas_splitBinLCLDDecProcess( + hSplitBin->hSplitBinLCLDDec, + &bits, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + SplitRendBFI ); +#ifndef API_5MS + isPostRendInputCldfb = 1; +#endif + +#ifdef API_5MS + /* copy data over to 5ms buffer */ + for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) + { + for ( slotIdx = 0; slotIdx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; ++slotIdx ) + { + mvr2r( Cldfb_RealBuffer_Binaural[chnlIdx][slotIdx], Cldfb_RealBuffer_Binaural_5ms[chnlIdx][slotIdx], CLDFB_NO_CHANNELS_MAX ); + mvr2r( Cldfb_ImagBuffer_Binaural[chnlIdx][slotIdx], Cldfb_ImagBuffer_Binaural_5ms[chnlIdx][slotIdx], CLDFB_NO_CHANNELS_MAX ); + } + } + /* cache the remaining 15ms */ + splitBinInput->numCachedSamples = numColPerChannelCacheSize; + writePtr = splitBinInput->bufferData; + for ( slotIdx = CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slotIdx < CLDFB_NO_COL_MAX; ++slotIdx ) + { + for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) + { + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + *writePtr++ = Cldfb_RealBuffer_Binaural[chnlIdx][slotIdx][smplIdx]; + } + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + *writePtr++ = Cldfb_ImagBuffer_Binaural[chnlIdx][slotIdx][smplIdx]; + } + } + } +#endif + } + else + { + error = splitBinLc3plusDecode( hSplitBin, &bits, tmpCrendBuffer, bits.pose_correction ); + if ( error != IVAS_ERR_OK ) + { + return error; + } +#ifdef API_5MS + /* cache the remaining 15ms */ + splitBinInput->numCachedSamples = numSamplesPerChannelCacheSize; + mvr2r( &tmpCrendBuffer[0][outBufNumSamplesPerChannel], + splitBinInput->bufferData, + numSamplesPerChannelCacheSize ); + mvr2r( &tmpCrendBuffer[1][outBufNumSamplesPerChannel], + splitBinInput->bufferData + numSamplesPerChannelCacheSize, + numSamplesPerChannelCacheSize ); +#endif + } +#ifdef API_5MS + } else { - if ( ( error = splitBinLc3plusDecode( hSplitBin, &bits, tmpCrendBuffer, bits.pose_correction ) ) != IVAS_ERR_OK ) + /* copy from cache */ + if ( bits.codec == IVAS_SPLIT_REND_CODEC_LCLD ) { - return error; + int16_t readOffset = ( numColPerChannelCacheSize - splitBinInput->numCachedSamples ); + readPtr = splitBinInput->bufferData; + isPostRendInputCldfb = 1; + + readPtr += 2 * readOffset * CLDFB_NO_CHANNELS_MAX * BINAURAL_CHANNELS; + for ( slotIdx = 0; slotIdx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; ++slotIdx ) + { + for ( chnlIdx = 0; chnlIdx < BINAURAL_CHANNELS; ++chnlIdx ) + { + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + Cldfb_RealBuffer_Binaural_5ms[chnlIdx][slotIdx][smplIdx] = *readPtr++; + } + for ( smplIdx = 0; smplIdx < CLDFB_NO_CHANNELS_MAX; ++smplIdx ) + { + Cldfb_ImagBuffer_Binaural_5ms[chnlIdx][slotIdx][smplIdx] = *readPtr++; + } + } + } + + splitBinInput->numCachedSamples -= outBufNumColPerChannel; + } + else + { + int16_t readOffset = numSamplesPerChannelCacheSize - splitBinInput->numCachedSamples; + mvr2r( splitBinInput->bufferData + readOffset, + tmpCrendBuffer[0], + outBufNumSamplesPerChannel ); + mvr2r( splitBinInput->bufferData + readOffset + numSamplesPerChannelCacheSize, + tmpCrendBuffer[1], + outBufNumSamplesPerChannel ); + splitBinInput->numCachedSamples -= outBufNumSamplesPerChannel; } } +#endif } else { @@ -7233,25 +7939,54 @@ static ivas_error renderSplitBinauralWithPostRot( for ( ch_idx = 0; ch_idx < BINAURAL_CHANNELS; ch_idx++ ) { +#ifdef API_5MS + float *RealBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + float *ImagBuffer[CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES]; + + for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX / MAX_PARAM_SPATIAL_SUBFRAMES; slot_idx++ ) +#else float *RealBuffer[CLDFB_NO_COL_MAX]; float *ImagBuffer[CLDFB_NO_COL_MAX]; for ( slot_idx = 0; slot_idx < CLDFB_NO_COL_MAX; slot_idx++ ) +#endif { +#ifdef API_5MS + RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural_5ms[ch_idx][slot_idx]; + ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural_5ms[ch_idx][slot_idx]; +#else RealBuffer[slot_idx] = Cldfb_RealBuffer_Binaural[ch_idx][slot_idx]; ImagBuffer[slot_idx] = Cldfb_ImagBuffer_Binaural[ch_idx][slot_idx]; +#endif } cldfbSynthesis( RealBuffer, ImagBuffer, &( tmpCrendBuffer[ch_idx][0] ), - hSplitBin->hBinHrSplitPostRend->cldfbSyn[0]->no_channels * CLDFB_NO_COL_MAX, + hSplitBin->hBinHrSplitPostRend->cldfbSyn[0]->no_channels * CLDFB_NO_COL_MAX +#ifdef API_5MS + / MAX_PARAM_SPATIAL_SUBFRAMES +#endif + , hSplitBin->hBinHrSplitPostRend->cldfbSyn[ch_idx] ); } } else if ( bits.pose_correction == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { - ivas_rend_CldfbSplitPostRendProcess( hSplitBin->hBinHrSplitPostRend, &hSplitBin->multiBinPoseData, QuaternionsPost, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, tmpCrendBuffer, isPostRendInputCldfb ); + ivas_rend_CldfbSplitPostRendProcess( + hSplitBin->hBinHrSplitPostRend, + &hSplitBin->multiBinPoseData, +#ifdef API_5MS + QuaternionPost, + Cldfb_RealBuffer_Binaural_5ms, + Cldfb_ImagBuffer_Binaural_5ms, +#else + QuaternionsPost, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, +#endif + tmpCrendBuffer, + isPostRendInputCldfb ); } } else @@ -7270,7 +8005,11 @@ static ivas_error renderSplitBinauralWithPostRot( } } +#ifdef API_5MS + convertInternalBitsBuffToBitsBuffer( splitBinInput->hBits, bits ); +#else convertInternalBitsBuffToBitsBuffer( hBits, bits ); +#endif accumulate2dArrayToBuffer( tmpCrendBuffer, &outAudio ); pop_wmops(); @@ -7287,7 +8026,10 @@ static ivas_error renderSbaToMultiBinaural( float tmpCrendBuffer[MAX_OUTPUT_CHANNELS][L_FRAME48k]; float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; - int16_t sf, i, j, pos_idx; +#ifndef API_5MS + int16_t sf; +#endif + int16_t i, j, pos_idx; COMBINED_ORIENTATION_DATA combinedOrientationDataLocal; COMBINED_ORIENTATION_HANDLE pCombinedOrientationDataLocal; ivas_error error; @@ -7305,6 +8047,16 @@ static ivas_error renderSbaToMultiBinaural( combinedOrientationDataLocal = *pCombinedOrientationDataLocal; if ( pMultiBinPoseData->poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { +#ifdef API_5MS + combinedOrientationDataLocal.Quaternion = combinedOrientationDataLocal.Quaternion; + for ( i = 0; i < 3; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + combinedOrientationDataLocal.Rmat[i][j] = combinedOrientationDataLocal.Rmat[i][j]; + } + } +#else for ( sf = 1; sf < RENDERER_HEAD_POSITIONS_PER_FRAME; sf++ ) { combinedOrientationDataLocal.Quaternions[sf] = combinedOrientationDataLocal.Quaternions[0]; @@ -7316,6 +8068,7 @@ static ivas_error renderSbaToMultiBinaural( } } } +#endif } tmpRotBuffer = sbaInput->base.inputBuffer; @@ -7323,6 +8076,18 @@ static ivas_error renderSbaToMultiBinaural( for ( pos_idx = 0; pos_idx < pMultiBinPoseData->num_poses; pos_idx++ ) { +#ifdef API_5MS + IVAS_QUATERNION Quaternion_orig, Quaternion_abs; + Quaternion_orig = combinedOrientationDataLocal.Quaternion; + Quaternion_abs.w = -3.0f; + Quat2EulerDegree( combinedOrientationDataLocal.Quaternion, &Quaternion_abs.z, &Quaternion_abs.y, &Quaternion_abs.x ); /*order in Quat2Euler seems to be reversed ?*/ + + Quaternion_abs.x += pMultiBinPoseData->relative_head_poses[pos_idx][0]; + Quaternion_abs.y += pMultiBinPoseData->relative_head_poses[pos_idx][1]; + Quaternion_abs.z += pMultiBinPoseData->relative_head_poses[pos_idx][2]; + combinedOrientationDataLocal.Quaternion = Quaternion_abs; + QuatToRotMat( combinedOrientationDataLocal.Quaternion, combinedOrientationDataLocal.Rmat ); +#else IVAS_QUATERNION Quaternions_orig[RENDERER_HEAD_POSITIONS_PER_FRAME], Quaternions_abs; for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) @@ -7337,6 +8102,7 @@ static ivas_error renderSbaToMultiBinaural( combinedOrientationDataLocal.Quaternions[i] = Quaternions_abs; QuatToRotMat( combinedOrientationDataLocal.Quaternions[i], combinedOrientationDataLocal.Rmat[i] ); } +#endif /* copy input for in-place rotation */ @@ -7354,15 +8120,26 @@ static ivas_error renderSbaToMultiBinaural( assert( sbaInput->crendWrapper->hCrend[0]->hReverb == NULL ); /* call CREND */ - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate, pos_idx ) ) != IVAS_ERR_OK ) + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, + NULL, NULL, NULL, NULL, + p_tmpCrendBuffer, + *sbaInput->base.ctx.pOutSampleRate, +#ifdef API_5MS + getNumSubframesInBuffer( &sbaInput->base.inputBuffer, *sbaInput->base.ctx.pOutSampleRate ), +#endif + pos_idx ) ) != IVAS_ERR_OK ) { return error; } +#ifdef API_5MS + combinedOrientationDataLocal.Quaternion = Quaternion_orig; +#else for ( i = 0; i < RENDERER_HEAD_POSITIONS_PER_FRAME; i++ ) { combinedOrientationDataLocal.Quaternions[i] = Quaternions_orig[i]; } +#endif /* move to output */ for ( i = 0; i < BINAURAL_CHANNELS; i++ ) @@ -7378,21 +8155,36 @@ static ivas_error renderSbaToMultiBinaural( return IVAS_ERR_OK; } - static void renderSbaToMultiBinauralCldfb( input_sba *sbaInput, float Cldfb_Out_Real[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], float Cldfb_Out_Imag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - const int16_t low_res_pre_rend_rot ) + const int16_t low_res_pre_rend_rot +#ifdef API_5MS + , + int16_t num_subframes +#endif +) { float Cldfb_RealBuffer[MAX_OUTPUT_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer[MAX_OUTPUT_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; copyBufferToCLDFBarray( sbaInput->base.inputBuffer, Cldfb_RealBuffer, Cldfb_ImagBuffer ); - ivas_rend_CldfbMultiBinRendProcess( sbaInput->cldfbRendWrapper.hCldfbRend, sbaInput->base.ctx.pCombinedOrientationData, &sbaInput->base.ctx.pSplitRendWrapper->multiBinPoseData, Cldfb_RealBuffer, Cldfb_ImagBuffer, Cldfb_Out_Real, Cldfb_Out_Imag, low_res_pre_rend_rot ); - - return; + ivas_rend_CldfbMultiBinRendProcess( + sbaInput->cldfbRendWrapper.hCldfbRend, + sbaInput->base.ctx.pCombinedOrientationData, + &sbaInput->base.ctx.pSplitRendWrapper->multiBinPoseData, + Cldfb_RealBuffer, + Cldfb_ImagBuffer, + Cldfb_Out_Real, + Cldfb_Out_Imag, + low_res_pre_rend_rot +#ifdef API_5MS + , + num_subframes +#endif + ); } static ivas_error renderSbaToSplitBinaural( @@ -7413,8 +8205,16 @@ static ivas_error renderSbaToSplitBinaural( if ( sbaInput->base.ctx.hhRendererConfig[0]->split_rend_config.rendererSelection == IVAS_SPLIT_REND_RENDERER_SELECTION_FASTCONV ) { - renderSbaToMultiBinauralCldfb( sbaInput, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, low_res_pre_rend_rot ); - + renderSbaToMultiBinauralCldfb( sbaInput, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + low_res_pre_rend_rot +#ifdef API_5MS + , + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) + // MAX_PARAM_SPATIAL_SUBFRAMES +#endif + ); accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); } else @@ -7446,7 +8246,9 @@ static ivas_error renderSbaToBinaural( int16_t i; const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; int8_t combinedOrientationEnabled; +#ifndef API_5MS int16_t subframe_idx; +#endif push_wmops( "renderSbaToBinaural" ); @@ -7455,9 +8257,16 @@ static ivas_error renderSbaToBinaural( { float Cldfb_RealBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; float Cldfb_ImagBuffer_Binaural[BINAURAL_CHANNELS][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX]; - - renderSbaToMultiBinauralCldfb( sbaInput, Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, 0 ); - + renderSbaToMultiBinauralCldfb( sbaInput, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + 0 +#ifdef API_5MS + , + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) + // 1 +#endif + ); accumulateCLDFBArrayToBuffer( Cldfb_RealBuffer_Binaural, Cldfb_ImagBuffer_Binaural, &outAudio ); } else @@ -7468,6 +8277,11 @@ static ivas_error renderSbaToBinaural( p_tmpCrendBuffer[i] = tmpCrendBuffer[i]; } +#ifdef API_5MS + /* fix compiling only, ToDo ext renderer needs to adapted to 5ms */ + hCombinedOrientationData = sbaInput->base.ctx.pCombinedOrientationData; + combinedOrientationEnabled = (int8_t) ( ( *hCombinedOrientationData )->enableCombinedOrientation ); +#else hCombinedOrientationData = sbaInput->base.ctx.pCombinedOrientationData; combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) @@ -7481,6 +8295,7 @@ static ivas_error renderSbaToBinaural( } } } +#endif /* apply rotation */ if ( combinedOrientationEnabled ) @@ -7513,9 +8328,14 @@ static ivas_error renderSbaToBinaural( } /* call CREND */ - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate -#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, sbaInput->base.inConfig, outConfig, + NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate +#ifdef API_5MS , + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT + , 0 #endif ) ) != IVAS_ERR_OK ) @@ -7546,7 +8366,9 @@ static ivas_error renderSbaToBinauralRoom( float *p_tmpCrendBuffer[MAX_OUTPUT_CHANNELS]; const COMBINED_ORIENTATION_HANDLE *hCombinedOrientationData; int8_t combinedOrientationEnabled; +#ifndef API_5MS int16_t subframe_idx; +#endif tmpRotBuffer = outAudio; /* avoid compilation warning */ @@ -7561,6 +8383,13 @@ static ivas_error renderSbaToBinauralRoom( combinedOrientationEnabled = 0; if ( hCombinedOrientationData != NULL ) { +#ifdef API_5MS + /* fix compiling only, ToDo ext renderer needs to adapted to 5ms */ + if ( ( *hCombinedOrientationData )->enableCombinedOrientation != 0 ) + { + combinedOrientationEnabled = 1; + } +#else for ( subframe_idx = 0; subframe_idx < MAX_PARAM_SPATIAL_SUBFRAMES; subframe_idx++ ) { if ( ( *hCombinedOrientationData )->enableCombinedOrientation[subframe_idx] != 0 ) @@ -7569,6 +8398,7 @@ static ivas_error renderSbaToBinauralRoom( break; } } +#endif } /* apply rotation */ @@ -7614,9 +8444,14 @@ static ivas_error renderSbaToBinauralRoom( copyBufferTo2dArray( tmpMcBuffer, tmpCrendBuffer ); /* call CREND */ - if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, AUDIO_CONFIG_7_1_4, outConfig, NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate -#ifdef SPLIT_REND_WITH_HEAD_ROT + if ( ( error = ivas_rend_crendProcess( sbaInput->crendWrapper, AUDIO_CONFIG_7_1_4, outConfig, + NULL, NULL, NULL, NULL, p_tmpCrendBuffer, *sbaInput->base.ctx.pOutSampleRate +#ifdef API_5MS , + getNumSubframesInBuffer( &outAudio, *sbaInput->base.ctx.pOutSampleRate ) +#endif +#ifdef SPLIT_REND_WITH_HEAD_ROT + , 0 #endif ) ) != IVAS_ERR_OK ) @@ -7642,18 +8477,22 @@ static ivas_error renderInputSplitBin( input_split_post_rend *splitBinInput, const AUDIO_CONFIG outConfig, IVAS_REND_AudioBuffer outAudio, +#ifndef API_5MS IVAS_REND_BitstreamBuffer *hBits, +#endif const int16_t SplitRendBFI ) { ivas_error error; IVAS_REND_AudioBuffer inAudio; inAudio = splitBinInput->base.inputBuffer; +#ifndef API_5MS if ( splitBinInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) { /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ return IVAS_ERR_INVALID_BUFFER_SIZE; } +#endif splitBinInput->base.numNewSamplesPerChannel = 0; @@ -7661,12 +8500,14 @@ static ivas_error renderInputSplitBin( v_multc( inAudio.data, splitBinInput->base.gain, inAudio.data, - inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); + inAudio.config.numSamplesPerChannel * inAudio.config.numChannels ); /* TODO: the output buffer is empty at this point, should be moved to a point after decoding the split bitstream */ switch ( outConfig ) { case AUDIO_CONFIG_BINAURAL: error = renderSplitBinauralWithPostRot( splitBinInput, outAudio, +#ifndef API_5MS hBits, +#endif SplitRendBFI ); break; default: @@ -7701,20 +8542,31 @@ static ivas_error renderInputSba( { ivas_error error; IVAS_REND_AudioBuffer inAudio; +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + int16_t cldfb2tdSampleFact; +#endif error = IVAS_ERR_OK; inAudio = sbaInput->base.inputBuffer; #ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( sbaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) && +#ifdef API_5MS + cldfb2tdSampleFact = outAudio.config.is_cldfb + ? 2 + : 1; +#endif + if ( ( sbaInput->base.numNewSamplesPerChannel +#ifdef API_5MS + * cldfb2tdSampleFact +#endif + != outAudio.config.numSamplesPerChannel ) && ( outConfig != AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) && ( outConfig != AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) #else if ( sbaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) #endif { - /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); } sbaInput->base.numNewSamplesPerChannel = 0; @@ -7770,8 +8622,12 @@ static ivas_error renderInputSba( #ifdef SPLIT_REND_WITH_HEAD_ROT static ivas_error renderActiveInputsSplitBin( IVAS_REND_HANDLE hIvasRend, - IVAS_REND_AudioBuffer outAudio, - IVAS_REND_BitstreamBuffer *hBits ) + IVAS_REND_AudioBuffer outAudio +#ifndef API_5MS + , + IVAS_REND_BitstreamBuffer *hBits +#endif +) { int16_t i; input_split_post_rend *pCurrentInput; @@ -7785,7 +8641,10 @@ static ivas_error renderActiveInputsSplitBin( continue; } - if ( ( error = renderInputSplitBin( pCurrentInput, hIvasRend->outputConfig, outAudio, hBits, + if ( ( error = renderInputSplitBin( pCurrentInput, hIvasRend->outputConfig, outAudio, +#ifndef API_5MS + hBits, +#endif hIvasRend->splitRendBFI ) ) != IVAS_ERR_OK ) { return error; @@ -7881,11 +8740,21 @@ static void renderMasaToMc( if ( masaInput->decDummy->renderer_type == RENDERER_STEREO_PARAMETRIC ) { - ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); + ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels +#ifdef API_5MS + , + getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) +#endif + ); } else { - ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels +#ifdef API_5MS + , + getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) +#endif + ); } accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); @@ -7908,7 +8777,12 @@ static void renderMasaToSba( copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); - ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); + ivas_dirac_dec( masaInput->decDummy, tmpBuffer, masaInput->base.inputBuffer.config.numChannels +#ifdef API_5MS + , + getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) +#endif + ); accumulate2dArrayToBuffer( tmpBuffer_buff, &outAudio ); @@ -7935,8 +8809,12 @@ static void renderMasaToBinaural( copyBufferTo2dArray( masaInput->base.inputBuffer, tmpBuffer_buff ); copyMasaMetadataToDiracRenderer( &masaInput->masaMetadata, masaInput->decDummy->hSpatParamRendCom ); - ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels ); - + ivas_dirac_dec_binaural( masaInput->decDummy, *masaInput->base.ctx.pCombinedOrientationData, tmpBuffer, masaInput->base.inputBuffer.config.numChannels +#ifdef API_5MS + , + getNumSubframesInBuffer( &outAudio, *masaInput->base.ctx.pOutSampleRate ) +#endif + ); #ifdef SPLIT_REND_WITH_HEAD_ROT if ( is_split_rend_mode ) { @@ -8087,6 +8965,9 @@ static ivas_error renderInputMasa( IVAS_REND_AudioBuffer outAudio ) { IVAS_REND_AudioBuffer inAudio; +#if defined( SPLIT_REND_WITH_HEAD_ROT ) && defined( API_5MS ) + int16_t cldfb2tdSampleFact; +#endif if ( !masaInput->metadataHasBeenFed ) { @@ -8095,14 +8976,22 @@ static ivas_error renderInputMasa( inAudio = masaInput->base.inputBuffer; #ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) && +#ifdef API_5MS + cldfb2tdSampleFact = outAudio.config.is_cldfb + ? 2 + : 1; +#endif + if ( ( masaInput->base.numNewSamplesPerChannel +#ifdef API_5MS + * cldfb2tdSampleFact +#endif + != outAudio.config.numSamplesPerChannel ) && ( outConfig != AUDIO_CONFIG_BINAURAL_SPLIT_CODED ) && ( outConfig != AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ) #else if ( masaInput->base.numNewSamplesPerChannel != outAudio.config.numSamplesPerChannel ) #endif { - /* Mismatch between the number of input samples vs number of requested output samples - currently not allowed */ - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Mismatch between the number of input samples vs number of requested output samples - currently not allowed" ); } masaInput->base.numNewSamplesPerChannel = 0; @@ -8173,7 +9062,9 @@ static ivas_error renderActiveInputsMasa( int16_t i; input_masa *pCurrentInput; ivas_error error; +#ifndef API_5MS int16_t sf_idx; +#endif for ( i = 0, pCurrentInput = hIvasRend->inputsMasa; i < RENDERER_MAX_MASA_INPUTS; ++i, ++pCurrentInput ) { @@ -8185,11 +9076,22 @@ static ivas_error renderActiveInputsMasa( if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && pCurrentInput->decDummy->hHeadTrackData != NULL ) { +#ifdef API_5MS + pCurrentInput->decDummy->hHeadTrackData->Quaternion = hIvasRend->headRotData.headPosition; + pCurrentInput->decDummy->hHeadTrackData->Pos = hIvasRend->headRotData.Pos; +#else for ( sf_idx = 0; sf_idx < RENDERER_HEAD_POSITIONS_PER_FRAME; ++sf_idx ) { +#ifdef API_5MS + /* fix compilation only, ToDo adapt ext renderer to 5ms */ + pCurrentInput->decDummy->hHeadTrackData->Quaternion = hIvasRend->headRotData.headPositions[sf_idx]; + pCurrentInput->decDummy->hHeadTrackData->Pos = hIvasRend->headRotData.Pos[sf_idx]; +#else pCurrentInput->decDummy->hHeadTrackData->Quaternions[sf_idx] = hIvasRend->headRotData.headPositions[sf_idx]; pCurrentInput->decDummy->hHeadTrackData->Pos[sf_idx] = hIvasRend->headRotData.Pos[sf_idx]; +#endif } +#endif } if ( ( error = renderInputMasa( pCurrentInput, hIvasRend->outputConfig, outAudio ) ) != IVAS_ERR_OK ) @@ -8373,6 +9275,15 @@ ivas_error IVAS_REND_SetIsmMetadataDelay( } +#ifdef API_5MS +/*-------------------------------------------------------------------* + * getSamplesInternal() + * + * + *-------------------------------------------------------------------*/ + +static ivas_error getSamplesInternal( +#else /*-------------------------------------------------------------------* * IVAS_REND_GetSamples() * @@ -8380,6 +9291,7 @@ ivas_error IVAS_REND_SetIsmMetadataDelay( *-------------------------------------------------------------------*/ ivas_error IVAS_REND_GetSamples( +#endif IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ #ifdef SPLIT_REND_WITH_HEAD_ROT @@ -8421,14 +9333,15 @@ ivas_error IVAS_REND_GetSamples( #ifdef SPLIT_REND_WITH_HEAD_ROT if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && + hIvasRend->outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_CODED && + hIvasRend->outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_PCM && ( outAudio.config.numSamplesPerChannel * 1000 / cldfb2tdSampleFact ) != BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->sampleRateOut ) #else if ( getAudioConfigType( hIvasRend->outputConfig ) == IVAS_REND_AUDIO_CONFIG_TYPE_BINAURAL && outAudio.config.numSamplesPerChannel * 1000 != BINAURAL_RENDERING_FRAME_SIZE_MS * hIvasRend->sampleRateOut ) #endif { - /* Binaural rendering requires specific frame size */ - return IVAS_ERR_INVALID_BUFFER_SIZE; + return IVAS_ERROR( IVAS_ERR_INVALID_BUFFER_SIZE, "Binaural rendering requires specific frame size" ); } /* Check that there is allowed configuration for MASA format output */ @@ -8467,7 +9380,11 @@ ivas_error IVAS_REND_GetSamples( return error; } - if ( numOutChannels != outAudio.config.numChannels ) + if ( numOutChannels != outAudio.config.numChannels +#ifdef SPLIT_REND_WITH_HEAD_ROT + && hIvasRend->outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_CODED && hIvasRend->outputConfig != AUDIO_CONFIG_BINAURAL_SPLIT_PCM +#endif + ) { return IVAS_ERR_WRONG_NUM_CHANNELS; } @@ -8484,19 +9401,21 @@ ivas_error IVAS_REND_GetSamples( int16_t num_poses_orig; num_poses_orig = hIvasRend->splitRendWrapper.multiBinPoseData.num_poses; outAudio = hIvasRend->splitRendEncBuffer; +#ifndef API_5MS if ( ( outAudioOrig.config.is_cldfb == 0 ) && ( hIvasRend->inputsMasa[0].base.inConfig == AUDIO_CONFIG_INVALID ) ) { outAudio.config.is_cldfb = 0; outAudio.config.numSamplesPerChannel >>= 1; } +#endif ivas_renderSplitGetMultiBinPoseData( &hIvasRend->hRendererConfig->split_rend_config, &hIvasRend->splitRendWrapper.multiBinPoseData, hIvasRend->headRotData.sr_pose_pred_axis ); assert( num_poses_orig == hIvasRend->splitRendWrapper.multiBinPoseData.num_poses && "number of poses should not change dynamically" ); - /* Clear output buffer */ + /* Clear output buffer for split rendering bitstream */ set_zero( outAudio.data, outAudio.config.numChannels * outAudio.config.numSamplesPerChannel ); } #endif @@ -8520,25 +9439,34 @@ ivas_error IVAS_REND_GetSamples( return error; } #ifdef SPLIT_REND_WITH_HEAD_ROT - if ( ( error = renderActiveInputsSplitBin( hIvasRend, outAudio, hBits ) ) != IVAS_ERR_OK ) + if ( ( error = renderActiveInputsSplitBin( hIvasRend, outAudio +#ifndef API_5MS + , + hBits +#endif + ) ) != IVAS_ERR_OK ) { return error; } if ( outAudio.config.is_cldfb == 0 ) { +#ifndef DISABLE_LIMITER #ifdef DEBUGGING hIvasRend->numClipping += #endif limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); +#endif } #else /* SPLIT_REND_WITH_HEAD_ROT */ +#ifndef DISABLE_LIMITER #ifdef DEBUGGING hIvasRend->numClipping += #endif limitRendererOutput( hIvasRend->hLimiter, outAudio.data, outAudio.config.numSamplesPerChannel, IVAS_LIMITER_THRESHOLD ); #endif +#endif /* SPLIT_REND_WITH_HEAD_ROT */ #ifdef SPLIT_REND_WITH_HEAD_ROT if ( hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_CODED || hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) @@ -8568,25 +9496,33 @@ ivas_error IVAS_REND_GetSamples( /* Encode split rendering bitstream */ convertBitsBufferToInternalBitsBuff( *hBits, &bits ); - - if ( ( error = ivas_renderMultiBinToSplitBinaural( &hIvasRend->splitRendWrapper, - hIvasRend->headRotData.headPositions, - hIvasRend->hRendererConfig->split_rend_config.splitRendBitRate, - hIvasRend->hRendererConfig->split_rend_config.codec, - &bits, - Cldfb_RealBuffer_Binaural, - Cldfb_ImagBuffer_Binaural, - ( const int16_t )( ( BINAURAL_MAXBANDS * hIvasRend->sampleRateOut ) / 48000 ), - tmpBinaural, - 1, - td_input, - ( hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ) ) != IVAS_ERR_OK ) + error = ivas_renderMultiBinToSplitBinaural( &hIvasRend->splitRendWrapper, +#ifdef API_5MS + hIvasRend->headRotData.headPosition, +#else + hIvasRend->headRotData.headPositions, +#endif + hIvasRend->hRendererConfig->split_rend_config.splitRendBitRate, + hIvasRend->hRendererConfig->split_rend_config.codec, +#ifdef API_5MS + hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms, +#endif + &bits, + Cldfb_RealBuffer_Binaural, + Cldfb_ImagBuffer_Binaural, + ( const int16_t )( ( BINAURAL_MAXBANDS * hIvasRend->sampleRateOut ) / 48000 ), + tmpBinaural, + 1, + td_input, + ( hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) ? 1 : 0 ); + if ( error != IVAS_ERR_OK ) { return error; } convertInternalBitsBuffToBitsBuffer( hBits, bits ); + /* reset to outAudioOrig in case of PCM output */ outAudio = outAudioOrig; if ( hIvasRend->outputConfig == AUDIO_CONFIG_BINAURAL_SPLIT_PCM ) @@ -8594,10 +9530,75 @@ ivas_error IVAS_REND_GetSamples( accumulate2dArrayToBuffer( tmpBinaural_buff, &outAudio ); } } +#endif /* SPLIT_REND_WITH_HEAD_ROT */ + + return IVAS_ERR_OK; +} + +#ifdef API_5MS +/*-------------------------------------------------------------------* + * IVAS_REND_GetSamples() + * + * + *-------------------------------------------------------------------*/ + +ivas_error IVAS_REND_GetSamples( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ +) +{ + return getSamplesInternal( hIvasRend, outAudio +#ifdef SPLIT_REND_WITH_HEAD_ROT + , + NULL #endif + ); +} + +#ifdef SPLIT_REND_WITH_HEAD_ROT +ivas_error IVAS_REND_GetSplitBinauralBitstream( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ + IVAS_REND_BitstreamBuffer *hBits /* o : buffer for output bitstream */ +) +{ + int16_t cldfb_in; + + cldfb_in = getCldfbRendFlag( hIvasRend, IVAS_REND_AUDIO_CONFIG_TYPE_UNKNOWN ); + hIvasRend->splitRendEncBuffer.config.is_cldfb = cldfb_in; + if ( hIvasRend->hRendererConfig->split_rend_config.codec == IVAS_SPLIT_REND_CODEC_LC3PLUS && + ( hIvasRend->hRendererConfig->split_rend_config.dof == 0 || hIvasRend->hRendererConfig->split_rend_config.poseCorrectionMode == IVAS_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) ) + { + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = hIvasRend->hRendererConfig->split_rend_config.codec_frame_size_ms * (int16_t) ( hIvasRend->sampleRateOut / 1000 ); + } + else + { + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel = (int16_t) ( hIvasRend->sampleRateOut / FRAMES_PER_SEC ); + } + hIvasRend->splitRendEncBuffer.config.numSamplesPerChannel *= cldfb_in ? 2 : 1; + + /* hIvasRend->splitRendEncBuffer used for BINAURAL_SPLIT_CODED output + outAudio used for BINAURAL_SPLIT_PCM output */ + return getSamplesInternal( hIvasRend, outAudio, hBits ); +} + +ivas_error IVAS_REND_GetSplitBinauralSamples( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ + bool *needNewFrame ) +{ + ivas_error error; + + if ( ( error = getSamplesInternal( hIvasRend, outAudio, NULL ) ) != IVAS_ERR_OK ) + { + return error; + } + *needNewFrame = hIvasRend->inputsSplitPost[0].numCachedSamples == 0; return IVAS_ERR_OK; } +#endif +#endif /*-------------------------------------------------------------------* diff --git a/lib_rend/lib_rend.h b/lib_rend/lib_rend.h index da1e76c79853bfff96ac0961909ae9574752fed0..a069248a3afd460f4458b4ced68bccfa2eaa69ee 100644 --- a/lib_rend/lib_rend.h +++ b/lib_rend/lib_rend.h @@ -91,6 +91,9 @@ typedef struct int32_t bitsRead; IVAS_SPLIT_REND_CODEC codec; IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrection; +#ifdef API_5MS + int16_t codec_frame_size_ms; +#endif } IVAS_REND_BitstreamBufferConfig; typedef struct { @@ -248,16 +251,46 @@ int16_t IVAS_REND_FeedRenderConfig( const IVAS_RENDER_CONFIG_DATA renderConfig /* i : Render configuration struct */ ); +#if defined API_5MS && defined SPLIT_REND_WITH_HEAD_ROT +ivas_error IVAS_REND_FeedSplitBinauralBitstream( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + const IVAS_REND_InputId inputId, /* i : ID of the input */ + IVAS_REND_BitstreamBuffer *hBits /* i : buffer for input bitstream */ +); +ivas_error IVAS_REND_GetSplitBinauralSamples( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ + bool* needNewFrame +); +ivas_error IVAS_REND_GetSplitBinauralBitstream( + IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ + IVAS_REND_AudioBuffer outAudio, /* i/o: buffer for output audio */ + IVAS_REND_BitstreamBuffer *hBits /* o : buffer for output bitstream */ +); +#endif + ivas_error IVAS_REND_SetHeadRotation( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ +#ifdef API_5MS + const IVAS_QUATERNION headRot, /* i : head orientations for next rendering call */ + const IVAS_VECTOR3 Pos /* i : listener positions for next rendering call */ +#else const IVAS_QUATERNION headRot[RENDERER_HEAD_POSITIONS_PER_FRAME], /* i : head orientations for next rendering call */ - const IVAS_VECTOR3 Pos[RENDERER_HEAD_POSITIONS_PER_FRAME] /* i : listener positions for next rendering call */ + const IVAS_VECTOR3 Pos[RENDERER_HEAD_POSITIONS_PER_FRAME] /* i : listener positions for next rendering call */ +#endif #ifdef SPLIT_REND_WITH_HEAD_ROT , IVAS_SPLIT_REND_ROT_AXIS rot_axis #endif ); +#ifdef API_5MS +/* Head rotation becomes enabled by calling IVAS_REND_SetHeadRotation. Use this to disable. */ +ivas_error IVAS_REND_DisableHeadRotation( + IVAS_REND_HANDLE hIvasRend /* i/o: Renderer handle */ +); +#endif + ivas_error IVAS_REND_SetOrientationTrackingMode( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ const HEAD_ORIENT_TRK_T orientation_tracking /* i : Head orientation tracking type */ @@ -293,10 +326,17 @@ ivas_error IVAS_REND_SetSplitRendBFI( ivas_error IVAS_REND_SetExternalOrientation( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ IVAS_QUATERNION *orientation, /* i : external orientation data */ +#ifdef API_5MS + int8_t enableHeadRotation, /* i : flag to enable head rotation for this frame */ + int8_t enableExternalOrientation, /* i : flag to enable external orientation for this frame */ + int8_t enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ + int16_t numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#else int8_t *enableHeadRotation, /* i : flag to enable head rotation for this frame */ int8_t *enableExternalOrientation, /* i : flag to enable external orientation for this frame */ int8_t *enableRotationInterpolation, /* i : flag to interpolate rotations from current and previous frames */ int16_t *numFramesToTargetOrientation /* i : number of frames until target orientation is reached */ +#endif ); ivas_error IVAS_REND_CombineHeadAndExternalOrientation( @@ -305,7 +345,7 @@ ivas_error IVAS_REND_CombineHeadAndExternalOrientation( ivas_error IVAS_REND_GetCombinedOrientation( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ - IVAS_QUATERNION *pRotation /* i/o: Quaternion pointer processed orientation */ + IVAS_QUATERNION *pOrientation /* i/o: Quaternion pointer processed orientation */ ); ivas_error IVAS_REND_GetMasaMetadata( @@ -339,10 +379,12 @@ ivas_error IVAS_REND_GetNumAllObjects( ivas_error IVAS_REND_GetSamples( IVAS_REND_HANDLE hIvasRend, /* i/o: Renderer handle */ IVAS_REND_AudioBuffer outAudio /* i/o: buffer for output audio */ +#ifndef API_5MS #ifdef SPLIT_REND_WITH_HEAD_ROT , IVAS_REND_BitstreamBuffer *hBits /*i/o: buffer for input/output bitstream. Needed in split rendering mode*/ #endif +#endif ); /* Functions to be called after rendering */ diff --git a/lib_util/render_config_reader.c b/lib_util/render_config_reader.c index 15dd45256ff94b50211c25ac790a7fa81634910f..b5c4abfbf89936ca3f06ef77c1f01096d8033a55 100644 --- a/lib_util/render_config_reader.c +++ b/lib_util/render_config_reader.c @@ -1333,6 +1333,9 @@ static ivas_error RenderConfigReader_readBinary( fread( pRenderConfigReader->pBitstream, sizeof( uint8_t ), file_size, pReverbConfigFile ); pRenderConfigReader->length = file_size; +#ifdef FIX_RENDER_CONFIG_BITSTREAM_READER + pRenderConfigReader->readOffset = 0; +#endif /****************************/ /* Read the presence flag */ @@ -2494,6 +2497,21 @@ ivas_error RenderConfigReader_read( errorHandler( pValue, ERROR_VALUE_INVALID ); } } +#ifdef API_5MS + else if ( strcmp( item, "FRAMESIZE" ) == 0 ) + { + if ( !sscanf( pValue, "%hd", &hRenderConfig->split_rend_config.codec_frame_size_ms ) ) + { + errorHandler( item, ERROR_VALUE_INVALID ); + } + if ( hRenderConfig->split_rend_config.codec_frame_size_ms != 5 && + hRenderConfig->split_rend_config.codec_frame_size_ms != 10 && + hRenderConfig->split_rend_config.codec_frame_size_ms != 20 ) + { + errorHandler( item, ERROR_VALUE_INVALID ); + } + } +#endif else if ( strcmp( item, "POSECORRECTION" ) == 0 ) { poseCorrProvided = true; diff --git a/lib_util/split_render_file_read_write.c b/lib_util/split_render_file_read_write.c index 215d3d32c6bcc1cfd0c52f8bbc079a5c7c244f60..ea9e6da739d511484448ea2fe49b54a9dc5ed0dd 100644 --- a/lib_util/split_render_file_read_write.c +++ b/lib_util/split_render_file_read_write.c @@ -198,7 +198,12 @@ ivas_error split_rend_write_bitstream_to_file( int32_t *bits_read, int32_t *bits_written, IVAS_SPLIT_REND_CODEC codec, - IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrection ) + IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrection +#ifdef API_5MS + , + int16_t codec_frame_size_ms +#endif +) { char header[SPLIT_RENDERER_FRAME_HEADER_LEN] = "SPLIT_FRAME"; size_t header_len, i, num_bytes; @@ -242,6 +247,13 @@ ivas_error split_rend_write_bitstream_to_file( { return IVAS_ERR_FAILED_FILE_WRITE; } +#ifdef API_5MS + /* Write frame size signalling */ + if ( fwrite( &codec_frame_size_ms, sizeof( codec_frame_size_ms ), 1, hSplitRendFileReadWrite->file ) != 1 ) + { + return IVAS_ERR_FAILED_FILE_WRITE; + } +#endif /* write num bytes */ if ( fwrite( bits_written, sizeof( int32_t ), 1, hSplitRendFileReadWrite->file ) != 1 ) @@ -274,7 +286,12 @@ ivas_error split_rend_read_bits_from_file( int32_t *bits_read, int32_t *bits_written, IVAS_SPLIT_REND_CODEC *codec, - IVAS_SPLIT_REND_POSE_CORRECTION_MODE *poseCorrection ) + IVAS_SPLIT_REND_POSE_CORRECTION_MODE *poseCorrection +#ifdef API_5MS + , + int16_t *codec_frame_size_ms +#endif +) { char header[SPLIT_RENDERER_FRAME_HEADER_LEN] = "SPLIT_FRAME"; char header_read[SPLIT_RENDERER_FRAME_HEADER_LEN]; @@ -330,6 +347,13 @@ ivas_error split_rend_read_bits_from_file( { return IVAS_ERR_FAILED_FILE_READ; } +#ifdef API_5MS + /* read frame size signalling */ + if ( fread( codec_frame_size_ms, sizeof( *codec_frame_size_ms ), 1, hSplitRendFileReadWrite->file ) != 1 ) + { + return IVAS_ERR_FAILED_FILE_READ; + } +#endif /* write num bytes */ if ( fread( &bit_len, sizeof( int32_t ), 1, hSplitRendFileReadWrite->file ) != 1 ) diff --git a/lib_util/split_render_file_read_write.h b/lib_util/split_render_file_read_write.h index 3f523eb87d44181cfde087cf49b7f8869b9179ec..dacc2f72bc852e229797c3c80b76f61db74c5929 100644 --- a/lib_util/split_render_file_read_write.h +++ b/lib_util/split_render_file_read_write.h @@ -61,7 +61,12 @@ ivas_error split_rend_write_bitstream_to_file( int32_t *bits_read, int32_t *bits_written, IVAS_SPLIT_REND_CODEC codec, - IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrection ); + IVAS_SPLIT_REND_POSE_CORRECTION_MODE poseCorrection +#ifdef API_5MS + , + int16_t codec_frame_size_ms +#endif +); /* read split rend coded bits from file */ ivas_error split_rend_read_bits_from_file( @@ -70,7 +75,12 @@ ivas_error split_rend_read_bits_from_file( int32_t *bits_read, int32_t *bits_written, IVAS_SPLIT_REND_CODEC *codec, - IVAS_SPLIT_REND_POSE_CORRECTION_MODE *poseCorrection ); + IVAS_SPLIT_REND_POSE_CORRECTION_MODE *poseCorrection +#ifdef API_5MS + , + int16_t *codec_frame_size_ms +#endif +); /* read split pre rend delay */ ivas_error split_rend_read_pre_rend_delay_ns( diff --git a/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c b/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c index 14ef6ded593e682d35ad60ca96dc2a4534a6b277..fe9c9118a77c0effa26f426c885ba19b5b0f781c 100644 --- a/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c +++ b/scripts/binauralRenderer_interface/Table_Format_Converter/generate_tables_from_rom_to_bin.c @@ -749,7 +749,6 @@ char *create_hrtf_tdrend( int32_t frequency, int32_t *hrtf_size ) // Left/right and coherence late reverb tables data_size_tmp = sizeof( float ) * LR_IAC_LENGTH_NR_FC; -#ifdef TD_REND_TUNING_RENAME if ( frequency == 48000 ) { memcpy( td_hrtf_wptr, &( defaultHRIR_left_avg_power_48kHz ), data_size_tmp ); @@ -777,35 +776,6 @@ char *create_hrtf_tdrend( int32_t frequency, int32_t *hrtf_size ) memcpy( td_hrtf_wptr, &( defaultHRIR_coherence_16kHz ), data_size_tmp ); td_hrtf_wptr += data_size_tmp; } -#else - if ( frequency == 48000 ) - { - memcpy( td_hrtf_wptr, &( orange53_left_avg_power_48kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - memcpy( td_hrtf_wptr, &( orange53_right_avg_power_48kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - memcpy( td_hrtf_wptr, &( orange53_coherence_48kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - } - else if ( frequency == 32000 ) - { - memcpy( td_hrtf_wptr, &( orange53_left_avg_power_32kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - memcpy( td_hrtf_wptr, &( orange53_right_avg_power_32kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - memcpy( td_hrtf_wptr, &( orange53_coherence_32kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - } - else if ( frequency == 16000 ) - { - memcpy( td_hrtf_wptr, &( orange53_left_avg_power_16kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - memcpy( td_hrtf_wptr, &( orange53_right_avg_power_16kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - memcpy( td_hrtf_wptr, &( orange53_coherence_16kHz ), data_size_tmp ); - td_hrtf_wptr += data_size_tmp; - } -#endif fclose( input_td_bin_file ); input_td_bin_file = NULL; diff --git a/scripts/self_test.py b/scripts/self_test.py index d7118832db57d14fad8db5451230c3ab79ae5de2..321f83419bfc9b370755b3f0063342d880db40b0 100755 --- a/scripts/self_test.py +++ b/scripts/self_test.py @@ -186,6 +186,13 @@ class SelfTest(IvasScriptsCommon.IvasScript): "--dectest", help="Test decoder binary (default:{})".format(default_dec_test), ) + self.parser.add_argument( + "--list_conditions", + "-l", + help="List all comditions in the parameter file", + action="store_true", + default=False, + ) if shutil.which("valgrind"): self.valgrind = [ "valgrind", @@ -858,6 +865,7 @@ class SelfTest(IvasScriptsCommon.IvasScript): # add the detailed results for this condition self.fail_results["detailedResults"].append(result_str) + self.fail_results["failed_modes"].append(mode) self.logger.progress( "Comparing conditions: {}/{} ({} running), {} failed ".format( self.stat["num_tests_done"], @@ -1140,6 +1148,11 @@ class SelfTest(IvasScriptsCommon.IvasScript): self.parse_args() + if self.args["list_conditions"] is True: + run_dict = self.parse_self_test_prm(self.args["test_prm"]) + for mode in run_dict.keys(): + self.logger.console(f"- {run_dict[mode]['cmd']['table_name']}") + return # create/update test vectors (export from SVN server if not existing) svn_action = None if not os.path.exists(TESTV_DIR): @@ -1540,6 +1553,7 @@ class SelfTest(IvasScriptsCommon.IvasScript): "corrupt_test_conditions": {"cnt": 0, "conditions": []}, "diff_nsamples_conditions": {"cnt": 0, "conditions": []}, "detailedResults": [], + "failed_modes":[], } self.run_pesq = self.args["pesq"] failed_ref_conditions = {} @@ -1948,6 +1962,15 @@ class SelfTest(IvasScriptsCommon.IvasScript): for l in r: self.logger.info(l) + self.logger.info("\n\n") + self.logger.info("Summary of all tests") + self.logger.info("---------------------\n") + for mode in run_dict.keys(): + if mode in self.fail_results["failed_modes"]: + self.logger.info(f"[FAIL] {run_dict[mode]['cmd']['table_name']}") + else: + self.logger.info(f"[OK] {run_dict[mode]['cmd']['table_name']}") + if __name__ == "__main__": if sys.version_info[0] < 3 or sys.version_info[1] < 7: diff --git a/scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test.c b/scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test.c index b8a5563c130589f42616a4e498cb21eea2d00267..4b49bc4315cc24b2b2810123390937a934e35aa8 100644 --- a/scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test.c +++ b/scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test.c @@ -94,7 +94,7 @@ static int encodeAndDecodeOneStereoFrame( LC3PLUS_CONFIG config ) #ifdef LC3PLUS_DEC_ALLOW_DISABLE_CACHING 1 /*caching enabled*/, #endif - &decHandle ); + &decHandle ); if ( IVAS_ERR_OK != err ) { return err; @@ -334,8 +334,6 @@ static int openCloseDecoderWithoutCaching( void ) } - - static int tryOpenDecoderWithInvalidFrameDuration( void ) { ivas_error err; @@ -624,4 +622,9 @@ int main( #endif return ret; } +#else +int main( void ) +{ + return EXIT_SUCCESS; +} #endif /* SPLIT_REND_WITH_HEAD_ROT */ diff --git a/tests/renderer/compare_audio.py b/tests/renderer/compare_audio.py index e7ba53963f38661b0f03bdecd881778263d7b559..516d52a98c419591a1924d2db4d7c7fc5a6cbffb 100644 --- a/tests/renderer/compare_audio.py +++ b/tests/renderer/compare_audio.py @@ -45,7 +45,6 @@ from pyaudio3dtools.audioarray import getdelay def compare_audio_arrays( left: np.ndarray, left_fs: int, right: np.ndarray, right_fs: int ) -> Tuple[float, float]: - if left_fs != right_fs: return ValueError(f"Differing samplerates: {left_fs} vs {right_fs}!") diff --git a/tests/renderer/constants.py b/tests/renderer/constants.py index 499c432973d1402d1f97c82340cd029a098b3166..1a5275c7d4bf20b604ddc983d0574508d6600977 100644 --- a/tests/renderer/constants.py +++ b/tests/renderer/constants.py @@ -201,171 +201,8 @@ METADATA_SCENES_TO_TEST_NO_BE = ["masa_scene"] OUTPUT_FORMATS_BINAURAL = ["BINAURAL", "BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"] HR_TRAJECTORIES_TO_TEST = [ "full_circle_in_15s", - "rotate_yaw_pitch_roll1", + # "rotate_yaw_pitch_roll1", ] -""" Per-testcase xfail SNR thresholds (dB) """ -pass_snr = dict() # not relevant for tests anymore, should be deprecated soon -_pass_snr = { - #################################################################### - # - # External Renderer vs Standalone and pyaudio3dtools renderers tests - # - #################################################################### - # Failure reason: Renderer uses getRSH() with int16_t vs float in python - "test_ambisonics[FOA-5_1]": 39, - "test_ambisonics[FOA-5_1_2]": 40, - "test_ambisonics[FOA-5_1_4]": 41, - "test_ambisonics[FOA-7_1]": 39, - "test_ambisonics[FOA-7_1_4]": 41, - "test_ambisonics[HOA2-5_1]": 26, - "test_ambisonics[HOA2-5_1_2]": 29, - "test_ambisonics[HOA2-5_1_4]": 31, - "test_ambisonics[HOA2-7_1]": 27, - "test_ambisonics[HOA2-7_1_4]": 32, - "test_ambisonics[HOA3-5_1]": 25, - "test_ambisonics[HOA3-5_1_2]": 27, - "test_ambisonics[HOA3-5_1_4]": 29, - "test_ambisonics[HOA3-7_1]": 25, - "test_ambisonics[HOA3-7_1_4]": 30, - # TODO needs debugging - "test_ambisonics_binaural_headrotation[HOA2-BINAURAL-full_circle_in_15s]": 18, - "test_ambisonics_binaural_headrotation[HOA3-BINAURAL-full_circle_in_15s]": 15, - # Failure reason: Crend unit test does not support intermediate conversion to 7_1_4 or SHD BRIRs - # Comparison with pyaudio3dtools results in bad SNR - "test_ambisonics_binaural_headrotation[FOA-BINAURAL_ROOM_IR-full_circle_in_15s]": 0, - "test_ambisonics_binaural_headrotation[FOA-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 0, - "test_ambisonics_binaural_headrotation[HOA2-BINAURAL_ROOM_IR-full_circle_in_15s]": 0, - "test_ambisonics_binaural_headrotation[HOA2-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 0, - "test_ambisonics_binaural_headrotation[HOA2-BINAURAL-rotate_yaw_pitch_roll1]": 4, - "test_ambisonics_binaural_headrotation[HOA3-BINAURAL_ROOM_IR-full_circle_in_15s]": 0, - "test_ambisonics_binaural_headrotation[HOA3-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 0, - "test_ambisonics_binaural_headrotation[HOA3-BINAURAL-rotate_yaw_pitch_roll1]": 3, - "test_ambisonics_binaural_static[FOA-BINAURAL_ROOM_IR]": 0, - "test_ambisonics_binaural_static[HOA2-BINAURAL_ROOM_IR]": 0, - "test_ambisonics_binaural_static[HOA3-BINAURAL_ROOM_IR]": 0, - # Failure reason: Renderer uses getRSH() with int16_t vs float in python - "test_custom_ls_input[t_design_4-FOA]": 43, - "test_custom_ls_input[t_design_4-HOA2]": 39, - "test_custom_ls_input[t_design_4-HOA3]": 36, - "test_custom_ls_output[FOA-16ch_8+4+4]": 40, - "test_custom_ls_output[FOA-4d4]": 40, - "test_custom_ls_output[FOA-itu_4+5+1]": 41, - "test_custom_ls_output[FOA-t_design_4]": 40, - "test_custom_ls_output[HOA2-16ch_8+4+4]": 32, - "test_custom_ls_output[HOA2-4d4]": 31, - "test_custom_ls_output[HOA2-itu_4+5+1]": 31, - "test_custom_ls_output[HOA2-t_design_4]": 34, - "test_custom_ls_output[HOA3-16ch_8+4+4]": 30, - "test_custom_ls_output[HOA3-4d4]": 29, - "test_custom_ls_output[HOA3-itu_4+5+1]": 30, - "test_custom_ls_output[HOA3-t_design_4]": 32, - # Failure reason: TD Object Renderer standalone does not support custom LS input - # Comparison with pyaudio3dtools results in bad SNR - "test_custom_ls_input_binaural[16ch_8+4+4-BINAURAL]": 8, - "test_custom_ls_input_binaural[16ch_8+4+4-BINAURAL_ROOM_IR]": 0, - "test_custom_ls_input_binaural[4d4-BINAURAL]": 6, - "test_custom_ls_input_binaural[4d4-BINAURAL_ROOM_IR]": 0, - "test_custom_ls_input_binaural[itu_4+5+1-BINAURAL]": 1, - "test_custom_ls_input_binaural[itu_4+5+1-BINAURAL_ROOM_IR]": 3, - "test_custom_ls_input_binaural[t_design_4-BINAURAL]": 5, - "test_custom_ls_input_binaural[t_design_4-BINAURAL_ROOM_IR]": 0, - "test_custom_ls_input_binaural_headrotation[16ch_8+4+4-BINAURAL-full_circle_in_15s]": 7, - "test_custom_ls_input_binaural_headrotation[16ch_8+4+4-BINAURAL-rotate_yaw_pitch_roll1]": 6, - "test_custom_ls_input_binaural_headrotation[16ch_8+4+4-BINAURAL_ROOM_IR-full_circle_in_15s]": 0, - "test_custom_ls_input_binaural_headrotation[16ch_8+4+4-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 0, - "test_custom_ls_input_binaural_headrotation[4d4-BINAURAL-full_circle_in_15s]": 7, - "test_custom_ls_input_binaural_headrotation[4d4-BINAURAL-rotate_yaw_pitch_roll1]": 5, - "test_custom_ls_input_binaural_headrotation[4d4-BINAURAL_ROOM_IR-full_circle_in_15s]": 0, - "test_custom_ls_input_binaural_headrotation[4d4-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 0, - "test_custom_ls_input_binaural_headrotation[itu_4+5+1-BINAURAL-full_circle_in_15s]": 1, - "test_custom_ls_input_binaural_headrotation[itu_4+5+1-BINAURAL-rotate_yaw_pitch_roll1]": 1, - "test_custom_ls_input_binaural_headrotation[itu_4+5+1-BINAURAL_ROOM_IR-full_circle_in_15s]": 3, - "test_custom_ls_input_binaural_headrotation[itu_4+5+1-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 3, - "test_custom_ls_input_binaural_headrotation[t_design_4-BINAURAL-full_circle_in_15s]": 4, - "test_custom_ls_input_binaural_headrotation[t_design_4-BINAURAL-rotate_yaw_pitch_roll1]": 4, - "test_custom_ls_input_binaural_headrotation[t_design_4-BINAURAL_ROOM_IR-full_circle_in_15s]": 0, - "test_custom_ls_input_binaural_headrotation[t_design_4-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 0, - # TODO needs debugging - "test_ism_binaural_headrotation[ISM2-BINAURAL-rotate_yaw_pitch_roll1]": 34, - "test_ism_binaural_headrotation[ISM3-BINAURAL-rotate_yaw_pitch_roll1]": 34, - "test_ism_binaural_headrotation[ISM4-BINAURAL-rotate_yaw_pitch_roll1]": 33, - # Failure reason: Crend unit test does not support intermediate conversion to 7_1_4 - "test_ism_binaural_headrotation[ISM1-BINAURAL_ROOM_IR-full_circle_in_15s]": 10, - "test_ism_binaural_headrotation[ISM1-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 4, - "test_ism_binaural_headrotation[ISM2-BINAURAL_ROOM_IR-full_circle_in_15s]": 10, - "test_ism_binaural_headrotation[ISM2-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 3, - "test_ism_binaural_headrotation[ISM3-BINAURAL_ROOM_IR-full_circle_in_15s]": 10, - "test_ism_binaural_headrotation[ISM3-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 4, - "test_ism_binaural_headrotation[ISM4-BINAURAL_ROOM_IR-full_circle_in_15s]": 10, - "test_ism_binaural_headrotation[ISM4-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 4, - "test_ism_binaural_static[ISM1-BINAURAL_ROOM_IR]": 23, - "test_ism_binaural_static[ISM2-BINAURAL_ROOM_IR]": 21, - "test_ism_binaural_static[ISM3-BINAURAL_ROOM_IR]": 21, - "test_ism_binaural_static[ISM4-BINAURAL_ROOM_IR]": 21, - # Failure Reason: Tangent law panning missing in python scripts - "test_ism[ISM1-STEREO]": 8, - "test_ism[ISM2-STEREO]": 13, - "test_ism[ISM3-STEREO]": 13, - "test_ism[ISM4-STEREO]": 13, - # Failure Reason: Casting of positions in renderer to int16_t vs. float in python - "test_ism[ISM1-5_1]": 43, - "test_ism[ISM1-5_1_2]": 43, - "test_ism[ISM1-5_1_4]": 43, - "test_ism[ISM1-7_1]": 40, - "test_ism[ISM1-7_1_4]": 41, - "test_ism[ISM1-FOA]": 49, - "test_ism[ISM1-HOA2]": 45, - "test_ism[ISM1-HOA3]": 42, - "test_ism[ISM2-5_1]": 47, - "test_ism[ISM2-5_1_2]": 44, - "test_ism[ISM2-5_1_4]": 43, - "test_ism[ISM2-7_1]": 44, - "test_ism[ISM2-7_1_4]": 41, - "test_ism[ISM2-FOA]": 47, - "test_ism[ISM2-HOA2]": 43, - "test_ism[ISM2-HOA3]": 40, - "test_ism[ISM3-5_1]": 45, - "test_ism[ISM3-5_1_2]": 43, - "test_ism[ISM3-5_1_4]": 42, - "test_ism[ISM3-7_1]": 43, - "test_ism[ISM3-7_1_4]": 41, - "test_ism[ISM3-FOA]": 47, - "test_ism[ISM3-HOA2]": 43, - "test_ism[ISM3-HOA3]": 40, - "test_ism[ISM4-5_1]": 46, - "test_ism[ISM4-5_1_2]": 43, - "test_ism[ISM4-5_1_4]": 43, - "test_ism[ISM4-7_1]": 45, - "test_ism[ISM4-7_1_4]": 41, - "test_ism[ISM4-FOA]": 47, - "test_ism[ISM4-HOA2]": 43, - "test_ism[ISM4-HOA3]": 40, - # TODO delay alignment of LFE in binaural output - # Failure reason: bitexact except for delay alignment of LFE signal (Issue 59) - "test_multichannel_binaural_headrotation[5_1-BINAURAL-full_circle_in_15s]": 7, - "test_multichannel_binaural_headrotation[5_1-BINAURAL-rotate_yaw_pitch_roll1]": 6, - "test_multichannel_binaural_headrotation[5_1_2-BINAURAL-full_circle_in_15s]": 9, - "test_multichannel_binaural_headrotation[5_1_2-BINAURAL-rotate_yaw_pitch_roll1]": 1, - "test_multichannel_binaural_headrotation[5_1_4-BINAURAL-full_circle_in_15s]": 10, - "test_multichannel_binaural_headrotation[5_1_4-BINAURAL-rotate_yaw_pitch_roll1]": 1, - "test_multichannel_binaural_headrotation[7_1-BINAURAL-full_circle_in_15s]": 8, - "test_multichannel_binaural_headrotation[7_1-BINAURAL-rotate_yaw_pitch_roll1]": 8, - "test_multichannel_binaural_headrotation[7_1_4-BINAURAL-full_circle_in_15s]": 8, - "test_multichannel_binaural_headrotation[7_1_4-BINAURAL-rotate_yaw_pitch_roll1]": 1, - # Failure reason: differences in LFE alignment and possibly rotation - "test_multichannel_binaural_headrotation[5_1-BINAURAL_ROOM_IR-full_circle_in_15s]": 14, - "test_multichannel_binaural_headrotation[5_1-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 12, - "test_multichannel_binaural_headrotation[5_1_2-BINAURAL_ROOM_IR-full_circle_in_15s]": 8, - "test_multichannel_binaural_headrotation[5_1_4-BINAURAL_ROOM_IR-full_circle_in_15s]": 6, - "test_multichannel_binaural_headrotation[5_1_4-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 6, - "test_multichannel_binaural_headrotation[7_1-BINAURAL_ROOM_IR-full_circle_in_15s]": 11, - "test_multichannel_binaural_headrotation[7_1-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 9, - "test_multichannel_binaural_headrotation[5_1_2-BINAURAL_ROOM_IR-rotate_yaw_pitch_roll1]": 6, - # Failure reason: mixed format, see above - "test_metadata[mixed_scene-5_1]": 47, - "test_metadata[mixed_scene-5_1_2]": 47, - "test_metadata[mixed_scene-7_1]": 48, - "test_metadata[mixed_scene-7_1_4]": 47, - "test_metadata[mixed_scene-5_1_4]": 47, -} +""" 5ms framing """ +FRAMING_5MS_TO_TEST = ["5ms", "20ms"] diff --git a/tests/renderer/test_renderer.py b/tests/renderer/test_renderer.py index fd751183bf54a237d0b4c745a4931e3f1e0127cd..67c6f0cd121e405626fd5af41ccad62398ba536e 100644 --- a/tests/renderer/test_renderer.py +++ b/tests/renderer/test_renderer.py @@ -39,25 +39,47 @@ from .utils import * @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ambisonics(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ambisonics_binaural_headrotation( + test_info, in_fmt, out_fmt, trj_file, framing_5ms +): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + run_renderer( in_fmt, out_fmt, test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), ) @@ -76,10 +98,12 @@ def test_ambisonics_binaural_headrotation_refrotzero( ref_kwargs={ "name_extension": "refrotzero", "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), "refrot_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), + "framing_5ms": "5ms", }, ) @@ -96,6 +120,7 @@ def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt out_fmt, ref_kwargs={ "name_extension": "refrotequal", + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath( @@ -104,6 +129,7 @@ def test_ambisonics_binaural_headrotation_refrotequal(test_info, in_fmt, out_fmt "refrot_file": HR_TRAJECTORY_DIR.joinpath( "azi_plus_2-ele_plus_2-every-25-rows.csv" ), + "framing_5ms": "5ms", }, ) @@ -125,10 +151,12 @@ def test_ambisonics_binaural_headrotation_refveczero( ref_kwargs={ "name_extension": "refveczero", "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), "refvec_file": HR_TRAJECTORY_DIR.joinpath("const000-Vector3.csv"), + "framing_5ms": "5ms", }, ) @@ -150,6 +178,7 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt out_fmt, ref_kwargs={ "name_extension": "refvecequal", + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath( @@ -158,6 +187,7 @@ def test_ambisonics_binaural_headrotation_refvecequal(test_info, in_fmt, out_fmt "refvec_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s-Vector3.csv" ), + "framing_5ms": "5ms", }, ) @@ -182,12 +212,14 @@ def test_ambisonics_binaural_headrotation_refvec_rotating(test_info, in_fmt, out "trj_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s.csv" ), + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "refvec_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s-ccw-Vector3.csv" ), + "framing_5ms": "5ms", }, ) @@ -212,12 +244,14 @@ def test_ambisonics_binaural_headrotation_refvec_rotating_fixed_pos_offset( "trj_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s-ccw.csv" ), + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "refvec_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s-fixed-pos-offset-Vector3.csv" ), + "framing_5ms": "5ms", }, ) @@ -242,10 +276,12 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( "refveclev_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s-Vector3.csv" ), + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "refvec_file": HR_TRAJECTORY_DIR.joinpath("full-circle-4s-Vector3.csv"), + "framing_5ms": "5ms", }, ) @@ -255,25 +291,44 @@ def test_ambisonics_binaural_headrotation_refveclev_vs_refvec( @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel_binaural_static(test_info, in_fmt, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name) + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel_binaural_headrotation( + test_info, in_fmt, out_fmt, trj_file, framing_5ms +): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") if (in_fmt == "5_1" or in_fmt == "7_1") and out_fmt == "BINAURAL": run_renderer( @@ -281,6 +336,7 @@ def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file out_fmt, test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), ) else: run_renderer( @@ -288,6 +344,7 @@ def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file out_fmt, test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), ) @@ -310,12 +367,14 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o "trj_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s.csv" ), + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), "refvec_file": HR_TRAJECTORY_DIR.joinpath( "full-circle-with-up-and-down-4s-ccw-Vector3.csv" ), + "framing_5ms": "5ms", }, ) @@ -325,49 +384,59 @@ def test_multichannel_binaural_headrotation_refvec_rotating(test_info, in_fmt, o @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism_binaural_static(test_info, in_fmt, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except: in_meta_files = None - if out_fmt == "BINAURAL": - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=in_meta_files) - else: - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=in_meta_files) + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + in_meta_files=in_meta_files, + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except: in_meta_files = None - if out_fmt == "BINAURAL": - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=in_meta_files, - ) - else: - run_renderer( - in_fmt, - out_fmt, - test_case_name=test_info.node.name, - trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=in_meta_files, - ) + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + in_meta_files=in_meta_files, + framing_5ms=(framing_5ms == "5ms"), + ) # This test compares rendering with: @@ -392,6 +461,7 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): "full-circle-with-up-and-down-4s.csv" ), "in_meta_files": in_meta_files, + "framing_5ms": "5ms", }, cut_kwargs={ "trj_file": HR_TRAJECTORY_DIR.joinpath("const000.csv"), @@ -399,6 +469,7 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): "full-circle-with-up-and-down-4s-ccw-Vector3.csv" ), "in_meta_files": in_meta_files, + "framing_5ms": "5ms", }, ) @@ -408,8 +479,15 @@ def test_ism_binaural_headrotation_refvec_rotating(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -def test_masa(test_info, in_fmt, out_fmt): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt]) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_masa(test_info, in_fmt, out_fmt, framing_5ms): + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), + ) """ Custom loudspeaker layouts """ @@ -417,8 +495,14 @@ def test_masa(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -def test_custom_ls_input(test_info, in_layout, out_fmt): - run_renderer(CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, test_case_name=test_info.node.name) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input(test_info, in_layout, out_fmt, framing_5ms): + run_renderer( + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + test_case_name=test_info.node.name, + framing_5ms=(framing_5ms == "5ms"), + ) @pytest.mark.parametrize("out_fmt", CUSTOM_LS_TO_TEST) @@ -443,10 +527,15 @@ def test_custom_ls_input_output(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_binaural(test_info, in_layout, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + run_renderer( CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, + framing_5ms=(framing_5ms == "5ms"), test_case_name=test_info.node.name, ) @@ -454,12 +543,19 @@ def test_custom_ls_input_binaural(test_info, in_layout, out_fmt): @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input_binaural_headrotation( + test_info, in_layout, out_fmt, trj_file, framing_5ms +): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + run_renderer( CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, test_case_name=test_info.node.name, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), ) @@ -468,12 +564,14 @@ def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, tr @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS[2:]) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) -def test_metadata(test_info, in_fmt, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_metadata(test_info, in_fmt, out_fmt, framing_5ms): run_renderer( "META", out_fmt, test_case_name=test_info.node.name, metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + framing_5ms=(framing_5ms == "5ms"), ) @@ -484,11 +582,21 @@ def test_metadata(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("in_fmt", ["MONO"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_static(test_info, in_fmt, out_fmt, non_diegetic_pan): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, non_diegetic_pan=non_diegetic_pan) + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + non_diegetic_pan=non_diegetic_pan, + ) @pytest.mark.parametrize("out_fmt", ["STEREO"]) @pytest.mark.parametrize("in_fmt", ["ISM1"]) @pytest.mark.parametrize("non_diegetic_pan", ["0", "-30", "45", "90", "-90"]) def test_non_diegetic_pan_ism_static(test_info, in_fmt, out_fmt, non_diegetic_pan): - run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, non_diegetic_pan=non_diegetic_pan) + run_renderer( + in_fmt, + out_fmt, + test_case_name=test_info.node.name, + non_diegetic_pan=non_diegetic_pan, + ) diff --git a/tests/renderer/test_renderer_be_comparison.py b/tests/renderer/test_renderer_be_comparison.py index 44ab8ecd40991e149e97381f536ae5a81dc701be..0647bf1ef908f6bd8ea71407ce5e5b1728105e2c 100644 --- a/tests/renderer/test_renderer_be_comparison.py +++ b/tests/renderer/test_renderer_be_comparison.py @@ -31,8 +31,8 @@ """ import pytest -from .utils import * +from .utils import * """ Ambisonics """ @@ -45,19 +45,34 @@ def test_ambisonics(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt): - compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, is_comparetest=True) +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ambisonics_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + compare_renderer_vs_mergetarget( + test_info, + in_fmt, + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + is_comparetest=True, + ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_AMBI) -def test_ambisonics_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ambisonics_binaural_headrotation( + test_info, in_fmt, out_fmt, trj_file, framing_5ms +): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") compare_renderer_vs_mergetarget( test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), is_comparetest=True, ) @@ -73,25 +88,40 @@ def test_multichannel(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel_binaural_static(test_info, in_fmt, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") - compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, is_comparetest=True) + compare_renderer_vs_mergetarget( + test_info, + in_fmt, + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + is_comparetest=True, + ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MC) -def test_multichannel_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_multichannel_binaural_headrotation( + test_info, in_fmt, out_fmt, trj_file, framing_5ms +): if in_fmt in ["MONO", "STEREO"]: pytest.skip("MONO or STEREO to Binaural rendering unsupported") + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") compare_renderer_vs_mergetarget( test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), is_comparetest=True, ) @@ -109,21 +139,34 @@ def test_ism(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism_binaural_static(test_info, in_fmt, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except: in_meta_files = None compare_renderer_vs_mergetarget( - test_info, in_fmt, out_fmt, in_meta_files=in_meta_files, is_comparetest=True + test_info, + in_fmt, + out_fmt, + in_meta_files=in_meta_files, + framing_5ms=(framing_5ms == "5ms"), + is_comparetest=True, ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_ISM) -def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") + try: in_meta_files = FORMAT_TO_METADATA_FILES[in_fmt] except: @@ -135,12 +178,14 @@ def test_ism_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), in_meta_files=in_meta_files, + framing_5ms=(framing_5ms == "5ms"), is_comparetest=True, ) """ MASA """ + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) def test_masa(test_info, in_fmt, out_fmt): @@ -148,33 +193,45 @@ def test_masa(test_info, in_fmt, out_fmt): test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] ) + @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -def test_masa_binaural_static(test_info, in_fmt, out_fmt): - +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_masa_binaural_static(test_info, in_fmt, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") - + compare_renderer_vs_mergetarget( - test_info, in_fmt, out_fmt, in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] + test_info, + in_fmt, + out_fmt, + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), ) + @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA) -def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): - +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") if out_fmt in ["BINAURAL_ROOM_IR", "BINAURAL_ROOM_REVERB"]: pytest.skip("Skipping binaural room outputs for MASA as unimplemented.") - + compare_renderer_vs_mergetarget( test_info, in_fmt, out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), - in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt] + in_meta_files=FORMAT_TO_METADATA_FILES[in_fmt], + framing_5ms=(framing_5ms == "5ms"), ) + """ Custom loudspeaker layouts """ @@ -182,7 +239,10 @@ def test_masa_binaural_headrotation(test_info, in_fmt, out_fmt, trj_file): @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) def test_custom_ls_input(test_info, in_layout, out_fmt): compare_renderer_vs_mergetarget( - test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, is_comparetest=True + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + is_comparetest=True, ) @@ -190,7 +250,10 @@ def test_custom_ls_input(test_info, in_layout, out_fmt): @pytest.mark.parametrize("in_fmt", OUTPUT_FORMATS) def test_custom_ls_output(test_info, in_fmt, out_fmt): compare_renderer_vs_mergetarget( - test_info, in_fmt, CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), is_comparetest=True + test_info, + in_fmt, + CUSTOM_LAYOUT_DIR.joinpath(f"{out_fmt}.txt"), + is_comparetest=True, ) @@ -207,21 +270,34 @@ def test_custom_ls_input_output(test_info, in_fmt, out_fmt): @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_binaural(test_info, in_layout, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input_binaural(test_info, in_layout, out_fmt, framing_5ms): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") compare_renderer_vs_mergetarget( - test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, is_comparetest=True + test_info, + CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), + out_fmt, + framing_5ms=(framing_5ms == "5ms"), + is_comparetest=True, ) @pytest.mark.parametrize("trj_file", HR_TRAJECTORIES_TO_TEST) @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS_BINAURAL) @pytest.mark.parametrize("in_layout", CUSTOM_LS_TO_TEST) -def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, trj_file): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_custom_ls_input_binaural_headrotation( + test_info, in_layout, out_fmt, trj_file, framing_5ms +): + if framing_5ms != "5ms": + pytest.xfail("Binaural output currently only supported with 5ms framing") compare_renderer_vs_mergetarget( test_info, CUSTOM_LAYOUT_DIR.joinpath(f"{in_layout}.txt"), out_fmt, trj_file=HR_TRAJECTORY_DIR.joinpath(f"{trj_file}.csv"), + framing_5ms=(framing_5ms == "5ms"), is_comparetest=True, ) @@ -231,12 +307,14 @@ def test_custom_ls_input_binaural_headrotation(test_info, in_layout, out_fmt, tr @pytest.mark.parametrize("out_fmt", OUTPUT_FORMATS) @pytest.mark.parametrize("in_fmt", METADATA_SCENES_TO_TEST) -def test_metadata(test_info, in_fmt, out_fmt): +@pytest.mark.parametrize("framing_5ms", FRAMING_5MS_TO_TEST) +def test_metadata(test_info, in_fmt, out_fmt, framing_5ms): compare_renderer_vs_mergetarget( test_info, "META", out_fmt, metadata_input=TEST_VECTOR_DIR.joinpath(f"{in_fmt}.txt"), + framing_5ms=(framing_5ms == "5ms"), is_comparetest=True, ) diff --git a/tests/renderer/utils.py b/tests/renderer/utils.py index 739af721b9e3f191a6e6a28b3c3c28090f1ef91b..8fdc7ead3a30060658494b20fa2f1073e42e6a19 100644 --- a/tests/renderer/utils.py +++ b/tests/renderer/utils.py @@ -31,12 +31,12 @@ """ import logging +import os import subprocess as sp import sys -import os from pathlib import Path from tempfile import TemporaryDirectory -from typing import Optional, Tuple, Dict +from typing import Dict, Optional, Tuple import numpy as np import pytest @@ -47,11 +47,13 @@ from .constants import * sys.path.append(SCRIPTS_DIR) import pyaudio3dtools + # fixture returns test information, enabling per-testcase SNR @pytest.fixture def test_info(request): return request + def run_cmd(cmd, env=None): logging.info(f"\nRunning command\n{' '.join(cmd)}\n") try: @@ -69,7 +71,6 @@ def check_BE( cut: np.ndarray, cut_fs: int, ): - if ref is None or np.array_equal(ref, np.zeros_like(ref)): pytest.fail("REF signal does not exist or is zero!") @@ -81,26 +82,16 @@ def check_BE( if np.isnan(snr) or gain_b == 0: pytest.fail("Invalid comparison result, check your signals!") - # try to get a minimum SNR from the config - if test_info.node.name in pass_snr: - snr_min = pass_snr.get(test_info.node.name) - else: - snr_min = np.inf - - # TODO temporary fix to pad TD Object Renderer Standalone output - if ref.shape != cut.shape: + if ref.shape[0] < cut.shape[0]: ref = np.pad(ref, [(0, cut.shape[0] - ref.shape[0]), (0, 0)]) + elif ref.shape[0] > cut.shape[0]: + cut = np.pad(cut, [(0, ref.shape[0] - cut.shape[0]), (0, 0)]) # check max_diff as well, since compare_audio_arrays will try to adjust for small delay differences if not np.allclose(ref, cut, rtol=0, atol=2) and max_diff > 2: - if snr >= snr_min: - pytest.xfail( - f"Expected failure with minimum SNR {snr_min} vs {snr:3.2f}dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" - ) - else: - pytest.fail( - f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" - ) + pytest.fail( + f"CuT not BE to REF! SNR : {snr:3.2f} dB, Gain CuT: {gain_b:1.3f}, Max Diff = {int(max_diff)}" + ) def run_renderer( @@ -118,6 +109,7 @@ def run_renderer( output_path_base: str = OUTPUT_PATH_CUT, binary_suffix: str = "", is_comparetest: Optional[bool] = False, + framing_5ms: Optional[bool] = False, test_case_name: Optional[str] = None, ) -> Tuple[np.ndarray, int]: """CuT creation with standalone renderer""" @@ -146,7 +138,10 @@ def run_renderer( else: config_name = "" - + if framing_5ms: + framing_name = "_5ms_framing" + else: + framing_name = "" if not isinstance(out_fmt, str): out_name = f"{out_fmt.stem}" @@ -170,7 +165,7 @@ def run_renderer( out_file = str( output_path_base.joinpath( - f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{name_extension}.wav" + f"{in_name}_to_{out_name}{trj_name}{non_diegetic_pan}{refrot_name}{refvec_name}{refveclev_name}{config_name}{framing_name}{name_extension}.wav" ) ) @@ -205,15 +200,21 @@ def run_renderer( if config_file is not None: cmd.extend(["-rc", str(config_file)]) + if framing_5ms: + cmd.extend(["-fr5"]) + # Set env variables for UBSAN env = os.environ.copy() if test_case_name and "UBSAN_OPTIONS" in env.keys(): - env["UBSAN_OPTIONS"] = env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_case_name}" + env["UBSAN_OPTIONS"] = ( + env["UBSAN_OPTIONS"] + f",log_path=usan_log_{test_case_name}" + ) run_cmd(cmd, env) return pyaudio3dtools.audiofile.readfile(out_file) + def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs): ref, ref_fs = run_renderer( in_fmt, @@ -223,16 +224,27 @@ def compare_renderer_vs_mergetarget(test_info, in_fmt, out_fmt, **kwargs): output_path_base=OUTPUT_PATH_REF, **kwargs, ) - cut, cut_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs) + cut, cut_fs = run_renderer( + in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs + ) check_BE(test_info, ref, ref_fs, cut, cut_fs) def compare_renderer_vs_pyscripts(test_info, in_fmt, out_fmt, **kwargs): ref, ref_fs = run_pyscripts(in_fmt, out_fmt, **kwargs) - cut, cut_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs) + cut, cut_fs = run_renderer( + in_fmt, out_fmt, test_case_name=test_info.node.name, **kwargs + ) check_BE(test_info, ref, ref_fs, cut, cut_fs) -def compare_renderer_args(test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict): - ref, ref_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **ref_kwargs) - cut, cut_fs = run_renderer(in_fmt, out_fmt, test_case_name=test_info.node.name, **cut_kwargs) + +def compare_renderer_args( + test_info, in_fmt, out_fmt, ref_kwargs: Dict, cut_kwargs: Dict +): + ref, ref_fs = run_renderer( + in_fmt, out_fmt, test_case_name=test_info.node.name, **ref_kwargs + ) + cut, cut_fs = run_renderer( + in_fmt, out_fmt, test_case_name=test_info.node.name, **cut_kwargs + ) check_BE(test_info, ref, ref_fs, cut, cut_fs) diff --git a/tests/split_rendering/constants.py b/tests/split_rendering/constants.py index 7ec9abc6628204e3dbe7be310b43886562b03784..d9b98e436a993200547a2203baa02e37033e14c1 100644 --- a/tests/split_rendering/constants.py +++ b/tests/split_rendering/constants.py @@ -208,4 +208,5 @@ SPLIT_POST_REND_CMD = [ "BINAURAL", "-tf", "", # 12 -> post-trajectory file + "-fr5", # split decoding must be on 5ms granularity ] diff --git a/tests/split_rendering/test_split_rendering.py b/tests/split_rendering/test_split_rendering.py index d6b959b5728634d82108d6946841378c30b32479..99672eda2150bdee4c253625970bbf9e88ebe7e7 100644 --- a/tests/split_rendering/test_split_rendering.py +++ b/tests/split_rendering/test_split_rendering.py @@ -41,9 +41,13 @@ def check_xfail(test_info, in_fmt, render_config, bitrate=None): and "0dof" in render_config and ( "lc3plus" in render_config - or (in_fmt in INPUT_FORMATS_ISM_SPLIT_REND or in_fmt in INPUT_FORMATS_MC_SPLIT_REND) or ( - "external_split" in test_info.node.name and in_fmt in INPUT_FORMATS_AMBI_SPLIT_REND + in_fmt in INPUT_FORMATS_ISM_SPLIT_REND + or in_fmt in INPUT_FORMATS_MC_SPLIT_REND + ) + or ( + "external_split" in test_info.node.name + and in_fmt in INPUT_FORMATS_AMBI_SPLIT_REND ) # CREND for external renderer ambisonics rendering uses LC3plus by default ) ): @@ -212,6 +216,24 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec ) +@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) +def test_masa_external_split(test_info, in_fmt, render_config, trajectory): + check_xfail(test_info, in_fmt, render_config) + + post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") + pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") + + run_external_split_rendering( + in_fmt=in_fmt, + render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + pre_trajectory=pre_trajectory, + post_trajectory=post_trajectory, + output_path_base=OUTPUT_PATH_CUT, + ) + + """ PLC """ @@ -245,6 +267,7 @@ full_chain_split_pcm_params = [ # ("MASA2", "256000", "split_renderer_config_3dof_384k_lcld"), ] + @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): check_xfail(test_info, in_fmt, render_config, bitrate) @@ -270,6 +293,7 @@ external_split_pcm_params = [ ("ISM1", "split_renderer_config_3dof_384k_lcld"), ] + @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) def test_external_split_pcm(test_info, in_fmt, render_config): check_xfail(test_info, in_fmt, render_config) diff --git a/tests/split_rendering/test_split_rendering_be_comparison.py b/tests/split_rendering/test_split_rendering_be_comparison.py index a300673660f1759122822c897ef269b003dfcb47..68daf3d07c750d11c0e9b49adc21bb3ba0181406 100644 --- a/tests/split_rendering/test_split_rendering_be_comparison.py +++ b/tests/split_rendering/test_split_rendering_be_comparison.py @@ -185,6 +185,24 @@ def test_masa_full_chain_split(test_info, in_fmt, bitrate, render_config, trajec ) +@pytest.mark.parametrize("trajectory", SPLIT_REND_HR_TRAJECTORIES_TO_TEST) +@pytest.mark.parametrize("render_config", RENDERER_CONFIGS_TO_TEST_MASA) +@pytest.mark.parametrize("in_fmt", INPUT_FORMATS_MASA_SPLIT_REND) +def test_masa_external_split(test_info, in_fmt, render_config, trajectory): + check_xfail(test_info, in_fmt, render_config) + + post_trajectory = HR_TRAJECTORY_DIR.joinpath(f"{trajectory}.csv") + pre_trajectory = post_trajectory.with_stem(f"{post_trajectory.stem}_delayed") + + compare_external_split_args( + test_info, + in_fmt=in_fmt, + render_config=RENDER_CFG_DIR.joinpath(f"{render_config}.txt"), + pre_trajectory=pre_trajectory, + post_trajectory=post_trajectory, + ) + + """ PLC """ @@ -218,6 +236,7 @@ full_chain_split_pcm_params = [ # ("MASA2", "256000", "split_renderer_config_3dof_384k_lcld"), ] + @pytest.mark.parametrize("in_fmt,bitrate,render_config", full_chain_split_pcm_params) def test_full_chain_split_pcm(test_info, in_fmt, bitrate, render_config): check_xfail(test_info, in_fmt, render_config, bitrate) @@ -243,6 +262,7 @@ external_split_pcm_params = [ ("ISM1", "split_renderer_config_3dof_384k_lcld"), ] + @pytest.mark.parametrize("in_fmt,render_config", external_split_pcm_params) def test_external_split_pcm(test_info, in_fmt, render_config): check_xfail(test_info, in_fmt, render_config) diff --git a/tests/split_rendering/utils.py b/tests/split_rendering/utils.py index 8130b4146746d476c5d7a9b2c86fb7ba9dded55f..f0d8bbfd710a68ca185186755e4dd7fa7ff31f49 100644 --- a/tests/split_rendering/utils.py +++ b/tests/split_rendering/utils.py @@ -62,14 +62,13 @@ def run_full_chain_split_rendering( """ with TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) - cut_in_file = tmp_dir.joinpath("cut_input.wav") - ivas_bitstream = tmp_dir.joinpath("ivas.192") - split_bitstream = tmp_dir.joinpath("split.bit") + test_file_stem = f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}__config_{render_config.stem}" + cut_in_file = tmp_dir.joinpath(f"{test_file_stem}_input.wav") + ivas_bitstream = tmp_dir.joinpath(f"{test_file_stem}_ivas.192") + split_bitstream = tmp_dir.joinpath(f"{test_file_stem}_split.bit") + out_file = output_path_base.joinpath(f"{test_file_stem}_output.wav") if renderer_fmt == "BINAURAL_SPLIT_PCM": - split_md_file = tmp_dir.joinpath("split_md.bin") - out_file = output_path_base.joinpath( - f"{in_fmt}_{bitrate}bps_{renderer_fmt}_{pre_trajectory.stem}_split_full_{post_trajectory.stem}_config_{render_config.stem}.wav" - ) + split_md_file = tmp_dir.joinpath(f"{test_file_stem}_split_md.bin") # check for metadata files if in_fmt.upper().startswith("ISM") or in_fmt.upper().startswith("MASA"): @@ -159,14 +158,14 @@ def run_external_split_rendering( with TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) - split_bitstream = tmp_dir.joinpath("split.bit") - if renderer_fmt == "BINAURAL_SPLIT_PCM": - split_md_file = tmp_dir.joinpath("split_md.bin") - out_file = output_path_base.joinpath( - f"{in_fmt}_{renderer_fmt}_{pre_trajectory.stem}_split_ext_{post_trajectory.stem}_config_{render_config.stem}.wav" - ) + test_file_stem = f"{in_fmt}_{renderer_fmt}_{pre_trajectory.stem}_split_ext_{post_trajectory.stem}__config_{render_config.stem}" if plc_error_pattern: - out_file = out_file.with_stem(f"{out_file.stem}_plc_{plc_error_pattern.stem}") + test_file_stem += f"_plc_{plc_error_pattern.stem}" + + split_bitstream = tmp_dir.joinpath(f"{test_file_stem}_split.bit") + out_file = output_path_base.joinpath(f"{test_file_stem}_out.wav") + if renderer_fmt == "BINAURAL_SPLIT_PCM": + split_md_file = tmp_dir.joinpath(f"{test_file_stem}_split_md.bin") # check for metadata files if in_fmt.upper().startswith("ISM") or in_fmt.upper().startswith("MASA"): @@ -193,6 +192,18 @@ def run_external_split_rendering( if in_meta_files: cmd[9:9] = ["-im", *in_meta_files] + # pre renderer for 0DoF needs -fr5 except where LCLD is used + if "0dof" in render_config.name: + if not ( + "fastconv" in render_config.name + or "lcld" in render_config.name + or ( + in_fmt.upper().startswith("MASA") + and "lc3plus" not in render_config.name + ) + ): + cmd.append("-fr5") + run_cmd(cmd) # run split renderer