From f32b4cf6686a7fb76b5370d9346db2c81c5ace99 Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Fri, 26 Apr 2024 18:16:06 +0200 Subject: [PATCH 1/2] Revert "Remove changes under SPLIT_REND_METADATA_SECTION_SIZE" This reverts commit e20ee047a18482d798d16d2ac3b10c04c04e90f2. --- lib_com/options.h | 1 + lib_isar/isar_cnst.h | 4 ++++ lib_isar/isar_splitRendererPre.c | 20 ++++++++++++++++++++ lib_isar/lib_isar_post_rend.c | 14 ++++++++++++++ lib_isar/lib_isar_pre_rend.c | 20 ++++++++++++++++++++ 5 files changed, 59 insertions(+) diff --git a/lib_com/options.h b/lib_com/options.h index 6455ca1871..796f9bd083 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -163,6 +163,7 @@ #define SPLIT_REND_EXTEND_ISAR_FILE_HEADER /* FhG: Adds more fields to ISAR bitstream file header: - LC3plus HIRES flag, coding sample rate - required to open LC3plus decoder before decoding first frame. - ISAR frame size - fixes 10ms framing in ISAR. */ +#define SPLIT_REND_METADATA_SECTION_SIZE /* FhG: prepends metadata section in the split bitstream frame with size information */ #define FIX_NUM_SUBFRAME_UPDATE diff --git a/lib_isar/isar_cnst.h b/lib_isar/isar_cnst.h index 17df3c9d98..ce6b7e1987 100644 --- a/lib_isar/isar_cnst.h +++ b/lib_isar/isar_cnst.h @@ -109,6 +109,10 @@ typedef enum #define ISAR_SPLIT_REND_HQ_MODE_BITS 1 #define ISAR_SPLIT_REND_ROT_AXIS_BITS 3 +#ifdef SPLIT_REND_METADATA_SECTION_SIZE +#define IVAS_SPLIT_REND_MD_SIZE_BITS 13 /* Max expected size of the MD section: 256000 kbps / 50 fps = 5120 < 2^13 = 8192 */ +#endif + #ifdef LC3PLUS_RTP_PAYLOAD_FORMAT #define IVAS_LC3PLUS_MAX_NUM_DECODERS 2 #endif diff --git a/lib_isar/isar_splitRendererPre.c b/lib_isar/isar_splitRendererPre.c index 10a6c07f12..bac037b712 100644 --- a/lib_isar/isar_splitRendererPre.c +++ b/lib_isar/isar_splitRendererPre.c @@ -1854,13 +1854,33 @@ static void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, const int16_t ro_md_flag ) { int32_t available_md_bits; +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + int32_t md_section_start, md_section_end; + + md_section_start = pBits->bits_written; +#endif available_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000; +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + /* Skip first few bits, where the size info of the pose correction metadata section + * will be written later (it is not yet known at this point) */ + available_md_bits -= IVAS_SPLIT_REND_MD_SIZE_BITS; + pBits->bits_written += IVAS_SPLIT_REND_MD_SIZE_BITS; +#endif + /* Write pose correction metadata */ if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, available_md_bits, low_res_pre_rend_rot, ro_md_flag ); } + +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + /* Update metadata section size field with the actual number of bits written */ + md_section_end = pBits->bits_written; + pBits->bits_written = md_section_start; /* Hack to write to the beginning of the metadata section */ + ISAR_SPLIT_REND_BITStream_write_int32( pBits, md_section_end - md_section_start, IVAS_SPLIT_REND_MD_SIZE_BITS ); + pBits->bits_written = md_section_end; +#endif } #endif diff --git a/lib_isar/lib_isar_post_rend.c b/lib_isar/lib_isar_post_rend.c index db8bcc8828..00c282f794 100644 --- a/lib_isar/lib_isar_post_rend.c +++ b/lib_isar/lib_isar_post_rend.c @@ -1399,6 +1399,11 @@ static ivas_error renderSplitBinauralWithPostRot( float *readPtr, *writePtr; int16_t iNumBlocksPerFrame, iNumLCLDIterationsPerFrame; const ISAR_POST_REND_HeadRotData *pHeadRotData; +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + int16_t split_rend_md_section_size_bits; + + split_rend_md_section_size_bits = 0; +#endif isPostRendInputCldfb = 0; push_wmops( "renderSplitBinauralWithPostRot" ); error = IVAS_ERR_OK; @@ -1437,6 +1442,15 @@ static ivas_error renderSplitBinauralWithPostRot( if ( hSplitBin->first_good_frame_received == 1 ) { +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + if ( bits.pose_correction != ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) + { + // This info is currently unused, but will later be needed e.g. when packing SR bitstream into RTP + split_rend_md_section_size_bits = (int16_t) ISAR_SPLIT_REND_BITStream_read_int32( &bits, IVAS_SPLIT_REND_MD_SIZE_BITS ); + (void) split_rend_md_section_size_bits; + } +#endif + if ( bits.pose_correction == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { if ( !SplitRendBFI ) diff --git a/lib_isar/lib_isar_pre_rend.c b/lib_isar/lib_isar_pre_rend.c index f9aedec524..48a6ffaaa4 100644 --- a/lib_isar/lib_isar_pre_rend.c +++ b/lib_isar/lib_isar_pre_rend.c @@ -330,13 +330,33 @@ static void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, const int16_t ro_md_flag ) { int32_t available_md_bits; +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + int32_t md_section_start, md_section_end; + + md_section_start = pBits->bits_written; +#endif available_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000; +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + /* Skip first few bits, where the size info of the pose correction metadata section + * will be written later (it is not yet known at this point) */ + available_md_bits -= IVAS_SPLIT_REND_MD_SIZE_BITS; + pBits->bits_written += IVAS_SPLIT_REND_MD_SIZE_BITS; +#endif + /* Write pose correction metadata */ if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, available_md_bits, low_res_pre_rend_rot, ro_md_flag ); } + +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + /* Update metadata section size field with the actual number of bits written */ + md_section_end = pBits->bits_written; + pBits->bits_written = md_section_start; /* Hack to write to the beginning of the metadata section */ + ISAR_SPLIT_REND_BITStream_write_int32( pBits, md_section_end - md_section_start, IVAS_SPLIT_REND_MD_SIZE_BITS ); + pBits->bits_written = md_section_end; +#endif } #endif -- GitLab From a0f5a8423326c6d7b5cbfad32f52490b6af24fa0 Mon Sep 17 00:00:00 2001 From: Kacper Sagnowski Date: Mon, 6 May 2024 18:12:52 +0200 Subject: [PATCH 2/2] Use the function write_pose_correction_section again --- lib_com/options.h | 4 +- lib_isar/isar_prot.h | 13 +++++++ lib_isar/isar_splitRendererPre.c | 63 ++++++++++---------------------- lib_isar/lib_isar_pre_rend.c | 32 +++++++++++----- 4 files changed, 58 insertions(+), 54 deletions(-) diff --git a/lib_com/options.h b/lib_com/options.h index d78cb4a638..b88e6f41c2 100644 --- a/lib_com/options.h +++ b/lib_com/options.h @@ -156,7 +156,9 @@ /*#define FIX_I4_OL_PITCH*/ /* fix open-loop pitch used for EVS core switching */ #define SPLIT_REND_WITH_HEAD_ROT /* Dlb,FhG: Split Rendering contributions 21 and 35 */ #define ISAR_BITSTREAM_UPDATE_LC3PLUS /* FhG: Multiple improvements to the ISAR bitstream when LC3plus is used. See MR 1456 for details. */ -#define SPLIT_REND_METADATA_SECTION_SIZE /* FhG: prepends metadata section in the split bitstream frame with size information */ +#ifdef ISAR_BITSTREAM_UPDATE_LC3PLUS +#define SPLIT_REND_METADATA_SECTION_SIZE /* FhG: In the split bitstream frame, prepends metadata section with size information */ +#endif #define FIX_NUM_SUBFRAME_UPDATE diff --git a/lib_isar/isar_prot.h b/lib_isar/isar_prot.h index ca9d86760d..2812caedb5 100644 --- a/lib_isar/isar_prot.h +++ b/lib_isar/isar_prot.h @@ -333,6 +333,19 @@ ivas_error isar_split_rend_choose_default_codec( const int16_t num_subframes /* i : number of subframes */ ); + +#ifdef SPLIT_REND_METADATA_SECTION_SIZE +void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, + const IVAS_QUATERNION headPosition, + const int32_t SplitRendBitRate, + ISAR_SPLIT_REND_BITS_HANDLE pBits, + float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], + const int16_t low_res_pre_rend_rot, + const int16_t pcm_out_flag, + const int16_t ro_md_flag ); +#endif + void ISAR_SPLIT_REND_BITStream_init( ISAR_SPLIT_REND_BITS_HANDLE pBits, const int32_t buf_len_bytes, diff --git a/lib_isar/isar_splitRendererPre.c b/lib_isar/isar_splitRendererPre.c index 5e772b3065..d91f0bbd5e 100644 --- a/lib_isar/isar_splitRendererPre.c +++ b/lib_isar/isar_splitRendererPre.c @@ -1843,48 +1843,6 @@ ivas_error splitRendLc3plusEncodeAndWrite( return IVAS_ERR_OK; } -#ifdef SPLIT_REND_METADATA_SECTION_SIZE -static void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, - const IVAS_QUATERNION headPosition, - const int32_t SplitRendBitRate, - ISAR_SPLIT_REND_BITS_HANDLE pBits, - float Cldfb_In_BinReal[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - float Cldfb_In_BinImag[][CLDFB_NO_COL_MAX][CLDFB_NO_CHANNELS_MAX], - const int16_t low_res_pre_rend_rot, - const int16_t pcm_out_flag, - const int16_t ro_md_flag ) -{ - int32_t available_md_bits; -#ifdef SPLIT_REND_METADATA_SECTION_SIZE - int32_t md_section_start, md_section_end; - - md_section_start = pBits->bits_written; -#endif - available_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000; - -#ifdef SPLIT_REND_METADATA_SECTION_SIZE - /* Skip first few bits, where the size info of the pose correction metadata section - * will be written later (it is not yet known at this point) */ - available_md_bits -= IVAS_SPLIT_REND_MD_SIZE_BITS; - pBits->bits_written += IVAS_SPLIT_REND_MD_SIZE_BITS; -#endif - - /* Write pose correction metadata */ - if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) - { - isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, available_md_bits, low_res_pre_rend_rot, ro_md_flag ); - } - -#ifdef SPLIT_REND_METADATA_SECTION_SIZE - /* Update metadata section size field with the actual number of bits written */ - md_section_end = pBits->bits_written; - pBits->bits_written = md_section_start; /* Hack to write to the beginning of the metadata section */ - ISAR_SPLIT_REND_BITStream_write_int32( pBits, md_section_end - md_section_start, IVAS_SPLIT_REND_MD_SIZE_BITS ); - pBits->bits_written = md_section_end; -#endif -} -#endif - /*------------------------------------------------------------------------- * Function isar_renderMultiTDBinToSplitBinaural() @@ -1909,7 +1867,10 @@ ivas_error isar_renderMultiTDBinToSplitBinaural( { ivas_error error; #ifdef ISAR_BITSTREAM_UPDATE_LC3PLUS - int32_t bit_len, available_bits, target_md_bits; + int32_t bit_len, available_bits; +#ifndef SPLIT_REND_METADATA_SECTION_SIZE + int32_t target_md_bits; +#endif #else int32_t bit_len, available_bits, target_md_bits, actual_md_bits; #endif @@ -1988,6 +1949,21 @@ ivas_error isar_renderMultiTDBinToSplitBinaural( } } + +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + if ( hSplitBin->multiBinPoseData.poseCorrectionMode != ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) + { + write_pose_correction_section( hSplitBin, + headPosition, + SplitRendBitRate, + pBits, + Cldfb_In_BinReal, + Cldfb_In_BinImag, + low_res_pre_rend_rot, + pcm_out_flag, + ro_md_flag ); + } +#else if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) { target_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000; @@ -1998,6 +1974,7 @@ ivas_error isar_renderMultiTDBinToSplitBinaural( isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, target_md_bits, low_res_pre_rend_rot, ro_md_flag ); } +#endif if ( pcm_out_flag == 0 ) { diff --git a/lib_isar/lib_isar_pre_rend.c b/lib_isar/lib_isar_pre_rend.c index 4df266ca8d..49ac6a65d4 100644 --- a/lib_isar/lib_isar_pre_rend.c +++ b/lib_isar/lib_isar_pre_rend.c @@ -312,14 +312,14 @@ void ISAR_PRE_REND_GetMultiBinPoseData( isar_renderSplitGetMultiBinPoseData( pSplit_rend_config, pMultiBinPoseData, rot_axis ); } +#ifdef SPLIT_REND_METADATA_SECTION_SIZE /*------------------------------------------------------------------------- * Function write_pose_correction_section() * * *------------------------------------------------------------------------*/ -#ifdef SPLIT_REND_METADATA_SECTION_SIZE -static void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, +void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, const IVAS_QUATERNION headPosition, const int32_t SplitRendBitRate, ISAR_SPLIT_REND_BITS_HANDLE pBits, @@ -330,19 +330,15 @@ static void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, const int16_t ro_md_flag ) { int32_t available_md_bits; -#ifdef SPLIT_REND_METADATA_SECTION_SIZE int32_t md_section_start, md_section_end; md_section_start = pBits->bits_written; -#endif available_md_bits = isar_get_split_rend_md_target_brate( SplitRendBitRate, pcm_out_flag ) * L_FRAME48k / 48000; -#ifdef SPLIT_REND_METADATA_SECTION_SIZE /* Skip first few bits, where the size info of the pose correction metadata section * will be written later (it is not yet known at this point) */ available_md_bits -= IVAS_SPLIT_REND_MD_SIZE_BITS; pBits->bits_written += IVAS_SPLIT_REND_MD_SIZE_BITS; -#endif /* Write pose correction metadata */ if ( hSplitBin->multiBinPoseData.poseCorrectionMode == ISAR_SPLIT_REND_POSE_CORRECTION_MODE_CLDFB ) @@ -350,13 +346,11 @@ static void write_pose_correction_section( SPLIT_REND_WRAPPER *hSplitBin, isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, available_md_bits, low_res_pre_rend_rot, ro_md_flag ); } -#ifdef SPLIT_REND_METADATA_SECTION_SIZE /* Update metadata section size field with the actual number of bits written */ md_section_end = pBits->bits_written; - pBits->bits_written = md_section_start; /* Hack to write to the beginning of the metadata section */ + pBits->bits_written = md_section_start; /* Write to the beginning of the metadata section */ ISAR_SPLIT_REND_BITStream_write_int32( pBits, md_section_end - md_section_start, IVAS_SPLIT_REND_MD_SIZE_BITS ); pBits->bits_written = md_section_end; -#endif } #endif @@ -388,7 +382,10 @@ ivas_error ISAR_PRE_REND_MultiBinToSplitBinaural( { ivas_error error; #ifdef ISAR_BITSTREAM_UPDATE_LC3PLUS - int32_t bit_len, target_md_bits, available_bits; + int32_t bit_len, available_bits; +#ifndef SPLIT_REND_METADATA_SECTION_SIZE + int32_t target_md_bits; +#endif #else int32_t bit_len, target_md_bits, actual_md_bits, available_bits; #endif @@ -430,6 +427,20 @@ ivas_error ISAR_PRE_REND_MultiBinToSplitBinaural( lc3plusTimeAlignCldfbPoseCorr( hSplitBin, Cldfb_In_BinReal, Cldfb_In_BinImag ); } +#ifdef SPLIT_REND_METADATA_SECTION_SIZE + if ( hSplitBin->multiBinPoseData.poseCorrectionMode != ISAR_SPLIT_REND_POSE_CORRECTION_MODE_NONE ) + { + write_pose_correction_section( hSplitBin, + headPosition, + SplitRendBitRate, + pBits, + Cldfb_In_BinReal, + Cldfb_In_BinImag, + low_res_pre_rend_rot, + pcm_out_flag, + ro_md_flag ); + } +#else #ifndef ISAR_BITSTREAM_UPDATE_LC3PLUS actual_md_bits = pBits->bits_written; #endif @@ -443,6 +454,7 @@ ivas_error ISAR_PRE_REND_MultiBinToSplitBinaural( isar_rend_CldfbSplitPreRendProcess( hSplitBin->hBinHrSplitPreRend, headPosition, &hSplitBin->multiBinPoseData, Cldfb_In_BinReal, Cldfb_In_BinImag, pBits, target_md_bits, low_res_pre_rend_rot, ro_md_flag ); } +#endif if ( pcm_out_flag == 0 ) { -- GitLab