Merge branch 'main' into 20240425_OSS_attribution_FhG (c931cfcf) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitignore

+75 −0

Original line number	Diff line number	Diff line
		@@ -88,3 +88,78 @@ coan_out_*
		/scripts/mem_analysis_enc_STEREO_sw.csv
		/scripts/mem_analysis_enc_STEREO_16k4_DTX.csv
		*.pwv
		/dcr_ism1
		/bwe_clicks_ism1
		/BWD_issue
		/.ipynb_checkpoints
		/scripts/.ipynb_checkpoints
		/scripts/wmops_analysis
		/scripts/wmops_analysis_new
		/tcx_hq_problem_ism1
		/scripts/dly_error_profiles/dly_profile_19_.dat
		/build.txt
		/bit.64k
		/bit.32k
		/bit
		/binaural_loudness.ipynb
		/tmp.txt
		/syn.wav
		/syn.ref
		/syn.64k.0.csv
		/syn.64k
		/syn.32k
		/stvST32c_-stereo_32000_32.bit
		/split_renderer_config_768_3dof_cldfbpc.txt
		/split_out_sba_lcld_20ms_out.wav
		/split_out_sba_lcld_20ms.pkt
		/split_out_mc_lc3plus_20ms_out.wav
		/split_out_mc_lc3plus_20ms.pkt
		/run_ivas_codec.sh
		/report.html
		/report-junit.xm
		/out.wav
		/ltv48_1ISM.cut14.wav.0.csv
		/ltv48_1ISM.cut14.pcm
		/ltv48_1ISM.cut14.dec.pcm.0.csv
		/ltv48_1ISM.cut14.64k.dec.pcm.0.csv
		/ltv48_1ISM.cut14.64k.dec.pcm
		/ltv48_1ISM.cut14.32k.dec.pcm
		/IvasBuilder.txt
		/ivas.192
		/ism_loudness_stv48_48kHz.md
		/ism_loudness_stv48_32kHz.md
		/ism_loudness_sel48_48kHz.md
		/ism_loudness_old.xlsx
		/ism_loudness_ltv48_ISM4.md
		/ism_loudness_ltv48_ISM3.md
		/ism_loudness_ltv48_ISM2.md
		/ism_loudness_ltv48_ISM1.md
		/ism_loudness_ltv48_48kHz.md
		/ism_loudness.xlsx
		/ism_loudness.py
		/HOA2_to_BINAURAL_ROOM_REVERBNone_rend_config_combined_20msNone_0-1000_2-500_1-500.wav
		/git_blame_string.sh
		/get_license_table.py
		/enc_out_mc.pkt
		/enc_out.pkt
		/cut_ism.py
		/scripts/Vlad_mergeNewsletters.py
		/scripts/Vlad_extract_max_num_ind.py
		/scripts/tables_obj.txt
		/scripts/tables_exe.txt
		/scripts/requirements.txt
		/scripts/parseNewsletterWmops.py
		/scripts/output.xlsx
		/scripts/functions_obj.txt
		/scripts/functions_exe.txt
		/scripts/dep_tree.txt
		/scripts/cmplx_evs_in_ivas.xlsx
		/scripts/bit
		/scripts/binaural_loudness.ipynb
		/scripts/config/Vlad_self_test_ltv.prm
		/scripts/config/Vlad_self_test.prm
		/scripts/config/Vlad_ci_linux_ltv.json
		/scripts/config/Vlad_ci_linux.json
		/cmake-build
		/bit.fix
		/syn

.gitlab-ci.yml

+22 −80

Original line number	Diff line number	Diff line
		@@ -12,7 +12,7 @@ variables:
		OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"
		OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"
		OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"
		OUT_FORMATS_ALL: "$OUT_FORMATS_ALL"
		OUT_FORMATS_ALL: "$OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT"
		IVAS_PIPELINE_NAME: ''
		MANUAL_PIPELINE_TYPE:
		description: "Type for the manual pipeline run. Use 'test-be-release' to run BE test against release codec."
		@@ -174,11 +174,6 @@ stages:
		- git pull
		- Pop-Location

		.enable-split-rendering: &enable-split-rendering
		# automatically enable #define SPLIT_REND_WITH_HEAD_ROT in options.h, handling both /**/-comment and //-comment
		- sed -i.bak -e "s/\/\[[:space:]]$#define[[:space:]]SPLIT_REND_WITH_HEAD_ROT$[[:space:]]\*\//\1/g" ./lib_com/options.h
		- sed -i.bak -e "s/\/\/[[:space:]]$#define[[:space:]]SPLIT_REND_WITH_HEAD_ROT$/\1/g" ./lib_com/options.h

		.disable-limiter: &disable-limiter
		# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment
		- sed -i.bak -e "s/\/\[[:space:]]$#define[[:space:]]DISABLE_LIMITER$[[:space:]]\*\//\1/g" ./lib_com/options.h
		@@ -227,12 +222,12 @@ stages:
		- if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi

		# run encoder and decoder with 20ms renderer framesize first, use reference creation mode
		- python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec \|\| exit_code20=$?
		- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec \|\| exit_code20=$?
		# for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option
		# set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active
		# we are only interested in runtime errors from the sanitizers and ignore the diffs
		- python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 \|\| exit_code5=$?
		- python3 -m pytest $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 \|\| exit_code10=$?
		- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 \|\| exit_code5=$?
		- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 \|\| exit_code10=$?

		- if [ $exit_code20 -ne 0 ] \|\| [ $exit_code10 -ne 0 ] \|\| [ $exit_code5 -ne 0 ]; then exit 1; fi

		@@ -460,30 +455,6 @@ build-codec-sanitizers-linux:
		- *activate-Werror-linux
		- bash ci/build_codec_sanitizers_linux.sh

		build-codec-include-split-linux-make:
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- *print-common-info
		- *enable-split-rendering
		- *activate-Werror-linux
		- make -j INCLUDE_SPLIT=1

		build-codec-include-split-linux-cmake:
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- *print-common-info
		- *enable-split-rendering
		- *activate-Werror-linux
		- mkdir build
		- cd build
		- cmake .. -DINCLUDE_SPLIT=1
		- cd ..
		- make -C build -j

		build-codec-windows-cmake:
		extends:
		- .build-job-windows
		@@ -494,19 +465,6 @@ build-codec-windows-cmake:
		- cmake -G "Visual Studio 15 2017" . -Bbuild
		- cmake --build build -j

		build-codec-windows-include-split-cmake:
		extends:
		- .build-job-windows
		- .rules-basis
		script:
		- *print-common-info-windows
		- *activate-WX-windows
		- get-content .\lib_com\options.h \| %{$_ -replace "/\#define[\s]SPLIT_REND_WITH_HEAD_ROT[\s]\/", "#define SPLIT_REND_WITH_HEAD_ROT"} \| set-content -Path ./options_patched.h
		- rm ./lib_com/options.h
		- mv ./options_patched.h ./lib_com/options.h
		- cmake -DINCLUDE_SPLIT=1 -G "Visual Studio 15 2017" . -Bbuild
		- cmake --build build -j

		build-codec-windows-msbuild:
		extends:
		- .build-job-windows
		@@ -561,7 +519,7 @@ codec-msan:
		before_script:
		- CLANG_NUM=1
		- SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS
		- SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"
		- SELF_TEST_PRM_FILE="scripts/config/self_test.prm"
		<<: *sanitizer-selftest-anchor

		# code selftest testvectors with address-sanitizer binaries
		@@ -571,7 +529,7 @@ codec-asan:
		before_script:
		- CLANG_NUM=2
		- SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS
		- SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"
		- SELF_TEST_PRM_FILE="scripts/config/self_test.prm"
		<<: *sanitizer-selftest-anchor

		# code selftest testvectors with undefined-behaviour-sanitizer binaries
		@@ -583,7 +541,7 @@ codec-usan:
		before_script:
		- CLANG_NUM=3
		- SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS
		- SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"
		- SELF_TEST_PRM_FILE="scripts/config/self_test.prm"
		<<: *sanitizer-selftest-anchor

		# compare bit-exactness between 5ms and 20 on the branch
		@@ -792,8 +750,7 @@ split-rendering-smoke-test:
		needs: ["build-codec-linux-make"]
		stage: test
		script:
		- *enable-split-rendering
		- make -j INCLUDE_SPLIT=1
		- make -j
		- testcase_timeout=10
		- python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout
		artifacts:
		@@ -814,7 +771,6 @@ lc3-wrapper-unit-test:
		needs: ["build-codec-linux-cmake"]
		stage: test
		script:
		- *enable-split-rendering
		- cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true
		- cmake --build cmake-build -- -j
		- scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test
		@@ -848,8 +804,7 @@ split-rendering-pytest-on-merge-request:
		- echo "Building reference codec at commit $target_commit"

		# build reference binaries
		- *enable-split-rendering
		- make -j INCLUDE_SPLIT=1
		- make -j
		- mv IVAS_cod IVAS_cod_ref
		- mv IVAS_dec IVAS_dec_ref
		- mv IVAS_rend IVAS_rend_ref
		@@ -865,8 +820,7 @@ split-rendering-pytest-on-merge-request:
		- git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts
		- git checkout $source_branch_commit_sha
		- make clean
		- *enable-split-rendering
		- make -j INCLUDE_SPLIT=1
		- make -j

		### Run test using scripts and input from main
		- if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts
		@@ -1122,29 +1076,17 @@ check-first-frame-is-sid:
		script:
		- *print-common-info
		- *update-ltv-repo
		# Temporary fix to test only SID start config files in this job
		# this rm makes check-for-testvectors only check for the signals we actually need in this test
		- rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json
		- *check-for-testvectors
		- cmake .
		- make -j
		- bash ci/run-first-frame-is-sid-test.sh

		- exit_code_no_sba=0
		- exit_code_hoa=0
		- exit_code_foa=0

		# run all modes and cut bitstream to start with an SID. Use stereo output to limit runtime, test is only about decoding
		- ism_md_cmd="--ism_metadata_files /usr/local/ltv/ltvISM1.csv /usr/local/ltv/ltvISM2.csv /usr/local/ltv/ltvISM3.csv /usr/local/ltv/ltvISM4.csv"
		- modes=$(scripts/runIvasCodec.py -l \| grep dtx \| grep -vE "FOA\|HOA" )
		- scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 200 -U 0:20 --oc stereo $ism_md_cmd --timeout 10 \|\| exit_code_no_sba=$?
		- modes=$(scripts/runIvasCodec.py -l \| grep dtx \| grep -E "HOA")
		- scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 70:80 --oc stereo --timeout 10 \|\| exit_code_hoa=$?
		- modes=$(scripts/runIvasCodec.py -l \| grep dtx \| grep "FOA")
		- scripts/runIvasCodec.py -z console -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 100 -U 75:110 --oc stereo timeout 10 \|\| exit_code_foa=$?

		- if [ $exit_code_no_sba -ne 0 ] \|\| [ $exit_code_hoa -ne 0 ] \|\| [ $exit_code_hoa -ne 0 ]; then exit 1; fi
		artifacts:
		paths:
		- out/logs
		- logs_enc
		- logs_dec_msan
		- logs_dec_asan
		- logs_dec_usan
		when: always
		name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart"
		expose_as: "logs-sidstart"
		@@ -1661,13 +1603,13 @@ ltv-msan:
		- .sanitizer-selftest-ltv
		rules:
		- if: $SANITIZER_SCHEDULE_E
		timeout: 3 hour
		timeout: 4 hour
		tags:
		- ivas-linux-fast
		before_script:
		- CLANG_NUM=1
		- SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS
		- SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"
		- SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"
		<<: *sanitizer-selftest-anchor

		# code selftest long testvectors with address-sanitizer binaries
		@@ -1677,14 +1619,14 @@ ltv-asan:
		rules:
		- if: $SANITIZER_SCHEDULE_E
		when: delayed
		start_in: 3 hours
		start_in: 4 hours
		tags:
		- ivas-linux-fast
		timeout: 3 hour
		before_script:
		- CLANG_NUM=2
		- SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS
		- SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"
		- SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"
		<<: *sanitizer-selftest-anchor

		# code selftest long testvectors with undefined-behaviour-sanitizer binaries
		@@ -1694,14 +1636,14 @@ ltv-usan:
		rules:
		- if: $SANITIZER_SCHEDULE_E
		when: delayed
		start_in: 6 hours
		start_in: 7 hours
		tags:
		- ivas-linux-fast
		timeout: 3 hour
		before_script:
		- CLANG_NUM=3
		- SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS
		- SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"
		- SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"
		<<: *sanitizer-selftest-anchor

		.sanitizer-test-template:

CMakeLists.txt

+2 −2

Original line number	Diff line number	Diff line
		@@ -74,14 +74,14 @@ if(UNIX)
		set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
		elseif("${CLANG}" MATCHES "3" OR "${CLANG}" MATCHES "usan")
		# NOTE: keep in sync with list in Makefile
		set(USAN_CHECKS_ENABLE
		set(USAN_CHECKS_LIST
		undefined # Default checks
		# Extra checks
		float-divide-by-zero
		implicit-conversion
		local-bounds
		)
		list(JOIN USAN_CHECKS_ENABLE "," USAN_CHECKS_ENABLE)
		string(JOIN , USAN_CHECKS_ENABLE ${USAN_CHECKS_LIST})

		set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}")
		set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}")

Workspace_msvc/lib_util.vcxproj

+2 −0

Original line number	Diff line number	Diff line
		@@ -100,6 +100,7 @@
		</Lib>
		</ItemDefinitionGroup>
		<ItemGroup>
		<ClCompile Include="..\lib_util\aeid_file_reader.c" />
		<ClCompile Include="..\lib_util\audio_file_reader.c" />
		<ClCompile Include="..\lib_util\audio_file_writer.c" />
		<ClCompile Include="..\lib_util\bitstream_reader.c" />
		@@ -126,6 +127,7 @@
		<ClCompile Include="..\lib_util\tsm_scale_file_reader.c" />
		</ItemGroup>
		<ItemGroup>
		<ClInclude Include="..\lib_util\aeid_file_reader.h" />
		<ClInclude Include="..\lib_util\audio_file_reader.h" />
		<ClInclude Include="..\lib_util\audio_file_writer.h" />
		<ClInclude Include="..\lib_util\bitstream_reader.h" />

apps/decoder.c

+64 −326

File changed.

Preview size limit exceeded, changes collapsed.