Merge branch 'main' of forge.3gpp.org:ivas-codec-pc/ivas-codec into FhG/renderer_and_ci_cleanup (c87f0714) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitignore

+6 −0

Original line number	Diff line number	Diff line
		@@ -47,10 +47,16 @@ scripts/ref/
		scripts/test/
		scripts/out/
		scripts/self_test_summary.txt
		scripts/cppp/
		binary/
		tests/renderer/cut
		tests/renderer/ref
		tests/dut
		tests/ref
		scripts/testv/_cut.pcm
		# default reference binary name
		IVAS_cod_ref
		IVAS_dec_ref

		# Python files that pop up when running scripts
		__pycache__/

.gitlab-ci.yml

+46 −62

Original line number	Diff line number	Diff line
		@@ -147,7 +147,7 @@ stages:

		.build-job-linux:
		stage: build
		timeout: "2 minutes"
		timeout: "4 minutes"
		tags:
		- ivas-linux

		@@ -250,12 +250,14 @@ codec-smoke-test:
		### analyze for failures
		- if cat smoke_test_output.txt \| grep -c "failed"; then echo "Smoke test without PLC failed"; exit 1; fi
		- if cat smoke_test_output_plc.txt \| grep -c "failed"; then echo "Smoke test with PLC failed"; exit 1; fi
		- if cat smoke_test_output_jbm_noEXT.txt \| grep -c "failed"; then echo "Smoke test with PLC failed"; exit 1; fi
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		paths:
		- out/logs/
		- smoke_test_output.txt
		- smoke_test_output_plc.txt
		- smoke_test_output_jbm_noEXT.txt
		expose_as: "Smoke test results"

		# code selftest testvectors with memory-sanitizer binaries
		@@ -267,6 +269,7 @@ msan-on-merge-request-linux:
		needs: ["build-codec-sanitizers-linux"]
		script:
		- *print-common-info
		- python3 ci/disable_ram_counting.py
		- make clean
		- make -j CLANG=1
		- python3 scripts/self_test.py --create \| tee test_output.txt
		@@ -288,6 +291,7 @@ asan-on-merge-request-linux:
		needs: ["build-codec-sanitizers-linux"]
		script:
		- *print-common-info
		- python3 ci/disable_ram_counting.py
		- make clean
		- make -j CLANG=2
		- python3 scripts/self_test.py --create \| tee test_output.txt
		@@ -309,7 +313,7 @@ external-renderer-pytest:
		stage: test
		script:
		- make -j IVAS_rend
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py
		- python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
		when: always
		@@ -331,7 +335,8 @@ external-renderer-cmake-asan-pytest:
		- python3 ci/disable_ram_counting.py
		- cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true
		- cmake --build cmake-build -- -j
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py
		- python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py

		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
		when: always
		@@ -353,7 +358,8 @@ external-renderer-cmake-msan-pytest:
		- python3 ci/disable_ram_counting.py
		- cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true
		- cmake --build cmake-build -- -j
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py
		- python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer.py

		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results"
		when: always
		@@ -377,7 +383,7 @@ external-renderer-pytest-on-merge-request:
		- *print-common-info

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[rend(erer)[ -]non[ -]*be\]") \|\| true
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[rend$erer$[ -]non[ -]*be\]") \|\| true
		# TODO: needs splitting the test between reference and cut generation
		#- ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[ref[ -]using[ -]main\]") \|\| true

		@@ -399,7 +405,7 @@ external-renderer-pytest-on-merge-request:

		# run test
		- exit_code=0
		- python3 -m pytest -q --log-level ERROR -n auto -rA --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py \|\| exit_code=$?
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check
		@@ -412,6 +418,7 @@ external-renderer-pytest-on-merge-request:
		when: always
		paths:
		- report-junit.xml
		- report.html
		expose_as: "pytest external renderer results"
		reports:
		junit:
		@@ -448,7 +455,7 @@ ivas-pytest-on-merge-request:

		### run pytest
		- exit_code=0
		- python3 -m pytest tests -v --junit-xml=report-junit.xml \|\| exit_code=$?
		- python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check
		@@ -461,6 +468,7 @@ ivas-pytest-on-merge-request:
		when: always
		paths:
		- report-junit.xml
		- report.html
		expose_as: "pytest ivas results"
		reports:
		junit:
		@@ -493,7 +501,7 @@ evs-pytest-on-merge-request:

		### run pytest for EVS cases
		- exit_code=0
		- python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml \|\| exit_code=$?
		- python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml \|\| exit_code=$?
		- zero_errors=$(cat report-junit-evs.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check
		@@ -506,11 +514,23 @@ evs-pytest-on-merge-request:
		when: always
		paths:
		- report-junit-evs.xml
		- report.html
		expose_as: "pytest evs results"
		reports:
		junit:
		- report-junit-evs.xml

		voip-be-on-merge-request:
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-merge-request
		stage: compare # Or should it be test? Comparison is done within one git revision
		needs: ["build-codec-linux-make", codec-smoke-test]
		timeout: "10 minutes"
		script:
		- *print-common-info
		- bash ci/ivas_voip_be_test.sh

		clang-format-check:
		extends:
		- .test-job-linux
		@@ -647,7 +667,7 @@ codec-comparison-on-main-push:

		### run pytest
		- exit_code=0
		- python3 -m pytest tests -v --junit-xml=report-junit.xml \|\| exit_code=$?
		- python3 -m pytest tests -v --html=report.html --self-contained-html --junit-xml=report-junit.xml \|\| exit_code=$?
		- if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true
		- if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
		@@ -660,6 +680,7 @@ codec-comparison-on-main-push:
		when: always
		paths:
		- report-junit.xml
		- report.html
		expose_as: "Results of comparison to previous merge commit"
		reports:
		junit: report-junit.xml
		@@ -863,12 +884,17 @@ coverage-test-on-main-scheduled:
		script:
		- *print-common-info
		- make GCOV=1 -j
		- cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run external renderer script
		- python3 tests/create_short_testvectors.py
		- python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest tests -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest tests/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- bash ci/smoke_test.sh coverage
		- python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py
		- bash ci/ivas_voip_be_test.sh coverage
		- lcov -c -d obj -o coverage.info
		- genhtml coverage.info -o coverage
		- commit_sha=$(git rev-parse HEAD)
		- genhtml coverage.info -o coverage -t "Coverage on main @ $commit_sha"
		artifacts:
		name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"
		when: always
		@@ -909,15 +935,13 @@ coverage-test-on-main-scheduled:
		- echo $log_files
		- ls wmops/logs
		- for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done
		- log_files_48=$(cat $public_dir/graphs_wmops_flc_48kHz.js \| grep logFile \| sed "s/.$wmops_newsletter_.\.csv$.*/\1/g")
		- echo $log_files_48
		- for f in $log_files_48; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done
		# copy index page blueprint
		- cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html
		# patch the format in the title
		- sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html
		# do separately here to avoid overwrite complaints by mv
		- mv -f ci/complexity_measurements/style.css ${public_dir}/
		- ls $public_dir

		.complexity-template:
		extends:
		@@ -925,7 +949,7 @@ coverage-test-on-main-scheduled:
		- .test-job-linux-needs-testv-dir
		tags:
		- test-complexity-measurement
		timeout: 3 hours
		timeout: 3 hours 30 minutes
		stage: test
		artifacts:
		name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"
		@@ -968,7 +992,7 @@ complexity-sba-hoa3-in-hoa3-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 3 hours
		start_in: 2 hours 30 minutes
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -984,7 +1008,7 @@ complexity-mc-in-7_1_4-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 6 hours
		start_in: 4 hours
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1000,7 +1024,7 @@ complexity-masa-in-7_1_4-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 9 hours
		start_in: 7 hours 30 minutes
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1016,7 +1040,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out:
		rules:
		- if: $MEASURE_COMPLEXITY_LINUX
		when: delayed
		start_in: 10 hours
		start_in: 8 hours 30 minutes
		script:
		- *print-common-info
		- *update-ltv-repo
		@@ -1030,55 +1054,15 @@ complexity-StereoDmxEVS-stereo-in-mono-out:
		# Other jobs
		# ---------------------------------------------------------------

		# job that sets up gitlab pages website
		pages:
		stage: deploy
		tags:
		- test-complexity-measurement
		- ivas-linux
		rules:
		# only run for pipelines that affect the data for the page
		- if: $MEASURE_COMPLEXITY_LINUX
		# TODO: add coverage job
		- if: $UPDATE_PAGES
		script:

		- API_URL_BASE=https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs
		- branch=$CI_COMMIT_REF_NAME

		- mkdir public
		# get artifacts for complexity jobs
		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-stereo-in-stereo-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output artifacts_comp_stereo.zip
		- cat artifacts_comp_stereo.zip
		- unzip -o artifacts_comp_stereo.zip
		- mv complexity-stereo-in-stereo-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-ism-in-binaural-out)
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output artifacts_comp_ism.zip
		- unzip -o artifacts_comp_ism.zip
		- mv complexity-ism-in-binaural-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-sba-hoa3-in-hoa3-out)
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output artifacts_comp_sba.zip
		- unzip -o artifacts_comp_sba.zip
		- mv complexity-sba-hoa3-in-hoa3-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-mc-in-7_1_4-out)
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output artifacts_comp_mc.zip
		- unzip -o artifacts_comp_mc.zip
		- mv complexity-mc-in-7_1_4-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-masa-in-7_1_4-out)
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output artifacts_comp_masa.zip
		- unzip -o artifacts_comp_masa.zip
		- mv complexity-masa-in-7_1_4-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-StereoDmxEVS-stereo-in-mono-out)
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output artifacts_comp_StereoDmxEVS.zip
		- unzip -o artifacts_comp_StereoDmxEVS.zip
		- mv complexity-StereoDmxEVS-stereo-in-mono-out-public ./public/

		- cp ci/index-pages.html public/index.html
		- python3 ci/setup_pages.py
		artifacts:
		paths:
		- public

CMakeLists.txt

+3 −0

Original line number	Diff line number	Diff line
		@@ -173,3 +173,6 @@ if(COPY_EXECUTABLES_FROM_BUILD_DIR)
		add_custom_command(TARGET IVAS_dec POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$<TARGET_FILE:IVAS_dec>" "${CMAKE_CURRENT_SOURCE_DIR}/")
		add_custom_command(TARGET IVAS_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$<TARGET_FILE:IVAS_rend>" "${CMAKE_CURRENT_SOURCE_DIR}/")
		endif()

		# Allow creating packages for CMake install
		install(TARGETS lib_enc lib_dec lib_rend lib_com lib_util ARCHIVE DESTINATION lib)

Workspace_msvc/lib_debug.vcxproj

+3 −7

Original line number	Diff line number	Diff line
		@@ -143,20 +143,16 @@
		</ItemDefinitionGroup>
		<ItemGroup>
		<ClCompile Include="..\lib_debug\debug.c" />
		<ClCompile Include="..\lib_debug\memory.c" />
		<ClCompile Include="..\lib_debug\mem_count.c" />
		<ClCompile Include="..\lib_debug\segsnr.c" />
		<ClCompile Include="..\lib_debug\snr.c" />
		<ClCompile Include="..\lib_debug\sba_debug.c" />
		<ClCompile Include="..\lib_debug\wmops.c" />
		<ClCompile Include="..\lib_debug\wmc_auto.c" />
		</ItemGroup>
		<ItemGroup>
		<ClInclude Include="..\lib_debug\debug.h" />
		<ClInclude Include="..\lib_debug\deb_out.h" />
		<ClInclude Include="..\lib_debug\errorhnd.h" />
		<ClInclude Include="..\lib_debug\mem_count.h" />
		<ClInclude Include="..\lib_debug\sba_debug.h" />
		<ClInclude Include="..\lib_debug\wmops.h" />
		<ClInclude Include="..\lib_debug\wmc_auto.h" />
		</ItemGroup>
		<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
		<ImportGroup Label="ExtensionTargets">

Workspace_msvc/lib_dec.vcxproj

+2 −1

Original line number	Diff line number	Diff line
		@@ -283,6 +283,7 @@
		<ClCompile Include="..\lib_dec\ivas_lfe_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_lfe_plc.c" />
		<ClCompile Include="..\lib_dec\ivas_masa_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_mcmasa_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_mct_core_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_mct_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_mct_dec_mct.c" />
		@@ -314,10 +315,10 @@
		<ClCompile Include="..\lib_dec\ivas_stereo_mdct_stereo_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_stereo_switching_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_stereo_td_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_stereo_td_low_rate_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_svd_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_tcx_core_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_td_decorr.c" />
		<ClCompile Include="..\lib_dec\ivas_td_low_rate_dec.c" />
		<ClCompile Include="..\lib_dec\ivas_vbap.c" />
		<ClCompile Include="..\lib_dec\jbm_jb4sb.c" />
		<ClCompile Include="..\lib_dec\jbm_jb4_circularbuffer.c" />