From c3fe889e83d2b0eb82f43be5f7951644c8643bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Mon, 13 Jan 2025 12:14:40 +0100 Subject: [PATCH 01/19] Port CI jobs relevant for split rendering --- .gitlab-ci.yml | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fe5ec338..c8df11fcb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -190,6 +190,17 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUG_MODE_INFO\ *\)\*\//\1/g" lib_com/options.h - fi +.mr-get-target-commit: &mr-get-target-commit # compare to last target branch commit before pipeline was created + - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) + +.get-commits-behind-count: &get-commits-behind-count + - echo $CI_COMMIT_SHA + - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) + +.check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs + - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run 'git merge origin/main' to update."; exit 1; fi + .build-reference-binaries: &build-reference-binaries - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries @@ -244,6 +255,13 @@ stages: - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + .update-ltv-repo: &update-ltv-repo - cd $LTV_DIR - git pull @@ -951,6 +969,93 @@ build-codec-linux-debugging-make: - *activate-debug-mode-info-if-set - make -j +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +lc3-wrapper-unit-test: + extends: + - .test-job-linux + - .rules-merge-request + needs: ["build-codec-linux-cmake"] + stage: test + script: + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test + +# compare split-rendering bitexactness between target and source branch +split-rendering-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "30 minutes" + stage: compare + script: + - *print-common-info + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true + - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + - *mr-fetch-target-branch + - *mr-get-target-commit + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + # build reference binaries + - make -j + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + + ### If ref_using_main is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + # back to source branch + - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts + - git checkout $source_branch_commit_sha + - make clean + - make -j + + ### Run test using scripts and input from main + - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest split rendering results" + reports: + junit: + - report-junit.xml + # --------------------------------------------------------------- # Short test jobs that run in merge request pipelines # --------------------------------------------------------------- -- GitLab From 00636b34a9113f33bc80e7d3870f6a46f1be969e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Mon, 13 Jan 2025 12:17:10 +0100 Subject: [PATCH 02/19] fixup! Port CI jobs relevant for split rendering --- .gitlab-ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8df11fcb..2b2a1ea0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -292,6 +292,44 @@ stages: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +# --------------------------------------------------------------- +# Job templates +# --------------------------------------------------------------- + +# When designing templates, try not to use too much inheritance and +# if multiple templates and extended on, remember that on conflict, +# latest overwrites the parameter. + +# templates for rules +.rules-basis: + rules: + - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) + when: never + - if: $CI_PIPELINE_SOURCE == 'trigger' # Don't run triggered pipeline by default + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-be-release' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'check-clipping' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + when: never + - when: on_success + +.rules-merge-request: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" -- GitLab From 3e5671a33bcf228367ee91768dac60def6d135c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Mon, 13 Jan 2025 12:18:40 +0100 Subject: [PATCH 03/19] fixup! fixup! Port CI jobs relevant for split rendering --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b2a1ea0a..1c5981540 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1015,11 +1015,10 @@ lc3-wrapper-unit-test: extends: - .test-job-linux - .rules-merge-request - needs: ["build-codec-linux-cmake"] + needs: ["build-codec-linux-make"] stage: test script: - - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - - cmake --build cmake-build -- -j + - make -j - scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test # compare split-rendering bitexactness between target and source branch -- GitLab From 5d8c9e1e2cdce9017a370415d3f8c4c44521389c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Thu, 16 Jan 2025 13:12:41 +0100 Subject: [PATCH 04/19] Change ref_using_main to ref_using_target --- .gitlab-ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c5981540..8099d6e68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1039,7 +1039,9 @@ split-rendering-pytest-on-merge-request: # write to temporary file as workaround for failures observed with piping echo - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true - - ref_using_main=$(grep -c --ignore-case "\[ref[ -]*using[ -]*main\]" tmp.txt) || true + - # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + - # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 # store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) @@ -1055,9 +1057,9 @@ split-rendering-pytest-on-merge-request: - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref - ### If ref_using_main is not set, checkout the source branch to use scripts and input from there - - if [ $ref_using_main == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi - exit_code=0 - testcase_timeout=60 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? @@ -1069,8 +1071,8 @@ split-rendering-pytest-on-merge-request: - make -j ### Run test using scripts and input from main - - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi + - if [ $ref_using_target == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? -- GitLab From 4e95e04085b8c7d386f13c6683a204db40e827b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=BCller?= Date: Thu, 16 Jan 2025 13:21:17 +0100 Subject: [PATCH 05/19] fixup! Change ref_using_main to ref_using_target --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8099d6e68..e00469dd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1039,8 +1039,8 @@ split-rendering-pytest-on-merge-request: # write to temporary file as workaround for failures observed with piping echo - echo $CI_MERGE_REQUEST_TITLE > tmp.txt - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true - - # TODO: ref_using_target comes from float repo, but does not apply here - disable for now - - # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true - ref_using_target=0 # store the current commit hash -- GitLab From c4eaca8a9e0a539c60478d4612477703f5e1cbe1 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Feb 2025 08:21:07 +0100 Subject: [PATCH 06/19] disable no-commits-for-draft-MRs rule --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e00469dd7..bde6d4017 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,6 +60,9 @@ workflow: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never + # do not spawn pipelines for Draft MRs, see https://docs.gitlab.com/ee/ci/yaml/workflow.html#skip-pipelines-for-draft-merge-requests + # - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ + # when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' -- GitLab From 719d24315bbebbf81cb78e5fbd0db4869460bcee Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Feb 2025 08:23:35 +0100 Subject: [PATCH 07/19] add split rendering smoke test --- .gitlab-ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bde6d4017..4dea76c7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1014,6 +1014,27 @@ build-codec-linux-debugging-make: # Test jobs for merge requests # --------------------------------------------------------------- +split-rendering-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - testcase_timeout=10 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "split rendering smoke results" + reports: + junit: + - report-junit.xml + lc3-wrapper-unit-test: extends: - .test-job-linux -- GitLab From cfcc9d4161bd3401fd85199f6b31b2fb8cee215f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Feb 2025 08:25:06 +0100 Subject: [PATCH 08/19] use cmake for building the lc3 unit test --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dea76c7e..a151f781b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1042,7 +1042,8 @@ lc3-wrapper-unit-test: needs: ["build-codec-linux-make"] stage: test script: - - make -j + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j - scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test # compare split-rendering bitexactness between target and source branch -- GitLab From e2119544aa151f2e8a2ee837c774c3ee2703eed9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 5 Feb 2025 08:28:46 +0100 Subject: [PATCH 09/19] add needed script anchors --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a151f781b..c2e74f6f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1021,6 +1021,9 @@ split-rendering-smoke-test: needs: ["build-codec-linux-make"] stage: test script: + - *print-common-info + - *update-scripts-repo + - make -j - testcase_timeout=10 - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout @@ -1042,6 +1045,9 @@ lc3-wrapper-unit-test: needs: ["build-codec-linux-make"] stage: test script: + - *print-common-info + - *update-scripts-repo + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test @@ -1057,6 +1063,7 @@ split-rendering-pytest-on-merge-request: stage: compare script: - *print-common-info + - *update-scripts-repo - *get-commits-behind-count - *check-commits-behind-count-in-compare-jobs -- GitLab From efe6375a9d6a5d50e2d671de4515515777ad3ac2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Feb 2025 09:57:18 +0100 Subject: [PATCH 10/19] add CMakeLists file --- CMakeLists.txt | 221 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..17799f0db --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,221 @@ +# CMake file for IVAS +# +# Usage with Unix Makefiles (Linux, OS/X): +# # create build directory +# mkdir build ; cd build +# # call CMake to generate build system, e.g. one of the following: +# cmake -D CMAKE_BUILD_TYPE=Debug ../ +# cmake -D CMAKE_BUILD_TYPE=Release ../ +# cmake -D CMAKE_BUILD_TYPE=Debug -D TARGET_PLATFORM=x86_64 ../ +# # build project +# make -j8 +# +# Usage with Visual Studio +# 1) download CMake from https://cmake.org/download/, don't use the Cygwin version! +# 2.1) build project using IDE +# In CMake GUI select the source dir (root of stereo-evs) and a new binary directory +# and press "Configure" and "Generate". Then open the Visual Studio solution file generated +# in the build directory. +# 2.2) build project using command line +# # create build directory +# mkdir build ; cd build +# # call CMake to generate build system, e.g. one of the following: +# cmake ../ +# cmake -G "Visual Studio 12 2013" ../ +# cmake -G "Visual Studio 12 2013 Win64" ../ +# # open the Visual Studio solution file generated in the build directory +# # or build on command line, e.g.: +# cmake --build . --config Debug +# cmake --build . --config Release + + +cmake_minimum_required(VERSION 3.1) + +set(CMAKE_C_STANDARD 99) + +# configuration options for UNIX +if(UNIX) + set(TARGET_PLATFORM "" CACHE STRING "i686 / x86_64") + set(CLANG "" CACHE STRING "1=msan / 2=asan / 3=usan") + set(GCOV OFF CACHE BOOL "enable GCOV") + set(STRIP OFF CACHE BOOL "enable STRIP") + + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "") + endif() + # TARGET_PLATFORM + if("${TARGET_PLATFORM}" MATCHES "i386" OR + "${TARGET_PLATFORM}" MATCHES "i586" OR + "${TARGET_PLATFORM}" MATCHES "i686" + ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32") + elseif("${TARGET_PLATFORM}" MATCHES "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m64") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m64") + endif() + # C compiler flags + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffp-contract=off") # disable floating point operation contraction + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pedantic -Wcast-qual -Wall -W -Wextra -Wno-long-long") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wpointer-arith -Wstrict-prototypes -Wmissing-prototypes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-parameter") + # to be uncommented in CI + # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + + # CLANG + if(CLANG) + find_program(clangBin NAMES /home/amm-archiv/soft/Linux/clang/current/bin/clang clang REQUIRED) + set(CMAKE_C_COMPILER "${clangBin}" CACHE STRING "") + if("${CLANG}" MATCHES "1" OR "${CLANG}" MATCHES "msan") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=memory") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") + elseif("${CLANG}" MATCHES "2" OR "${CLANG}" MATCHES "asan") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + elseif("${CLANG}" MATCHES "3" OR "${CLANG}" MATCHES "usan") + # NOTE: keep in sync with list in Makefile + set(USAN_CHECKS_ENABLE + undefined # Default checks + # Extra checks + float-divide-by-zero + implicit-conversion + local-bounds + ) + list(JOIN USAN_CHECKS_ENABLE "," USAN_CHECKS_ENABLE) + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}") + else() + message(FATAL_ERROR "Unknown CLANG setting: ${CLANG}") + endif() + endif() + # GCOV + if(GCOV) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage -fprofile-update=atomic") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage -fprofile-update=atomic") + endif() + # STRIP + if(STRIP) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdata-sections -ffunction-sections") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-gc-sections -static") + endif() + + message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") + message("CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}") + # write settings in CMake cache + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}" CACHE STRING "") + set(CMAKE_C_FLAGS_DEBUG "-O0 -g3" CACHE STRING "") + set(CMAKE_C_FLAGS_RELEASE "-O2 -DRELEASE" CACHE STRING "") # TODO should contain -DNDEBUG to disable assert() +elseif(WIN32) + # MSVC compiler flags + add_definitions( + -D_CRT_SECURE_NO_WARNINGS + /MP + ) +endif() + +# configuration options for all platforms +set(WMOPS OFF CACHE BOOL "enable WMOPS") +if(WMOPS) + add_definitions("-DWMOPS=1") +endif() + +project(stereo-evs) +set_property(GLOBAL PROPERTY USE_FOLDERS ON) # make Visual Studio projects look nicer +include(CTest) + +file(GLOB libComSrcs "lib_com/*.c") +file(GLOB libComHeaders "lib_com/*.h") +add_library(lib_com ${libComSrcs} ${libComHeaders}) +if(UNIX) + target_link_libraries(lib_com PRIVATE m) +endif() +target_include_directories(lib_com PUBLIC lib_com PRIVATE lib_enc lib_dec lib_rend lib_debug lib_isar) +target_include_directories(lib_com PRIVATE lib_lc3plus) + +file(GLOB libDebugSrcs "lib_debug/*.c") +file(GLOB libDebugHeaders "lib_debug/*.h") +add_library(lib_debug ${libDebugSrcs} ${libDebugHeaders}) +target_link_libraries(lib_debug lib_com) +target_include_directories(lib_debug PUBLIC lib_debug PRIVATE lib_enc lib_dec lib_rend lib_isar) + +file(GLOB libEncSrcs "lib_enc/*.c") +file(GLOB libEncHeaders "lib_enc/*.h") +add_library(lib_enc ${libEncSrcs} ${libEncHeaders}) +target_link_libraries(lib_enc lib_com lib_debug) +target_include_directories(lib_enc PUBLIC lib_enc PRIVATE lib_dec lib_rend lib_isar) +target_include_directories(lib_enc PRIVATE lib_lc3plus) + +file(GLOB libLC3plusSrcs "lib_lc3plus/*.c") +file(GLOB libLC3plusHeaders "lib_lc3plus/*.h") +add_library(lib_lc3plus ${libLC3plusSrcs} ${libLC3plusHeaders}) +target_include_directories(lib_lc3plus PUBLIC lib_lc3plus PRIVATE lib_com lib_debug) + +file(GLOB libRendSrcs "lib_rend/*.c") +file(GLOB libRendHeaders "lib_rend/*.h") + +add_library(lib_rend ${libRendSrcs} ${libRendHeaders}) +target_link_libraries(lib_rend lib_dec lib_com lib_debug) # Todo refactor: This dependency on lib_dec should be removed. +target_link_libraries(lib_rend lib_lc3plus lib_isar) +target_include_directories(lib_rend PUBLIC lib_rend PRIVATE lib_enc lib_isar) + + +file(GLOB libDecSrcs "lib_dec/*.c") +file(GLOB libDecHeaders "lib_dec/*.h") +add_library(lib_dec ${libDecSrcs} ${libDecHeaders}) +target_link_libraries(lib_dec lib_com lib_rend lib_debug lib_isar) +target_include_directories(lib_dec PUBLIC lib_dec lib_rend PRIVATE lib_enc lib_isar) + +file(GLOB libUtilSrcs "lib_util/*.c") +file(GLOB libUtilHeaders "lib_util/*.h") +add_library(lib_util ${libUtilSrcs} ${libUtilHeaders}) +target_include_directories(lib_util PUBLIC lib_util PRIVATE lib_com lib_enc lib_dec lib_rend lib_debug) +target_include_directories(lib_util PRIVATE lib_lc3plus lib_isar) + +if(NOT WMOPS) + add_executable(ivas_lc3plus_unit_test ${CMAKE_SOURCE_DIR}/scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test.c) + target_link_libraries(ivas_lc3plus_unit_test lib_rend lib_dec lib_util lib_com lib_debug lib_isar) +endif() + +file(GLOB libISARSrcs "lib_isar/*.c") +file(GLOB libISARHeaders "lib_isar/*.h") + +add_library(lib_isar ${libISARSrcs} ${libISARHeaders}) +target_link_libraries(lib_isar lib_com lib_debug lib_lc3plus) # Todo refactor: This dependency on lib_dec should be removed. +target_include_directories(lib_isar PUBLIC lib_isar PRIVATE lib_enc lib_dec lib_rend) + + +add_executable(IVAS_cod apps/encoder.c) +target_link_libraries(IVAS_cod lib_enc lib_util) +if(WIN32) + target_link_libraries(IVAS_cod Ws2_32) +endif() + +add_executable(IVAS_dec apps/decoder.c) +target_link_libraries(IVAS_dec lib_dec lib_util) +if(WIN32) + target_link_libraries(IVAS_dec Ws2_32) +endif() + +add_executable(IVAS_rend apps/renderer.c) +target_link_libraries(IVAS_rend lib_rend lib_util lib_isar) +target_include_directories(IVAS_rend PRIVATE lib_enc) + +add_executable(ISAR_post_rend apps/isar_post_rend.c) +target_link_libraries(ISAR_post_rend lib_isar lib_util) +target_include_directories(ISAR_post_rend PRIVATE lib_isar) + +if(COPY_EXECUTABLES_FROM_BUILD_DIR) + # Optionally copy executables to the same place where Make puts them (useful for tests that expect executables in specific places) + add_custom_command(TARGET IVAS_cod POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + add_custom_command(TARGET IVAS_dec POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + add_custom_command(TARGET IVAS_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + add_custom_command(TARGET ISAR_post_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + if (NOT WMOPS) + add_custom_command(TARGET ivas_lc3plus_unit_test POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/split_rendering/lc3plus") + endif() +endif() + +# Allow creating packages for CMake install +install(TARGETS lib_enc lib_dec lib_rend lib_com lib_util ARCHIVE DESTINATION lib) -- GitLab From 1a123fc971435e2fe4d8962871823377a3ef213f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Feb 2025 13:25:52 +0100 Subject: [PATCH 11/19] use new folder structure for lc3plus tests --- .gitlab-ci.yml | 2 +- CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b6bc7b1cc..11cf88e1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1050,7 +1050,7 @@ lc3-wrapper-unit-test: - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - cmake --build cmake-build -- -j - - scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test + - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test # compare split-rendering bitexactness between target and source branch split-rendering-pytest-on-merge-request: diff --git a/CMakeLists.txt b/CMakeLists.txt index 17799f0db..887bc6df0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -174,7 +174,7 @@ target_include_directories(lib_util PUBLIC lib_util PRIVATE lib_com lib_enc lib_ target_include_directories(lib_util PRIVATE lib_lc3plus lib_isar) if(NOT WMOPS) - add_executable(ivas_lc3plus_unit_test ${CMAKE_SOURCE_DIR}/scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test.c) + add_executable(ivas_lc3plus_unit_test ${CMAKE_SOURCE_DIR}/scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test.c) target_link_libraries(ivas_lc3plus_unit_test lib_rend lib_dec lib_util lib_com lib_debug lib_isar) endif() -- GitLab From 041a84a88bd0b6fb260f23366ad55072de5dba66 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 18 Feb 2025 14:11:47 +0100 Subject: [PATCH 12/19] fix copy command in CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 887bc6df0..adc86288c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -213,7 +213,7 @@ if(COPY_EXECUTABLES_FROM_BUILD_DIR) add_custom_command(TARGET IVAS_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") add_custom_command(TARGET ISAR_post_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") if (NOT WMOPS) - add_custom_command(TARGET ivas_lc3plus_unit_test POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/split_rendering/lc3plus") + add_custom_command(TARGET ivas_lc3plus_unit_test POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/split_rendering/lc3plus_basop") endif() endif() -- GitLab From 8295f90d72abf9233508fbe0b7e0e2b30ee5f8ca Mon Sep 17 00:00:00 2001 From: rtyag Date: Wed, 26 Feb 2025 12:41:24 +1100 Subject: [PATCH 13/19] add MLD to the SR pytest --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86ae985d1..995875be9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1133,7 +1133,7 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --mld --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check -- GitLab From b2b43289ffbd1613ba80803b771ec1ad15ab5c73 Mon Sep 17 00:00:00 2001 From: rtyag Date: Wed, 26 Feb 2025 13:25:40 +1100 Subject: [PATCH 14/19] revert -- add MLD to the SR pytest, need changes to split rendering pytest first --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 995875be9..86ae985d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1133,7 +1133,7 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --mld --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check -- GitLab From 17167fdb9d4c9e7460f5fabdbbbf7b3042ae5aaf Mon Sep 17 00:00:00 2001 From: rtyag Date: Mon, 3 Mar 2025 15:43:04 +1100 Subject: [PATCH 15/19] test MLD, SSNR and PEAQ with split rendering tests --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index faaf7fa54..b4fbe56eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" + BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch-sr-obj-scores" PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 @@ -1372,8 +1372,11 @@ split-rendering-pytest-on-merge-request: - if [ $ref_using_target == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + - comp_args="--mld --ssnr --odg" + - echo "$comp_args" + # run test - - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check -- GitLab From 8fdbf84cbcff08af6e4891b170a24ec1bea7a9ed Mon Sep 17 00:00:00 2001 From: rtyag Date: Wed, 5 Mar 2025 14:12:25 +1100 Subject: [PATCH 16/19] changing ci branch back to basop-ci-branch --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4fbe56eb..0ccb41bd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ variables: MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch-sr-obj-scores" + BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 -- GitLab From 8bc215286d3f77b06cba38066fd7f1f98c542f75 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Mar 2025 09:08:02 +0100 Subject: [PATCH 17/19] add cmak build job + cleanup --- .gitlab-ci.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ccb41bd9..988e9fbc5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,9 +79,6 @@ workflow: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - # do not spawn pipelines for Draft MRs, see https://docs.gitlab.com/ee/ci/yaml/workflow.html#skip-pipelines-for-draft-merge-requests - # - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/ - # when: never - if: $CI_PIPELINE_SOURCE == 'merge_request_event' variables: IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' @@ -319,6 +316,7 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile + - sed -i.bak "s/# \(set(CMAKE_C_FLAGS \"\${CMAKE_C_FLAGS} -Werror\")\)/\1/" CMakeLists.txt .activate-WX-windows: &activate-WX-windows - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" @@ -1204,6 +1202,24 @@ build-codec-linux-make: - *activate-Werror-linux - make -j +build-codec-linux-cmake: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + tags: + - ivas-basop-linux + script: + - *print-common-info + - *activate-Werror-linux + - cmake -B cmake-build -G "Unix Makefiles" + - cmake --build cmake-build -- -j + # ensure that codec builds on linux with instrumentation active build-codec-linux-instrumented-make: rules: -- GitLab From 02c899cd50ba4bcb1f9c88bc958940977e323cb8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Mar 2025 09:13:01 +0100 Subject: [PATCH 18/19] get script repo with lc3plus test sources in build job --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 988e9fbc5..26067a319 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1217,6 +1217,7 @@ build-codec-linux-cmake: script: - *print-common-info - *activate-Werror-linux + - *update-scripts-repo - cmake -B cmake-build -G "Unix Makefiles" - cmake --build cmake-build -- -j -- GitLab From 7ef3abfd3193aa49cc92dafa1e31d0fd82ba98b7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 6 Mar 2025 09:18:59 +0100 Subject: [PATCH 19/19] remove Werror again --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26067a319..025e865e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1216,7 +1216,6 @@ build-codec-linux-cmake: - ivas-basop-linux script: - *print-common-info - - *activate-Werror-linux - *update-scripts-repo - cmake -B cmake-build -G "Unix Makefiles" - cmake --build cmake-build -- -j -- GitLab