diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dde066dc2e84ac924b436af72873ce35369e9853..025e865e47a1e1438865c32ee68846bbe2593a08 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -284,6 +284,13 @@ stages: - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + .update-ltv-repo: &update-ltv-repo - cd $LTV_DIR - git pull @@ -309,17 +316,49 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile + - sed -i.bak "s/# \(set(CMAKE_C_FLAGS \"\${CMAKE_C_FLAGS} -Werror\")\)/\1/" CMakeLists.txt .activate-WX-windows: &activate-WX-windows - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +# --------------------------------------------------------------- +# Job templates +# --------------------------------------------------------------- + +# When designing templates, try not to use too much inheritance and +# if multiple templates and extended on, remember that on conflict, +# latest overwrites the parameter. + +# templates for rules +.rules-basis: + rules: + - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) + when: never + - if: $CI_PIPELINE_SOURCE == 'trigger' # Don't run triggered pipeline by default + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-be-release' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'check-clipping' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + when: never + - when: on_success + .rules-merge-request: + extends: .rules-basis rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never - - when: never .rules-pytest-to-ref-short: rules: @@ -1163,6 +1202,24 @@ build-codec-linux-make: - *activate-Werror-linux - make -j +build-codec-linux-cmake: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-linux + tags: + - ivas-basop-linux + script: + - *print-common-info + - *update-scripts-repo + - cmake -B cmake-build -G "Unix Makefiles" + - cmake --build cmake-build -- -j + # ensure that codec builds on linux with instrumentation active build-codec-linux-instrumented-make: rules: @@ -1235,6 +1292,126 @@ build-codec-linux-debugging-make: - *activate-debug-mode-info-if-set - make -j +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +split-rendering-smoke-test: + extends: + - .test-job-linux + - .rules-merge-request + needs: ["build-codec-linux-make"] + stage: test + script: + - *print-common-info + - *update-scripts-repo + + - make -j + - testcase_timeout=10 + - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + expose_as: "split rendering smoke results" + reports: + junit: + - report-junit.xml + +lc3-wrapper-unit-test: + extends: + - .test-job-linux + - .rules-merge-request + needs: ["build-codec-linux-make"] + stage: test + script: + - *print-common-info + - *update-scripts-repo + + - cmake -B cmake-build -G "Unix Makefiles" -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true + - cmake --build cmake-build -- -j + - scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test + +# compare split-rendering bitexactness between target and source branch +split-rendering-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "30 minutes" + stage: compare + script: + - *print-common-info + - *update-scripts-repo + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + - *mr-fetch-target-branch + - *mr-get-target-commit + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + # build reference binaries + - make -j + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + # back to source branch + - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts + - git checkout $source_branch_commit_sha + - make clean + - make -j + + ### Run test using scripts and input from main + - if [ $ref_using_target == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + - comp_args="--mld --ssnr --odg" + - echo "$comp_args" + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest split rendering results" + reports: + junit: + - report-junit.xml + # --------------------------------------------------------------- # Short test jobs that run in merge request pipelines # --------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..adc86288c6d2062591090decdb6954098185c923 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,221 @@ +# CMake file for IVAS +# +# Usage with Unix Makefiles (Linux, OS/X): +# # create build directory +# mkdir build ; cd build +# # call CMake to generate build system, e.g. one of the following: +# cmake -D CMAKE_BUILD_TYPE=Debug ../ +# cmake -D CMAKE_BUILD_TYPE=Release ../ +# cmake -D CMAKE_BUILD_TYPE=Debug -D TARGET_PLATFORM=x86_64 ../ +# # build project +# make -j8 +# +# Usage with Visual Studio +# 1) download CMake from https://cmake.org/download/, don't use the Cygwin version! +# 2.1) build project using IDE +# In CMake GUI select the source dir (root of stereo-evs) and a new binary directory +# and press "Configure" and "Generate". Then open the Visual Studio solution file generated +# in the build directory. +# 2.2) build project using command line +# # create build directory +# mkdir build ; cd build +# # call CMake to generate build system, e.g. one of the following: +# cmake ../ +# cmake -G "Visual Studio 12 2013" ../ +# cmake -G "Visual Studio 12 2013 Win64" ../ +# # open the Visual Studio solution file generated in the build directory +# # or build on command line, e.g.: +# cmake --build . --config Debug +# cmake --build . --config Release + + +cmake_minimum_required(VERSION 3.1) + +set(CMAKE_C_STANDARD 99) + +# configuration options for UNIX +if(UNIX) + set(TARGET_PLATFORM "" CACHE STRING "i686 / x86_64") + set(CLANG "" CACHE STRING "1=msan / 2=asan / 3=usan") + set(GCOV OFF CACHE BOOL "enable GCOV") + set(STRIP OFF CACHE BOOL "enable STRIP") + + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "") + endif() + # TARGET_PLATFORM + if("${TARGET_PLATFORM}" MATCHES "i386" OR + "${TARGET_PLATFORM}" MATCHES "i586" OR + "${TARGET_PLATFORM}" MATCHES "i686" + ) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32") + elseif("${TARGET_PLATFORM}" MATCHES "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m64") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m64") + endif() + # C compiler flags + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ffp-contract=off") # disable floating point operation contraction + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pedantic -Wcast-qual -Wall -W -Wextra -Wno-long-long") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wpointer-arith -Wstrict-prototypes -Wmissing-prototypes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-parameter") + # to be uncommented in CI + # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") + + # CLANG + if(CLANG) + find_program(clangBin NAMES /home/amm-archiv/soft/Linux/clang/current/bin/clang clang REQUIRED) + set(CMAKE_C_COMPILER "${clangBin}" CACHE STRING "") + if("${CLANG}" MATCHES "1" OR "${CLANG}" MATCHES "msan") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=memory") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") + elseif("${CLANG}" MATCHES "2" OR "${CLANG}" MATCHES "asan") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + elseif("${CLANG}" MATCHES "3" OR "${CLANG}" MATCHES "usan") + # NOTE: keep in sync with list in Makefile + set(USAN_CHECKS_ENABLE + undefined # Default checks + # Extra checks + float-divide-by-zero + implicit-conversion + local-bounds + ) + list(JOIN USAN_CHECKS_ENABLE "," USAN_CHECKS_ENABLE) + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=${USAN_CHECKS_ENABLE} -fsanitize-recover=${USAN_CHECKS_ENABLE}") + else() + message(FATAL_ERROR "Unknown CLANG setting: ${CLANG}") + endif() + endif() + # GCOV + if(GCOV) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage -fprofile-update=atomic") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage -fprofile-update=atomic") + endif() + # STRIP + if(STRIP) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdata-sections -ffunction-sections") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-gc-sections -static") + endif() + + message("CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") + message("CMAKE_EXE_LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}") + # write settings in CMake cache + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}" CACHE STRING "") + set(CMAKE_C_FLAGS_DEBUG "-O0 -g3" CACHE STRING "") + set(CMAKE_C_FLAGS_RELEASE "-O2 -DRELEASE" CACHE STRING "") # TODO should contain -DNDEBUG to disable assert() +elseif(WIN32) + # MSVC compiler flags + add_definitions( + -D_CRT_SECURE_NO_WARNINGS + /MP + ) +endif() + +# configuration options for all platforms +set(WMOPS OFF CACHE BOOL "enable WMOPS") +if(WMOPS) + add_definitions("-DWMOPS=1") +endif() + +project(stereo-evs) +set_property(GLOBAL PROPERTY USE_FOLDERS ON) # make Visual Studio projects look nicer +include(CTest) + +file(GLOB libComSrcs "lib_com/*.c") +file(GLOB libComHeaders "lib_com/*.h") +add_library(lib_com ${libComSrcs} ${libComHeaders}) +if(UNIX) + target_link_libraries(lib_com PRIVATE m) +endif() +target_include_directories(lib_com PUBLIC lib_com PRIVATE lib_enc lib_dec lib_rend lib_debug lib_isar) +target_include_directories(lib_com PRIVATE lib_lc3plus) + +file(GLOB libDebugSrcs "lib_debug/*.c") +file(GLOB libDebugHeaders "lib_debug/*.h") +add_library(lib_debug ${libDebugSrcs} ${libDebugHeaders}) +target_link_libraries(lib_debug lib_com) +target_include_directories(lib_debug PUBLIC lib_debug PRIVATE lib_enc lib_dec lib_rend lib_isar) + +file(GLOB libEncSrcs "lib_enc/*.c") +file(GLOB libEncHeaders "lib_enc/*.h") +add_library(lib_enc ${libEncSrcs} ${libEncHeaders}) +target_link_libraries(lib_enc lib_com lib_debug) +target_include_directories(lib_enc PUBLIC lib_enc PRIVATE lib_dec lib_rend lib_isar) +target_include_directories(lib_enc PRIVATE lib_lc3plus) + +file(GLOB libLC3plusSrcs "lib_lc3plus/*.c") +file(GLOB libLC3plusHeaders "lib_lc3plus/*.h") +add_library(lib_lc3plus ${libLC3plusSrcs} ${libLC3plusHeaders}) +target_include_directories(lib_lc3plus PUBLIC lib_lc3plus PRIVATE lib_com lib_debug) + +file(GLOB libRendSrcs "lib_rend/*.c") +file(GLOB libRendHeaders "lib_rend/*.h") + +add_library(lib_rend ${libRendSrcs} ${libRendHeaders}) +target_link_libraries(lib_rend lib_dec lib_com lib_debug) # Todo refactor: This dependency on lib_dec should be removed. +target_link_libraries(lib_rend lib_lc3plus lib_isar) +target_include_directories(lib_rend PUBLIC lib_rend PRIVATE lib_enc lib_isar) + + +file(GLOB libDecSrcs "lib_dec/*.c") +file(GLOB libDecHeaders "lib_dec/*.h") +add_library(lib_dec ${libDecSrcs} ${libDecHeaders}) +target_link_libraries(lib_dec lib_com lib_rend lib_debug lib_isar) +target_include_directories(lib_dec PUBLIC lib_dec lib_rend PRIVATE lib_enc lib_isar) + +file(GLOB libUtilSrcs "lib_util/*.c") +file(GLOB libUtilHeaders "lib_util/*.h") +add_library(lib_util ${libUtilSrcs} ${libUtilHeaders}) +target_include_directories(lib_util PUBLIC lib_util PRIVATE lib_com lib_enc lib_dec lib_rend lib_debug) +target_include_directories(lib_util PRIVATE lib_lc3plus lib_isar) + +if(NOT WMOPS) + add_executable(ivas_lc3plus_unit_test ${CMAKE_SOURCE_DIR}/scripts/split_rendering/lc3plus_basop/ivas_lc3plus_unit_test.c) + target_link_libraries(ivas_lc3plus_unit_test lib_rend lib_dec lib_util lib_com lib_debug lib_isar) +endif() + +file(GLOB libISARSrcs "lib_isar/*.c") +file(GLOB libISARHeaders "lib_isar/*.h") + +add_library(lib_isar ${libISARSrcs} ${libISARHeaders}) +target_link_libraries(lib_isar lib_com lib_debug lib_lc3plus) # Todo refactor: This dependency on lib_dec should be removed. +target_include_directories(lib_isar PUBLIC lib_isar PRIVATE lib_enc lib_dec lib_rend) + + +add_executable(IVAS_cod apps/encoder.c) +target_link_libraries(IVAS_cod lib_enc lib_util) +if(WIN32) + target_link_libraries(IVAS_cod Ws2_32) +endif() + +add_executable(IVAS_dec apps/decoder.c) +target_link_libraries(IVAS_dec lib_dec lib_util) +if(WIN32) + target_link_libraries(IVAS_dec Ws2_32) +endif() + +add_executable(IVAS_rend apps/renderer.c) +target_link_libraries(IVAS_rend lib_rend lib_util lib_isar) +target_include_directories(IVAS_rend PRIVATE lib_enc) + +add_executable(ISAR_post_rend apps/isar_post_rend.c) +target_link_libraries(ISAR_post_rend lib_isar lib_util) +target_include_directories(ISAR_post_rend PRIVATE lib_isar) + +if(COPY_EXECUTABLES_FROM_BUILD_DIR) + # Optionally copy executables to the same place where Make puts them (useful for tests that expect executables in specific places) + add_custom_command(TARGET IVAS_cod POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + add_custom_command(TARGET IVAS_dec POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + add_custom_command(TARGET IVAS_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + add_custom_command(TARGET ISAR_post_rend POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/") + if (NOT WMOPS) + add_custom_command(TARGET ivas_lc3plus_unit_test POST_BUILD VERBATIM COMMAND "${CMAKE_COMMAND}" -E copy "$" "${CMAKE_CURRENT_SOURCE_DIR}/scripts/split_rendering/lc3plus_basop") + endif() +endif() + +# Allow creating packages for CMake install +install(TARGETS lib_enc lib_dec lib_rend lib_com lib_util ARCHIVE DESTINATION lib)