centralize build jobs (be6cadd6) · Commits · IVAS Codec Public Collaboration / IVAS Codec CI

includes/basop/default-variables-basop.yml

+0 −1

Original line number	Diff line number	Diff line
		@@ -58,7 +58,6 @@ variables:
		CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50"
		CI_REGRESSION_THRESH_SSNR: "-1"
		CI_REGRESSION_THRESH_ODG: "-0.05"
		INSTR_DIR: "scripts/c-code_instrument"
		BUILD_WITH_DEBUG_MODE_INFO: ""
		ENCODER_TEST: ""
		DELTA_ODG: ""

includes/shared/build-jobs.yml

+139 −0

Original line number	Diff line number	Diff line
		# ---------------------------------------------------------------
		# Build jobs
		# ---------------------------------------------------------------

		# ensure that codec builds on linux
		build-codec-linux-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES \|\| $MANUAL_PIPELINE_TYPE == 'complexity' \|\| $MEASURE_COMPLEXITY_LINUX
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- make -j

		# ensure that codec builds on linux with cmake
		build-codec-linux-cmake:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- \|
		if [ "$CI_PROJECT_ID" == "$PROJECT_ID_BASOP" ]; then
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		fi
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- cmake -B cmake-build -G "Unix Makefiles"
		- cmake --build cmake-build -- -j

		# ensure that codec builds on linux with instrumentation active
		build-codec-linux-instrumented-make:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $MEASURE_COMPLEXITY_LINUX \|\| $MANUAL_PIPELINE_TYPE == 'complexity'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "12 minutes"
		variables:
		INSTRUMENTATION_ARGS: ""
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- \|
		if [ "$CI_PROJECT_ID" == "$PROJECT_ID_BASOP" ]; then
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		INSTRUMENTATION_ARGS="-m MEM_ONLY"
		fi
		- bash scripts/prepare_instrumentation.sh $INSTRUMENTATION_ARGS
		- make -j -C $INSTR_DIR

		# make sure that the codec builds with msan, asan and usan
		build-codec-sanitizers-linux:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'sanitizer'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- \|
		if [ "$CI_PROJECT_ID" == "$PROJECT_ID_BASOP" ]; then
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		fi
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- bash ci/build_codec_sanitizers_linux.sh

		# ensure the codec builds with DEBUGGING activated
		# warnings are not treated as errors here
		build-codec-linux-debugging-make:
		@@ -17,3 +115,44 @@ build-codec-linux-debugging-make:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/enable-debugging-macro.sh
		- make -j

		build-codec-windows-msbuild:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES \|\| $MANUAL_PIPELINE_TYPE == 'complexity' \|\| $MEASURE_COMPLEXITY_LINUX
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-windows
		timeout: "10 minutes"
		tags:
		- ivas-windows
		script:
		- powershell $CI_PROJECT_DIR\snippets\print-common-info-windows.ps1
		- powershell $CI_PROJECT_DIR\snippets\activate-WX-windows.ps1
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug

		build-codec-windows-cmake:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES \|\| $MANUAL_PIPELINE_TYPE == 'complexity' \|\| $MEASURE_COMPLEXITY_LINUX
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-windows
		timeout: "10 minutes"
		tags:
		- ivas-windows
		script:
		- powershell $CI_PROJECT_DIR\snippets\print-common-info-windows.ps1
		- powershell $CI_PROJECT_DIR\snippets\activate-WX-windows.ps1
		- cmake -G "Visual Studio 15 2017" . -Bbuild
		- cmake --build build -j

includes/shared/default-variables.yml

+1 −0

Original line number	Diff line number	Diff line
		@@ -17,3 +17,4 @@ variables:
		DISABLE_HRTF: "false"
		PROJECT_ID_FLOAT: "49"
		PROJECT_ID_BASOP: "77"
		INSTR_DIR: "scripts/c-code_instrument"

main-basop.yml

+0 −123

Original line number	Diff line number	Diff line
		@@ -95,24 +95,6 @@ workflow:
		variables:
		IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'

		# ---------------------------------------------------------------
		# Generic script anchors
		# ---------------------------------------------------------------

		# These can be used later on to do common tasks

		# Prints useful information for every job and should be used at the beginning of each job
		.print-common-info-windows: &print-common-info-windows
		- \|
		echo "Printing common information for build job."
		echo "Current job is run on commit $CI_COMMIT_SHA"
		echo "Commit time was $CI_COMMIT_TIMESTAMP"
		("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") \| Invoke-Expression

		.activate-WX-windows: &activate-WX-windows
		- (Get-Content -Path "CMakeLists.txt") -replace '# $add_compile_options\("\/WX"$\)', '$1' \| Set-Content -Path "CMakeLists.txt"
		- Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" \| ForEach-Object { (Get-Content -Path $_.FullName) -replace '<TreatWarningAsError>false', '<TreatWarningAsError>true' \| Set-Content -Path $_.FullName }

		# ---------------------------------------------------------------
		# Job templates
		# ---------------------------------------------------------------
		@@ -803,111 +785,6 @@ pytest-to-ref-pre:
		- $FLOAT_REF_COMMIT_FILE
		reports:
		dotenv: commits.env
		# ---------------------------------------------------------------
		# Build jobs
		# ---------------------------------------------------------------

		# TODO: template rules for the build jobs
		# ensure that codec builds on linux
		build-codec-linux-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES \|\| $MANUAL_PIPELINE_TYPE == 'complexity' \|\| $MEASURE_COMPLEXITY_LINUX
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- make -j

		build-codec-linux-cmake:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- cmake -B cmake-build -G "Unix Makefiles"
		- cmake --build cmake-build -- -j

		# ensure that codec builds on linux with instrumentation active
		build-codec-linux-instrumented-make:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $MEASURE_COMPLEXITY_LINUX \|\| $MANUAL_PIPELINE_TYPE == 'complexity'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "12 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- bash scripts/prepare_instrumentation.sh -m MEM_ONLY
		- make -j -C $INSTR_DIR

		# make sure that the codec builds with msan, asan and usan
		build-codec-sanitizers-linux:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'sanitizer'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-linux
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- bash ci/build_codec_sanitizers_linux.sh

		build-codec-windows-msbuild:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES \|\| $MANUAL_PIPELINE_TYPE == 'complexity' \|\| $MEASURE_COMPLEXITY_LINUX
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		extends:
		- .build-job-windows
		timeout: "10 minutes"
		tags:
		- ivas-windows
		script:
		- *print-common-info-windows
		- *activate-WX-windows
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug

		# ---------------------------------------------------------------
		# Test jobs for merge requests

main-float.yml

+0 −83

Original line number	Diff line number	Diff line
		@@ -75,13 +75,6 @@ workflow:

		# These can be used later on to do common tasks

		.print-common-info-windows: &print-common-info-windows
		- \|
		echo "Printing common information for build job."
		echo "Current job is run on commit $CI_COMMIT_SHA"
		echo "Commit time was $CI_COMMIT_TIMESTAMP"
		("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") \| Invoke-Expression

		.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec
		### build test binaries, initial clean for paranoia reasons
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh
		@@ -140,10 +133,6 @@ workflow:
		- cp $LTV_DIR_WIN\*.met scripts\testv
		- cp $LTV_DIR_WIN\*.csv scripts\testv

		.activate-WX-windows: &activate-WX-windows
		- (Get-Content -Path "CMakeLists.txt") -replace '# $add_compile_options\("\/WX"$\)', '$1' \| Set-Content -Path "CMakeLists.txt"
		- Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" \| ForEach-Object { (Get-Content -Path $_.FullName) -replace '<TreatWarningAsError>false', '<TreatWarningAsError>true' \| Set-Content -Path $_.FullName }

		# to be reused in MR and LTV-scheduled sanitizer test jobs
		# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section
		.sanitizer-selftest-anchor:
		@@ -316,78 +305,6 @@ basop-compat-encoder-ltv:
		script:
		- exit 0


		# ---------------------------------------------------------------
		# Build jobs
		# ---------------------------------------------------------------

		build-codec-linux-make:
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- make -j

		build-codec-linux-cmake:
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- mkdir build
		- cd build
		- cmake ..
		- cd ..
		- make -C build -j

		build-codec-instrumented-linux:
		extends:
		- .build-job-linux
		- .rules-basis
		timeout: "10 minutes"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- ./scripts/prepare_instrumentation.sh
		- make -j -C scripts/c-code_instrument
		artifacts:
		when: always
		access: all
		expire_in: "30 days"
		paths:
		- wmc_tool_output.txt

		# make sure that the codec builds with msan, asan and usan
		build-codec-sanitizers-linux:
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/activate-Werror-linux.sh
		- bash ci/build_codec_sanitizers_linux.sh

		build-codec-windows-cmake:
		extends:
		- .build-job-windows
		- .rules-basis
		script:
		- *print-common-info-windows
		- *activate-WX-windows
		- cmake -G "Visual Studio 15 2017" . -Bbuild
		- cmake --build build -j

		build-codec-windows-msbuild:
		extends:
		- .build-job-windows
		- .rules-basis
		script:
		- *print-common-info-windows
		- *activate-WX-windows
		- MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug

		# ---------------------------------------------------------------
		# Test jobs for merge requests
		# ---------------------------------------------------------------