Add job for storing backup copy of logs, and way to inject backups (268db15d) · Commits · SA4 / Audio / IVAS BASOP

.gitlab-ci.yml

+49 −3

Original line number	Diff line number	Diff line
		@@ -32,6 +32,9 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs'
		variables:
		IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs'
		variables:
		IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'
		variables:
		IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'
		@@ -1659,19 +1662,24 @@ ivas-long-term-job-logs:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs"
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs"
		tags:
		- ivas-basop-linux
		- $RUNNER_TAG
		stage: maintenance
		timeout: "25 minutes"
		script:
		- !reference [ .job-linux, before_script ]
		- \|
		set -euxo pipefail
		echo "Running on RUNNER_TAG: $RUNNER_TAG"
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

		# Get previous logs of this job incase we are running on a different machine
		# Clean up logs
		if [ -d "logs" ]; then rm -rf "logs"; fi
		# TODO add variable to inject logs from backup copy

		# Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set
		if [ -d "$LOGS_BACKUP_SOURCE_DIR" ]; then
		cp -r $LOGS_BACKUP_SOURCE_DIR/logs .
		fi

		# handle case where the job is running for the first time. e.g where the job was not found.
		#id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)
		@@ -1719,6 +1727,44 @@ ivas-long-term-job-logs:
		expose_as: "ivas long term job logs results"


		backup-long-term-job-logs:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs"
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs"
		tags:
		- $RUNNER_TAG
		stage: maintenance
		timeout: "25 minutes"
		script:
		- !reference [ .job-linux, before_script ]
		- \|
		set -euxo pipefail
		echo "Running on RUNNER_TAG: $RUNNER_TAG"
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

		id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)
		echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"

		if [ "$id_previous" != "-1" ]; then
		# Unzip artifacts to recover past logs dir
		curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		unzip artifacts.zip
		else
		# create logs dir if it doesn't exist
		mkdir logs
		fi

		# Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set
		if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then
		cp -r logs $LOGS_BACKUP_TARGET_DIR/logs
		exit 0;
		else
		echo "Error: LOGS_BACKUP_TARGET_DIR not set for backup job!"
		exit -1;
		fi




		# ---------------------------------------------------------------

.gitlab-ci/variables.yml

+4 −0

Original line number	Diff line number	Diff line
		@@ -46,6 +46,9 @@ variables:
		FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"
		CUT_COMMIT_FILE: "CuT-git-sha.txt"
		MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"
		RUNNER_TAG: "ivas-basop-linux"
		LOGS_BACKUP_SOURCE_DIR: ""
		LOGS_BACKUP_TARGET_DIR: ""
		MANUAL_PIPELINE_TYPE:
		description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."
		value: 'default'
		@@ -66,3 +69,4 @@ variables:
		- 'renderer-framesize-be'
		- 'peaq-enc-passthrough'
		- 'long-term-logs'
		- 'backup-long-term-logs'