Merge branch 'main' into 250-tb-bwe-state-memories-reset-simplification (c40e254a) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+1 −60

Original line number	Diff line number	Diff line
		@@ -1080,74 +1080,15 @@ complexity-StereoDmxEVS-stereo-in-mono-out:
		# Other jobs
		# ---------------------------------------------------------------


		# helper for pages job
		.unzip-or-cat: &unzip-or-cat
		- unzip -t $ARTIFACTS >> /dev/null
		- if [ $? -eq 0 ]; then unzip -o $ARTIFACTS; rm $ARTIFACTS; else cat $ARTIFACTS; rm $ARTIFACTS; fi

		# job that sets up gitlab pages website
		# is run on a separate schedule and collects artifacts from other jobs (currently
		# only the complexity measurements) multiple times a day
		pages:
		stage: deploy
		tags:
		- ivas-linux
		rules:
		- if: $UPDATE_PAGES
		# TODO: add coverage job
		script:

		- API_URL_BASE=https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs
		- branch=$CI_COMMIT_REF_NAME

		- mkdir public
		- ARTIFACTS=artifacts.zip

		### fetch artifacts from latest run of complexity jobs
		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-stereo-in-stereo-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output $ARTIFACTS
		- *unzip-or-cat
		- mv complexity-stereo-in-stereo-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-ism-in-binaural-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output $ARTIFACTS
		- *unzip-or-cat
		- mv complexity-ism-in-binaural-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-sba-hoa3-in-hoa3-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output $ARTIFACTS
		- *unzip-or-cat
		- mv complexity-sba-hoa3-in-hoa3-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-mc-in-7_1_4-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output $ARTIFACTS
		- *unzip-or-cat
		- mv complexity-mc-in-7_1_4-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-masa-in-7_1_4-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output $ARTIFACTS
		- *unzip-or-cat
		- mv complexity-masa-in-7_1_4-out-public ./public/

		- job_id=$(python3 ci/get_id_of_last_job_occurence.py $branch complexity-StereoDmxEVS-stereo-in-mono-out)
		- echo $job_id
		- echo "$API_URL_BASE/$job_id/artifacts"
		- curl --request GET "$API_URL_BASE/$job_id/artifacts" --output $ARTIFACTS
		- *unzip-or-cat
		- mv complexity-StereoDmxEVS-stereo-in-mono-out-public ./public/

		- cp ci/index-pages.html public/index.html
		- python3 ci/setup_pages.py
		artifacts:
		paths:
		- public

ci/get_id_of_last_job_occurence.py

+39 −38

Original line number	Diff line number	Diff line
		@@ -38,16 +38,7 @@ PAGE_SUFFIX = "&page={}"
		API_BASE_URL = "https://forge.3gpp.org/rep/api/v4/projects/49"


		parser = argparse.ArgumentParser()
		parser.add_argument("branch_name")
		parser.add_argument("job_name")

		args = parser.parse_args()

		branch_name = args.branch_name
		job_name = args.job_name


		def get_job_id(branch_name, job_name):
		job_id = -1
		# check last 500 pipelines max
		for page in range(100):
		@@ -78,4 +69,14 @@ for page in range(100):
		if job_id >= 0:
		break

		return job_id

		if __name__ == '__main__':
		parser = argparse.ArgumentParser()
		parser.add_argument("branch_name")
		parser.add_argument("job_name")

		args = parser.parse_args()

		job_id = get_job_id(args.branch_name, args.job_name)
		print(job_id)
		No newline at end of file

ci/index-pages.html

+6 −0

Original line number	Diff line number	Diff line
		@@ -16,4 +16,10 @@
		<li><a href="complexity-StereoDmxEVS-stereo-in-mono-out-public/index.html">StereoDmxEVS</a></li>
		</ul>

		<h2>Test Coverage</h2>

		<ul>
		<li><a href="coverage/index.html">Coverage report</a></li>
		</ul>

		</body>

ci/setup_pages.py

0 → 100755

+70 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		import os
		import sys
		import pathlib
		import subprocess
		from .get_id_of_last_job_occurence import get_job_id

		JOBS = [
		"complexity-stereo-in-stereo-out", "complexity-ism-in-binaural-out", "complexity-sba-hoa3-in-hoa3-out", "complexity-mc-in-7_1_4-out", "complexity-masa-in-7_1_4-out", "complexity-StereoDmxEVS-stereo-in-mono-out", "coverage-test-on-main-scheduled"
		]
		ARTIFACTS = "artifacts.zip"
		API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs"
		PUBLIC = "./public"


		def main():

		public_folder = pathlib.Path(PUBLIC)
		public_folder.mkdir()

		failed_count = 0
		for job in JOBS:
		job_id = get_job_id(os.environ["CI_COMMIT_REF_NAME"], job)
		try:
		curl_for_artifacts(job_id)

		job_public = job + "-public"
		if job == "coverage-test-on-main-scheduled":
		job_public = "coverage"

		pathlib.Path(job_public).rename(public_folder.joinpath(job_public))

		except subprocess.CalledProcessError:
		print(f"Could not get artifacts for {job}")
		failed_count += 1

		if failed_count == len(JOBS):
		sys.exit(1)

		pathlib.Path("ci/index-pages.html").rename(public_folder.joinpath("index.html"))
		sys.exit(0)


		def curl_for_artifacts(job_id):
		cmd = [
		"curl",
		"--request",
		"GET",
		API_URL_BASE + f"/{job_id}/artifacts",
		"--output",
		ARTIFACTS
		]
		subprocess.run(cmd, check=True)

		# check for valid archive (if not, it is likely a 404 page, then display that)
		cmd = [
		"unzip",
		"-t",
		ARTIFACTS
		]
		try:
		subprocess.run(cmd, check=True)
		except subprocess.CalledProcessError:
		with open(ARTIFACTS, "r") as f:
		print(f.read())
		raise subprocess.CalledProcessError("Unzip check failed")


		if __name__ == "__main__":
		main()
		No newline at end of file