From 2925ab69ebb9d5d2029356ac29bcbf753b0613ff Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 22 May 2024 15:11:17 +0200 Subject: [PATCH 01/52] allow for project id to be passed in to get_id_of_last_job_occurence.py --- .gitlab-ci.yml | 2 +- ci/get_id_of_last_job_occurence.py | 66 +++++++++++++++--------------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0efa75256..c07e7a785a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2289,7 +2289,7 @@ coverage-test-on-main-scheduled: &complexity-measurements-setup # create necessary environment - mkdir -p wmops/logs - - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME) + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 449902f50a..01ccbc9384 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -1,49 +1,50 @@ #!/usr/bin/env python3 """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. +(C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository. All Rights Reserved. + +This software is protected by copyright law and by international treaties. +The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +contributors to this repository retain full ownership rights in their respective contributions in +the software. This notice grants no license of any kind, including but not limited to patent +license, nor is any license granted by implication, estoppel or otherwise. + +Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +contributions. + +This software is provided "AS IS", without any express or implied warranties. The software is in the +development stage. It is intended exclusively for experts who have experience with such software and +solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +and fitness for a particular purpose are hereby disclaimed and excluded. + +Any dispute, controversy or claim arising under or in relation to providing this software shall be +submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +the United Nations Convention on Contracts on the International Sales of Goods. """ import argparse import requests + PER_PAGE_SUFFIX = "?per_page=50" PAGE_SUFFIX = "&page={}" -API_BASE_URL = "https://forge.3gpp.org/rep/api/v4/projects/49" +API_URL_TMPL = "https://forge.3gpp.org/rep/api/v4/projects/{}/pipelines" -def get_job_id(branch_name, job_name): +def get_job_id(branch_name, job_name, project_id): job_id = -1 # check last 500 pipelines max for page in range(100): - url_pls = API_BASE_URL + "/pipelines" + url_pls = API_URL_TMPL.format(project_id) # need both suffixes here to descend through the pages and get also older pipelines suffix = PER_PAGE_SUFFIX + PAGE_SUFFIX.format(page) @@ -75,8 +76,9 @@ def get_job_id(branch_name, job_name): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("branch_name") - parser.add_argument("job_name") + parser.add_argument("branch_name", help="Name of the branch to search on") + parser.add_argument("job_name", help="Name of the job to get the id of") + parser.add_argument("project_id", help="ID of projectto search in", type=int) args = parser.parse_args() -- GitLab From e40af98b1f774fe0b3c780e24763a79c5ee23482 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 22 May 2024 15:49:39 +0200 Subject: [PATCH 02/52] use new clargument in function call --- ci/get_id_of_last_job_occurence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 01ccbc9384..7aec4017d3 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -82,5 +82,5 @@ if __name__ == "__main__": args = parser.parse_args() - job_id = get_job_id(args.branch_name, args.job_name) + job_id = get_job_id(args.branch_name, args.job_name, args.project_id) print(job_id) -- GitLab From f120f5d4d767c4be996625268b0c6bd563642929 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 22 May 2024 17:12:56 +0200 Subject: [PATCH 03/52] add some printouts for debugging in the CI --- ci/get_id_of_last_job_occurence.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 7aec4017d3..03d789471d 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -41,6 +41,7 @@ API_URL_TMPL = "https://forge.3gpp.org/rep/api/v4/projects/{}/pipelines" def get_job_id(branch_name, job_name, project_id): + print(branch_name, job_name, project_id) job_id = -1 # check last 500 pipelines max for page in range(100): @@ -62,6 +63,7 @@ def get_job_id(branch_name, job_name, project_id): # find actual job by name for job in resp_jobs.json(): + print(job["name"], job["status"]) if job["name"] == job_name and job["status"] == "success": job_id = job["id"] break -- GitLab From 12a31df1b608ddd3f732353004b4d142c90865bf Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 23 May 2024 11:42:05 +0900 Subject: [PATCH 04/52] Fix missing space --- ci/get_id_of_last_job_occurence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 03d789471d..3b1872b3d7 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -80,7 +80,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("branch_name", help="Name of the branch to search on") parser.add_argument("job_name", help="Name of the job to get the id of") - parser.add_argument("project_id", help="ID of projectto search in", type=int) + parser.add_argument("project_id", help="ID of project to search in", type=int) args = parser.parse_args() -- GitLab From cfd61a90c4a0692a2039621318c9bd26e256fa75 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 09:04:32 +0200 Subject: [PATCH 05/52] add flag --success_only --- .gitlab-ci.yml | 2 +- ci/get_id_of_last_job_occurence.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c07e7a785a..9292c4b9c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2289,7 +2289,7 @@ coverage-test-on-main-scheduled: &complexity-measurements-setup # create necessary environment - mkdir -p wmops/logs - - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID --success_only) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 3b1872b3d7..50beac5bb7 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -40,8 +40,7 @@ PAGE_SUFFIX = "&page={}" API_URL_TMPL = "https://forge.3gpp.org/rep/api/v4/projects/{}/pipelines" -def get_job_id(branch_name, job_name, project_id): - print(branch_name, job_name, project_id) +def get_job_id(branch_name, job_name, project_id, success_only): job_id = -1 # check last 500 pipelines max for page in range(100): @@ -63,8 +62,8 @@ def get_job_id(branch_name, job_name, project_id): # find actual job by name for job in resp_jobs.json(): - print(job["name"], job["status"]) - if job["name"] == job_name and job["status"] == "success": + include_job = not success_only or job["status"] == "success" + if include_job and job["name"] == job_name: job_id = job["id"] break if job_id >= 0: @@ -81,8 +80,9 @@ if __name__ == "__main__": parser.add_argument("branch_name", help="Name of the branch to search on") parser.add_argument("job_name", help="Name of the job to get the id of") parser.add_argument("project_id", help="ID of project to search in", type=int) + parser.add_argument("--success_only", help="Only include jobs with status 'success'", action="store_true") args = parser.parse_args() - job_id = get_job_id(args.branch_name, args.job_name, args.project_id) + job_id = get_job_id(args.branch_name, args.job_name, args.project_id, args.success_only) print(job_id) -- GitLab From 0946984b5676cd64893242a77cd4d48b61749d77 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 10:15:02 +0200 Subject: [PATCH 06/52] factor out artifact collection --- ci/setup_pages.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 10a2e9e84e..4f4f2cfe95 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -15,6 +15,9 @@ JOBS = [ "complexity-StereoDmxEVS-stereo-in-mono-out", "coverage-test-on-main-scheduled", ] +JOBS_BASOP = [ + + ] ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" PUBLIC = "./public" @@ -25,8 +28,19 @@ def main(): public_folder = pathlib.Path(PUBLIC) public_folder.mkdir() + failed_count = get_artifacts_for_jobs_and_return_num_failed(JOBS, public_folder) + + if failed_count == len(JOBS): + sys.exit(1) + + pathlib.Path("ci/index-pages.html").rename(public_folder.joinpath("index.html")) + sys.exit(0) + + +def get_artifacts_for_jobs_and_return_num_failed(jobs: list, public_folder: pathlib.Path) -> int: failed_count = 0 - for job in JOBS: + + for job in jobs: job_id = get_job_id(os.environ["CI_COMMIT_REF_NAME"], job) print(f"{job_id} - {job}") try: @@ -45,11 +59,7 @@ def main(): print(f"Could not get artifacts for {job}") failed_count += 1 - if failed_count == len(JOBS): - sys.exit(1) - - pathlib.Path("ci/index-pages.html").rename(public_folder.joinpath("index.html")) - sys.exit(0) + return failed_count def curl_for_artifacts(job_id): -- GitLab From f43bf021bce8ef6cf9a5f5dfc9542070bfd46458 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 10:45:00 +0200 Subject: [PATCH 07/52] make artifact collection more generic --- ci/setup_pages.py | 56 ++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 4f4f2cfe95..c87bb95648 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -6,54 +6,60 @@ import sys from get_id_of_last_job_occurence import get_job_id -JOBS = [ - "complexity-stereo-in-stereo-out", - "complexity-ism-in-binaural-out", - "complexity-sba-hoa3-in-hoa3-out", - "complexity-mc-in-7_1_4-out", - "complexity-masa-in-7_1_4-out", - "complexity-StereoDmxEVS-stereo-in-mono-out", - "coverage-test-on-main-scheduled", -] -JOBS_BASOP = [ - - ] +PUBLIC_FOL_MAGIC = "-public" + +JOBS_FLOAT_REPO = { + "complexity-stereo-in-stereo-out": [PUBLIC_FOL_MAGIC], + "complexity-ism-in-binaural-out": [PUBLIC_FOL_MAGIC], + "complexity-sba-hoa3-in-hoa3-out": [PUBLIC_FOL_MAGIC], + "complexity-mc-in-7_1_4-out": [PUBLIC_FOL_MAGIC], + "complexity-masa-in-7_1_4-out": [PUBLIC_FOL_MAGIC], + "complexity-StereoDmxEVS-stereo-in-mono-out": [PUBLIC_FOL_MAGIC], + "coverage-test-on-main-scheduled": ["coverage", "coverage_stv"], +} ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" PUBLIC = "./public" def main(): - public_folder = pathlib.Path(PUBLIC) public_folder.mkdir() - failed_count = get_artifacts_for_jobs_and_return_num_failed(JOBS, public_folder) + failed_count = get_artifacts_for_jobs_and_return_num_failed( + JOBS_FLOAT_REPO, public_folder + ) - if failed_count == len(JOBS): + if failed_count == len(JOBS_FLOAT_REPO): sys.exit(1) pathlib.Path("ci/index-pages.html").rename(public_folder.joinpath("index.html")) sys.exit(0) -def get_artifacts_for_jobs_and_return_num_failed(jobs: list, public_folder: pathlib.Path) -> int: +def get_artifacts_for_jobs_and_return_num_failed( + jobs: dict, public_folder: pathlib.Path +) -> int: + """ + Get specified artifact folders for all jobs given and put them into the public folder. + + jobs: dictionary with the job names in the keys and a list of the + public folders to copy from the artifacts in the values + if "-public" is in the list, the actual folder name to copy is the key with "-public" appended + public_folder: target folder to copy stuff to + """ failed_count = 0 - for job in jobs: + for job, artifact_folders in jobs.items(): job_id = get_job_id(os.environ["CI_COMMIT_REF_NAME"], job) print(f"{job_id} - {job}") try: curl_for_artifacts(job_id) - job_public = job + "-public" - if job == "coverage-test-on-main-scheduled": - job_public = "coverage" - pathlib.Path("coverage_stv").rename( - public_folder.joinpath("coverage_stv") - ) - - pathlib.Path(job_public).rename(public_folder.joinpath(job_public)) + for fol in artifact_folders: + if fol == PUBLIC_FOL_MAGIC: + fol = job + fol + pathlib.Path(fol).rename(public_folder.joinpath(fol)) except subprocess.CalledProcessError: print(f"Could not get artifacts for {job}") -- GitLab From 7a06d73bac4e78256ed32e26f7965fe569db1c06 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 11:06:33 +0200 Subject: [PATCH 08/52] add first test part for BASOP --- ci/setup_pages.py | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index c87bb95648..3328f5b69c 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -7,6 +7,8 @@ import sys from get_id_of_last_job_occurence import get_job_id PUBLIC_FOL_MAGIC = "-public" +PROJECT_ID_FLOAT_REPO = 49 +PROJECT_ID_BASOP_REPO = 77 JOBS_FLOAT_REPO = { "complexity-stereo-in-stereo-out": [PUBLIC_FOL_MAGIC], @@ -17,36 +19,54 @@ JOBS_FLOAT_REPO = { "complexity-StereoDmxEVS-stereo-in-mono-out": [PUBLIC_FOL_MAGIC], "coverage-test-on-main-scheduled": ["coverage", "coverage_stv"], } +JOBS_BASOP_REPO = { + "ivas-pytest-mld-long-dec": ["mld.csv"], +} + +JOBS_FOR_PROJECT_ID = {49: JOBS_FLOAT_REPO, 77: JOBS_BASOP_REPO} + ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" -PUBLIC = "./public" +PUBLIC_FOLDER = pathlib.Path("./public") def main(): - public_folder = pathlib.Path(PUBLIC) - public_folder.mkdir() - failed_count = get_artifacts_for_jobs_and_return_num_failed( - JOBS_FLOAT_REPO, public_folder - ) + project_id = os.environ["CI_PROJECT_ID"] + jobs = JOBS_FOR_PROJECT_ID[project_id] + + PUBLIC_FOLDER.mkdir() - if failed_count == len(JOBS_FLOAT_REPO): + failed_count = get_artifacts_for_jobs_and_return_num_failed(jobs) + + if failed_count == len(jobs): + print("Artifact collection failed for all jobs to check.") sys.exit(1) - pathlib.Path("ci/index-pages.html").rename(public_folder.joinpath("index.html")) sys.exit(0) -def get_artifacts_for_jobs_and_return_num_failed( - jobs: dict, public_folder: pathlib.Path -) -> int: +def setup_pages(project_id: int): + if project_id == PROJECT_ID_FLOAT_REPO: + pathlib.Path("ci/index-pages.html").rename(PUBLIC_FOLDER.joinpath("index.html")) + elif project_id == PROJECT_ID_BASOP_REPO: + # basic test to see if mld file is collected correctly + mld_file = pathlib.Path(PUBLIC_FOLDER.joinpath("mld.csv")) + if mld_file.exists(): + with open(PUBLIC_FOLDER.joinpath("index.html"), "w") as f: + with open(mld_file) as mld_f: + f.write(mld_f.read()) + else: + raise ValueError(f"Invalid project ID: '{project_id}'") + + +def get_artifacts_for_jobs_and_return_num_failed( jobs: dict) -> int: """ Get specified artifact folders for all jobs given and put them into the public folder. jobs: dictionary with the job names in the keys and a list of the public folders to copy from the artifacts in the values if "-public" is in the list, the actual folder name to copy is the key with "-public" appended - public_folder: target folder to copy stuff to """ failed_count = 0 @@ -59,7 +79,7 @@ def get_artifacts_for_jobs_and_return_num_failed( for fol in artifact_folders: if fol == PUBLIC_FOL_MAGIC: fol = job + fol - pathlib.Path(fol).rename(public_folder.joinpath(fol)) + pathlib.Path(fol).rename(PUBLIC_FOLDER.joinpath(fol)) except subprocess.CalledProcessError: print(f"Could not get artifacts for {job}") -- GitLab From 75c08f0330ebb0fd7800fe026beae5a9be886d0b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 13:22:02 +0200 Subject: [PATCH 09/52] add cast for project id --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 3328f5b69c..c6b45aecc6 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -32,7 +32,7 @@ PUBLIC_FOLDER = pathlib.Path("./public") def main(): - project_id = os.environ["CI_PROJECT_ID"] + project_id = int(os.environ["CI_PROJECT_ID"]) jobs = JOBS_FOR_PROJECT_ID[project_id] PUBLIC_FOLDER.mkdir() -- GitLab From b4290476e9abcc366b4b40542fed7dfadfaeb414 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 13:40:05 +0200 Subject: [PATCH 10/52] add missing arguments for get_idx --- ci/setup_pages.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index c6b45aecc6..31a5a4fce0 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -31,13 +31,15 @@ PUBLIC_FOLDER = pathlib.Path("./public") def main(): - project_id = int(os.environ["CI_PROJECT_ID"]) jobs = JOBS_FOR_PROJECT_ID[project_id] PUBLIC_FOLDER.mkdir() - failed_count = get_artifacts_for_jobs_and_return_num_failed(jobs) + success_only = project_id == PROJECT_ID_FLOAT_REPO + failed_count = get_artifacts_for_jobs_and_return_num_failed( + jobs, project_id, success_only + ) if failed_count == len(jobs): print("Artifact collection failed for all jobs to check.") @@ -60,7 +62,9 @@ def setup_pages(project_id: int): raise ValueError(f"Invalid project ID: '{project_id}'") -def get_artifacts_for_jobs_and_return_num_failed( jobs: dict) -> int: +def get_artifacts_for_jobs_and_return_num_failed( + jobs: dict, project_id: int, success_only: bool +) -> int: """ Get specified artifact folders for all jobs given and put them into the public folder. @@ -71,7 +75,9 @@ def get_artifacts_for_jobs_and_return_num_failed( jobs: dict) -> int: failed_count = 0 for job, artifact_folders in jobs.items(): - job_id = get_job_id(os.environ["CI_COMMIT_REF_NAME"], job) + job_id = get_job_id( + os.environ["CI_COMMIT_REF_NAME"], job, project_id, success_only + ) print(f"{job_id} - {job}") try: curl_for_artifacts(job_id) -- GitLab From 8b194af08da1e1aee95a562375b6322169828e26 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 14:07:46 +0200 Subject: [PATCH 11/52] add debug printout --- ci/setup_pages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 31a5a4fce0..64bb48b052 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -103,6 +103,7 @@ def curl_for_artifacts(job_id): "--output", ARTIFACTS, ] + print(cmd) subprocess.run(cmd, check=True) # check for valid archive (if not, it is likely a 404 page, then display that) -- GitLab From 1b7f914d32481a65519c1bae71cd0275a977dc51 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 14:10:47 +0200 Subject: [PATCH 12/52] add more debug printouts --- ci/setup_pages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 64bb48b052..d8f5e02381 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -75,6 +75,7 @@ def get_artifacts_for_jobs_and_return_num_failed( failed_count = 0 for job, artifact_folders in jobs.items(): + print(os.environ["CI_COMMIT_REF_NAME"], job, project_id, success_only) job_id = get_job_id( os.environ["CI_COMMIT_REF_NAME"], job, project_id, success_only ) -- GitLab From 89f20750a3f8c68fec24a218276bedbcf689ca3e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 14:15:10 +0200 Subject: [PATCH 13/52] always search on default branch --- ci/setup_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index d8f5e02381..3837a2bb01 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -75,9 +75,9 @@ def get_artifacts_for_jobs_and_return_num_failed( failed_count = 0 for job, artifact_folders in jobs.items(): - print(os.environ["CI_COMMIT_REF_NAME"], job, project_id, success_only) + print(os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) job_id = get_job_id( - os.environ["CI_COMMIT_REF_NAME"], job, project_id, success_only + os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only ) print(f"{job_id} - {job}") try: -- GitLab From b1454c194d8fa55ff112cdbde9d73e82846bc53c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 15:16:26 +0200 Subject: [PATCH 14/52] add html table creation from mld csv --- ci/setup_pages.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 3837a2bb01..1856ef2b2f 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -45,23 +45,46 @@ def main(): print("Artifact collection failed for all jobs to check.") sys.exit(1) + setup_pages(project_id) + sys.exit(0) def setup_pages(project_id: int): + + index_html = PUBLIC_FOLDER.joinpath("index.html") + if project_id == PROJECT_ID_FLOAT_REPO: - pathlib.Path("ci/index-pages.html").rename(PUBLIC_FOLDER.joinpath("index.html")) + pathlib.Path("ci/index-pages.html").rename(index_html) elif project_id == PROJECT_ID_BASOP_REPO: - # basic test to see if mld file is collected correctly mld_file = pathlib.Path(PUBLIC_FOLDER.joinpath("mld.csv")) - if mld_file.exists(): - with open(PUBLIC_FOLDER.joinpath("index.html"), "w") as f: - with open(mld_file) as mld_f: - f.write(mld_f.read()) + html_table_from_csv_file(mld_file, index_html) else: raise ValueError(f"Invalid project ID: '{project_id}'") +def html_table_from_csv_file(csv_path, html_path): + TABLE_ROW = "{}" + TABLE_HEADER_CELL = "{}" + TABLE_CELL = "{}" + + with open(csv_path) as csv: + reader = csv.DictReader(csv, delimiter=";") + + with open(html_path, "w") as html: + print("", f=html) + header_row = "".join( + [TABLE_HEADER_CELL.format(x for x in reader.fieldnames)] + ) + print(header_row, f=html) + for csv_row in reader: + html_row = TABLE_ROW.format( + "".join([TABLE_CELL.format(x for x in csv_row)]) + ) + print(html_row, f=html) + print("
", f=html) + + def get_artifacts_for_jobs_and_return_num_failed( jobs: dict, project_id: int, success_only: bool ) -> int: -- GitLab From 1b667444ad1d9569d9030abf85bd9ac7b63a5679 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 15:18:14 +0200 Subject: [PATCH 15/52] correctly use package --- ci/setup_pages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 1856ef2b2f..711d1de2ff 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -3,6 +3,7 @@ import os import pathlib import subprocess import sys +import csv from get_id_of_last_job_occurence import get_job_id @@ -68,8 +69,8 @@ def html_table_from_csv_file(csv_path, html_path): TABLE_HEADER_CELL = "{}" TABLE_CELL = "{}" - with open(csv_path) as csv: - reader = csv.DictReader(csv, delimiter=";") + with open(csv_path) as csv_file: + reader = csv.DictReader(csv_file, delimiter=";") with open(html_path, "w") as html: print("", f=html) -- GitLab From c7f12545520a2353a7ef74228714a1391d2b0dd8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 15:19:48 +0200 Subject: [PATCH 16/52] fix wrong argument name --- ci/setup_pages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 711d1de2ff..21d93eab6f 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -73,17 +73,17 @@ def html_table_from_csv_file(csv_path, html_path): reader = csv.DictReader(csv_file, delimiter=";") with open(html_path, "w") as html: - print("
", f=html) + print("
", file=html) header_row = "".join( [TABLE_HEADER_CELL.format(x for x in reader.fieldnames)] ) - print(header_row, f=html) + print(header_row, file=html) for csv_row in reader: html_row = TABLE_ROW.format( "".join([TABLE_CELL.format(x for x in csv_row)]) ) - print(html_row, f=html) - print("
", f=html) + print(html_row, ffile=html) + print("", file=html) def get_artifacts_for_jobs_and_return_num_failed( -- GitLab From 5c3577336676565abb27466eb269f547decfb042 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 23 May 2024 15:20:36 +0200 Subject: [PATCH 17/52] fix typo --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 21d93eab6f..3873b6107a 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -82,7 +82,7 @@ def html_table_from_csv_file(csv_path, html_path): html_row = TABLE_ROW.format( "".join([TABLE_CELL.format(x for x in csv_row)]) ) - print(html_row, ffile=html) + print(html_row, file=html) print("", file=html) -- GitLab From df6c9575c8f0e369ad7bd579df1e236c984e743a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 09:20:48 +0200 Subject: [PATCH 18/52] fix typo --- ci/setup_pages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 3873b6107a..e4b068aa19 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -5,7 +5,7 @@ import subprocess import sys import csv -from get_id_of_last_job_occurence import get_job_id +from .get_id_of_last_job_occurence import get_job_id PUBLIC_FOL_MAGIC = "-public" PROJECT_ID_FLOAT_REPO = 49 @@ -75,12 +75,12 @@ def html_table_from_csv_file(csv_path, html_path): with open(html_path, "w") as html: print("", file=html) header_row = "".join( - [TABLE_HEADER_CELL.format(x for x in reader.fieldnames)] + [TABLE_HEADER_CELL.format(x) for x in reader.fieldnames] ) print(header_row, file=html) for csv_row in reader: html_row = TABLE_ROW.format( - "".join([TABLE_CELL.format(x for x in csv_row)]) + "".join([TABLE_CELL.format(csv_row[x]) for x in reader.fieldnames]) ) print(html_row, file=html) print("
", file=html) -- GitLab From 84f33156026c76c79f42e49437f53150b26cd029 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 09:24:30 +0200 Subject: [PATCH 19/52] remove relative import --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index e4b068aa19..d00373e1c2 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -5,7 +5,7 @@ import subprocess import sys import csv -from .get_id_of_last_job_occurence import get_job_id +from get_id_of_last_job_occurence import get_job_id PUBLIC_FOL_MAGIC = "-public" PROJECT_ID_FLOAT_REPO = 49 -- GitLab From edb1c25650636ada6be7b7a9062696e2c41d024c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 10:36:54 +0200 Subject: [PATCH 20/52] make get_job_id return a list of ids --- ci/get_id_of_last_job_occurence.py | 42 +++++++++++++++++++++++------- ci/setup_pages.py | 8 +++--- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 50beac5bb7..4253c9ab9c 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -40,9 +40,15 @@ PAGE_SUFFIX = "&page={}" API_URL_TMPL = "https://forge.3gpp.org/rep/api/v4/projects/{}/pipelines" -def get_job_id(branch_name, job_name, project_id, success_only): - job_id = -1 - # check last 500 pipelines max +def get_job_ids( + branch_name: str, job_name: str, project_id: int, success_only: bool, n: int +) -> list[int]: + """ + Find id of last jobs with name for prject with id . + If is true, only jobs with status success are considered. + """ + job_ids = list() + # check last 5000 pipelines max for page in range(100): url_pls = API_URL_TMPL.format(project_id) @@ -64,15 +70,18 @@ def get_job_id(branch_name, job_name, project_id, success_only): for job in resp_jobs.json(): include_job = not success_only or job["status"] == "success" if include_job and job["name"] == job_name: - job_id = job["id"] - break - if job_id >= 0: + job_ids.append(job["id"]) + + if len(job_ids) == n: + break + + if len(job_ids) == n: break - if job_id >= 0: + if len(job_ids) == n: break - return job_id + return job_ids if __name__ == "__main__": @@ -80,9 +89,22 @@ if __name__ == "__main__": parser.add_argument("branch_name", help="Name of the branch to search on") parser.add_argument("job_name", help="Name of the job to get the id of") parser.add_argument("project_id", help="ID of project to search in", type=int) - parser.add_argument("--success_only", help="Only include jobs with status 'success'", action="store_true") + parser.add_argument( + "--success_only", + help="Only include jobs with status 'success'", + action="store_true", + ) args = parser.parse_args() - job_id = get_job_id(args.branch_name, args.job_name, args.project_id, args.success_only) + n = 1 + job_ids = get_job_ids( + args.branch_name, args.job_name, args.project_id, args.success_only, n + ) + + try: + job_id = job_ids[0] + except IndexError: + job_id = -1 + print(job_id) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index d00373e1c2..a8c30fb053 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -5,7 +5,7 @@ import subprocess import sys import csv -from get_id_of_last_job_occurence import get_job_id +from get_id_of_last_job_occurence import get_job_ids PUBLIC_FOL_MAGIC = "-public" PROJECT_ID_FLOAT_REPO = 49 @@ -100,9 +100,11 @@ def get_artifacts_for_jobs_and_return_num_failed( for job, artifact_folders in jobs.items(): print(os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) - job_id = get_job_id( - os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only + job_ids = get_job_ids( + os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only, 1 ) + job_id = -1 if len(job_ids) == 0 else job_ids[0] + print(f"{job_id} - {job}") try: curl_for_artifacts(job_id) -- GitLab From 647c3f6a51289e85e4f82cad74bd11b5ed8f044c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:04:05 +0200 Subject: [PATCH 21/52] change job handling in setup_pages.py --- ci/setup_pages.py | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index a8c30fb053..77fbe61e73 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -4,6 +4,7 @@ import pathlib import subprocess import sys import csv +from collections import namedtuple from get_id_of_last_job_occurence import get_job_ids @@ -11,17 +12,28 @@ PUBLIC_FOL_MAGIC = "-public" PROJECT_ID_FLOAT_REPO = 49 PROJECT_ID_BASOP_REPO = 77 -JOBS_FLOAT_REPO = { - "complexity-stereo-in-stereo-out": [PUBLIC_FOL_MAGIC], - "complexity-ism-in-binaural-out": [PUBLIC_FOL_MAGIC], - "complexity-sba-hoa3-in-hoa3-out": [PUBLIC_FOL_MAGIC], - "complexity-mc-in-7_1_4-out": [PUBLIC_FOL_MAGIC], - "complexity-masa-in-7_1_4-out": [PUBLIC_FOL_MAGIC], - "complexity-StereoDmxEVS-stereo-in-mono-out": [PUBLIC_FOL_MAGIC], - "coverage-test-on-main-scheduled": ["coverage", "coverage_stv"], -} + +Job = namedtuple("Job", "name", "artifacts", "n") + +JOBS_FLOAT_REPO = [ + Job( + "complexity-stereo-in-stereo-out", ["complexity-stereo-in-stereo-out-public"], 1 + ), + Job("complexity-ism-in-binaural-out", ["complexity-ism-in-binaural-out-public"], 1), + Job( + "complexity-sba-hoa3-in-hoa3-out", ["complexity-sba-hoa3-in-hoa3-out-public"], 1 + ), + Job("complexity-mc-in-7_1_4-out", ["complexity-mc-in-7_1_4-out-public"], 1), + Job("complexity-masa-in-7_1_4-out", ["complexity-masa-in-7_1_4-out-public"], 1), + Job( + "complexity-StereoDmxEVS-stereo-in-mono-out", + ["complexity-StereoDmxEVS-stereo-in-mono-out-public"], + 1, + ), + Job("coverage-test-on-main-scheduled", ["coverage", "coverage_stv"], 1), +] JOBS_BASOP_REPO = { - "ivas-pytest-mld-long-dec": ["mld.csv"], + Job("ivas-pytest-mld-long-dec", ["mld.csv"], 2), } JOBS_FOR_PROJECT_ID = {49: JOBS_FLOAT_REPO, 77: JOBS_BASOP_REPO} @@ -32,11 +44,10 @@ PUBLIC_FOLDER = pathlib.Path("./public") def main(): - project_id = int(os.environ["CI_PROJECT_ID"]) - jobs = JOBS_FOR_PROJECT_ID[project_id] - PUBLIC_FOLDER.mkdir() + project_id = int(os.environ["CI_PROJECT_ID"]) + jobs = JOBS_FOR_PROJECT_ID[project_id] success_only = project_id == PROJECT_ID_FLOAT_REPO failed_count = get_artifacts_for_jobs_and_return_num_failed( jobs, project_id, success_only @@ -52,7 +63,6 @@ def main(): def setup_pages(project_id: int): - index_html = PUBLIC_FOLDER.joinpath("index.html") if project_id == PROJECT_ID_FLOAT_REPO: @@ -87,7 +97,7 @@ def html_table_from_csv_file(csv_path, html_path): def get_artifacts_for_jobs_and_return_num_failed( - jobs: dict, project_id: int, success_only: bool + jobs: list, project_id: int, success_only: bool ) -> int: """ Get specified artifact folders for all jobs given and put them into the public folder. @@ -98,7 +108,7 @@ def get_artifacts_for_jobs_and_return_num_failed( """ failed_count = 0 - for job, artifact_folders in jobs.items(): + for job in jobs: print(os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) job_ids = get_job_ids( os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only, 1 @@ -109,9 +119,7 @@ def get_artifacts_for_jobs_and_return_num_failed( try: curl_for_artifacts(job_id) - for fol in artifact_folders: - if fol == PUBLIC_FOL_MAGIC: - fol = job + fol + for fol in jobs.artifacts: pathlib.Path(fol).rename(PUBLIC_FOLDER.joinpath(fol)) except subprocess.CalledProcessError: -- GitLab From 1ff2050d1371f3acd62af6b9dfd413d79e793670 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:24:11 +0200 Subject: [PATCH 22/52] remove explicit names of artifacts and simply move all --- ci/setup_pages.py | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 77fbe61e73..9714280299 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -5,6 +5,7 @@ import subprocess import sys import csv from collections import namedtuple +from tempdir import TemporaryDirectory from get_id_of_last_job_occurence import get_job_ids @@ -13,27 +14,22 @@ PROJECT_ID_FLOAT_REPO = 49 PROJECT_ID_BASOP_REPO = 77 -Job = namedtuple("Job", "name", "artifacts", "n") +Job = namedtuple("Job", "name", "n") JOBS_FLOAT_REPO = [ - Job( - "complexity-stereo-in-stereo-out", ["complexity-stereo-in-stereo-out-public"], 1 - ), - Job("complexity-ism-in-binaural-out", ["complexity-ism-in-binaural-out-public"], 1), - Job( - "complexity-sba-hoa3-in-hoa3-out", ["complexity-sba-hoa3-in-hoa3-out-public"], 1 - ), - Job("complexity-mc-in-7_1_4-out", ["complexity-mc-in-7_1_4-out-public"], 1), - Job("complexity-masa-in-7_1_4-out", ["complexity-masa-in-7_1_4-out-public"], 1), + Job("complexity-stereo-in-stereo-out", 1), + Job("complexity-ism-in-binaural-out", 1), + Job("complexity-sba-hoa3-in-hoa3-out", 1), + Job("complexity-mc-in-7_1_4-out", 1), + Job("complexity-masa-in-7_1_4-out", 1), Job( "complexity-StereoDmxEVS-stereo-in-mono-out", - ["complexity-StereoDmxEVS-stereo-in-mono-out-public"], 1, ), - Job("coverage-test-on-main-scheduled", ["coverage", "coverage_stv"], 1), + Job("coverage-test-on-main-scheduled", 1), ] JOBS_BASOP_REPO = { - Job("ivas-pytest-mld-long-dec", ["mld.csv"], 2), + Job("ivas-pytest-mld-long-dec", 2), } JOBS_FOR_PROJECT_ID = {49: JOBS_FLOAT_REPO, 77: JOBS_BASOP_REPO} @@ -109,27 +105,30 @@ def get_artifacts_for_jobs_and_return_num_failed( failed_count = 0 for job in jobs: - print(os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) + print(os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only) job_ids = get_job_ids( - os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only, 1 + os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only, job.n ) - job_id = -1 if len(job_ids) == 0 else job_ids[0] - print(f"{job_id} - {job}") + print(f"{job_ids} - {job.name}") try: - curl_for_artifacts(job_id) + for job_id in job_ids: + with TemporaryDirectory() as tmp_dir: + curl_for_artifacts(job_id, str(tmp_dir)) - for fol in jobs.artifacts: - pathlib.Path(fol).rename(PUBLIC_FOLDER.joinpath(fol)) + for artifact in tmp_dir.iterdir(): + tmp_dir.joinpath(artifact).rename( + PUBLIC_FOLDER.joinpath(artifact) + ) except subprocess.CalledProcessError: - print(f"Could not get artifacts for {job}") + print(f"Could not get artifacts for {job.name}") failed_count += 1 return failed_count -def curl_for_artifacts(job_id): +def curl_for_artifacts(job_id: int, exdir: str): cmd = [ "curl", "--request", @@ -151,7 +150,7 @@ def curl_for_artifacts(job_id): raise subprocess.CalledProcessError(-1, "Unzip check failed") # do the actual unzipping - cmd = ["unzip", ARTIFACTS] + cmd = ["unzip", ARTIFACTS, "-d", exdir] subprocess.run(cmd, check=True) -- GitLab From d258650ed3dfeb7c966101187b57b1fced002db3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:31:52 +0200 Subject: [PATCH 23/52] for testing, simply print names of two csv files on page --- ci/setup_pages.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 9714280299..9280ec94c1 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -32,7 +32,7 @@ JOBS_BASOP_REPO = { Job("ivas-pytest-mld-long-dec", 2), } -JOBS_FOR_PROJECT_ID = {49: JOBS_FLOAT_REPO, 77: JOBS_BASOP_REPO} +JOBS_FOR_PROJECT_ID = {PROJECT_ID_FLOAT_REPO: JOBS_FLOAT_REPO, PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO} ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" @@ -64,8 +64,11 @@ def setup_pages(project_id: int): if project_id == PROJECT_ID_FLOAT_REPO: pathlib.Path("ci/index-pages.html").rename(index_html) elif project_id == PROJECT_ID_BASOP_REPO: - mld_file = pathlib.Path(PUBLIC_FOLDER.joinpath("mld.csv")) - html_table_from_csv_file(mld_file, index_html) + mld_files = [f for f in PUBLIC_FOLDER.iterdir() if f.suffix == ".csv"] + # just to check that there are two files + with open(index_html, "w") as f: + for mld_f in mld_files: + f.write(mld_f.name) else: raise ValueError(f"Invalid project ID: '{project_id}'") @@ -117,9 +120,15 @@ def get_artifacts_for_jobs_and_return_num_failed( curl_for_artifacts(job_id, str(tmp_dir)) for artifact in tmp_dir.iterdir(): - tmp_dir.joinpath(artifact).rename( - PUBLIC_FOLDER.joinpath(artifact) - ) + # this is only needed until the new naming of the csv files is on main + if project_id == PROJECT_ID_BASOP_REPO: + tmp_dir.joinpath(artifact).rename( + PUBLIC_FOLDER.joinpath(artifact.name + f"-id_{job_id}") + ) + else: + tmp_dir.joinpath(artifact).rename( + PUBLIC_FOLDER.joinpath(artifact) + ) except subprocess.CalledProcessError: print(f"Could not get artifacts for {job.name}") -- GitLab From 4761479408beaebfffabc5b69a51b891e1c4bcb4 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:37:00 +0200 Subject: [PATCH 24/52] fix import --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 9280ec94c1..25332acbcc 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -5,7 +5,7 @@ import subprocess import sys import csv from collections import namedtuple -from tempdir import TemporaryDirectory +from tempfile import TemporaryDirectory from get_id_of_last_job_occurence import get_job_ids -- GitLab From 52fd379164dcfb58f0c30e0148a094682ce5bfcd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:39:25 +0200 Subject: [PATCH 25/52] fix namedtuple call --- ci/setup_pages.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 25332acbcc..5b547d4091 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -14,7 +14,7 @@ PROJECT_ID_FLOAT_REPO = 49 PROJECT_ID_BASOP_REPO = 77 -Job = namedtuple("Job", "name", "n") +Job = namedtuple("Job", ["name", "n"]) JOBS_FLOAT_REPO = [ Job("complexity-stereo-in-stereo-out", 1), @@ -22,10 +22,7 @@ JOBS_FLOAT_REPO = [ Job("complexity-sba-hoa3-in-hoa3-out", 1), Job("complexity-mc-in-7_1_4-out", 1), Job("complexity-masa-in-7_1_4-out", 1), - Job( - "complexity-StereoDmxEVS-stereo-in-mono-out", - 1, - ), + Job("complexity-StereoDmxEVS-stereo-in-mono-out", 1), Job("coverage-test-on-main-scheduled", 1), ] JOBS_BASOP_REPO = { -- GitLab From 07c2b68e1acb0a3493c32dc310822cd3a4f1d4d0 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:41:18 +0200 Subject: [PATCH 26/52] use tmpdir path as path object --- ci/setup_pages.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 5b547d4091..dd1177f1ee 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -114,7 +114,9 @@ def get_artifacts_for_jobs_and_return_num_failed( try: for job_id in job_ids: with TemporaryDirectory() as tmp_dir: - curl_for_artifacts(job_id, str(tmp_dir)) + curl_for_artifacts(job_id, tmp_dir) + + tmp_dir = pathlib.Path(tmp_dir) for artifact in tmp_dir.iterdir(): # this is only needed until the new naming of the csv files is on main -- GitLab From beb5cb35a070b600049098c408b8919dc88b5470 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:46:10 +0200 Subject: [PATCH 27/52] add debug printout --- ci/setup_pages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index dd1177f1ee..4a9f7a1f6b 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -120,6 +120,7 @@ def get_artifacts_for_jobs_and_return_num_failed( for artifact in tmp_dir.iterdir(): # this is only needed until the new naming of the csv files is on main + print(artifact) if project_id == PROJECT_ID_BASOP_REPO: tmp_dir.joinpath(artifact).rename( PUBLIC_FOLDER.joinpath(artifact.name + f"-id_{job_id}") -- GitLab From 513d24593a12e68af72daae2d970bd3b85840501 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:50:54 +0200 Subject: [PATCH 28/52] ensure suffix is kept when renaming files --- ci/setup_pages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 4a9f7a1f6b..3528f57639 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -122,8 +122,9 @@ def get_artifacts_for_jobs_and_return_num_failed( # this is only needed until the new naming of the csv files is on main print(artifact) if project_id == PROJECT_ID_BASOP_REPO: + name_with_id = artifact.stem + f"-id_{job_id}.csv" tmp_dir.joinpath(artifact).rename( - PUBLIC_FOLDER.joinpath(artifact.name + f"-id_{job_id}") + PUBLIC_FOLDER.joinpath(name_with_id) ) else: tmp_dir.joinpath(artifact).rename( -- GitLab From 9a4f2c35cff6a0fc56617973265a4781b945d342 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 11:52:46 +0200 Subject: [PATCH 29/52] add correct suffix everywhere --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 3528f57639..c0f8986182 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -122,7 +122,7 @@ def get_artifacts_for_jobs_and_return_num_failed( # this is only needed until the new naming of the csv files is on main print(artifact) if project_id == PROJECT_ID_BASOP_REPO: - name_with_id = artifact.stem + f"-id_{job_id}.csv" + name_with_id = artifact.stem + f"-id_{job_id}" + artifact.suffix tmp_dir.joinpath(artifact).rename( PUBLIC_FOLDER.joinpath(name_with_id) ) -- GitLab From 3d2cb3f18b8e0c59e1a98e1856c3de81eb420d4e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 18:03:15 +0200 Subject: [PATCH 30/52] add script for creating report subpage from mld csv files --- ci/basop-pages/create_report_pages.py | 148 ++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 ci/basop-pages/create_report_pages.py diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py new file mode 100644 index 0000000000..b42de0ad03 --- /dev/null +++ b/ci/basop-pages/create_report_pages.py @@ -0,0 +1,148 @@ +import csv +import pathlib +import argparse + + +CSV_DELIM = ";" +SUBPAGE_TMPL_CSS = """ + +""" + +SUBPAGE_TMPL_HTML = """ + + + + + + + + + + + + + + +{table_body} + +
TestcaseMLDMax Abs Diff
{id_current}{id_previous}{id_current}{id_previous}
+""" +TD_TMPL = "{}" +TR_TMPL = "{}" + +COLUMNS = ["testcase", "MLD", "MAXIMUM ABS DIFF"] + + +def create_subpage( + html_out, csv_current: str, csv_previous: str, id_current: int, id_previous: int +): + merged_reports = merge_and_cleanup_mld_reports( + csv_current, csv_previous, id_current, id_previous + ) + table_body = "\n".join( + tr_from_row(row, id_current, id_previous) for row in merged_reports + ) + new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format( + id_current=id_current, id_previous=id_previous, table_body=table_body + ) + with open(html_out, "w") as f: + f.write(new_subpage) + + +def tr_from_row(row, id_current, id_previous): + tr = list() + for c in COLUMNS: + try: + tr.append(TD_TMPL.format(row[c])) + except KeyError: + tr.append(TD_TMPL.format(row[f"{c}-{id_current}"])) + tr.append(TD_TMPL.format(row[f"{c}-{id_previous}"])) + + return TR_TMPL.format("\n".join(tr)) + + +def merge_and_cleanup_mld_reports( + csv_current: str, csv_previous: str, id_current: int, id_previous: int +): + with open(csv_current) as f: + current_reader = csv.DictReader(f, delimiter=CSV_DELIM) + current = list(current_reader) + with open(csv_previous) as f: + previous = list(csv.DictReader(f, delimiter=CSV_DELIM)) + + # TODO: handle newly added testcases - for now assume that both have the same columns + merge_key = "testcase" + other_keys = [k for k in current_reader.fieldnames if k != merge_key] + merged = merge_tables( + current, previous, id_current, id_previous, merge_key, other_keys + ) + + # TODO: sort on result as well + mld_col = f"MLD-{id_current}" + + def sort_func(x): + return float("inf") if x[mld_col] == "None" else float(x[mld_col]) + + merged = sorted(merged, key=sort_func, reverse=True) + print(merged) + + # remove the unecessary whole path from the testcase names + for row in merged: + row["testcase"] = pathlib.Path(row["testcase"]).name + + return merged + + +def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): + merged = list() + + for row1 in tbl1: + new_row = dict() + for key in other_keys: + new_row[f"{key}-{suffix1}"] = row1[key] + + for row2 in tbl2: + if row1[merge_key] == row2[merge_key]: + new_row[merge_key] = row1[merge_key] + for key in other_keys: + new_row[f"{key}-{suffix2}"] = row2[key] + break + + merged.append(new_row) + + return merged + + +create_subpage( + "/tmp/html_out.html", + "/Users/knj/Downloads/mld(1).csv", + "/Users/knj/Downloads/mld(2).csv", + "CURR", + "PREV", +) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("html_out") + parser.add_argument("csv_current") + parser.add_argument("csv_previous") + parser.add_argument("id_current", type=int) + parser.add_argument("id_previous", type=int) + args = parser.parse_args() + + create_subpage( + args.html_out, + args.csv_current, + args.csv_previous, + args.id_current, + args.id_previous, + ) -- GitLab From 3c1b6994912cbe6e3c5463b02a1a6c970f449b96 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 24 May 2024 18:08:30 +0200 Subject: [PATCH 31/52] add heading and info about jobs that are being compared --- ci/basop-pages/create_report_pages.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index b42de0ad03..90d1c6d1d9 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -19,6 +19,16 @@ SUBPAGE_TMPL_CSS = """ SUBPAGE_TMPL_HTML = """ +

Report for job {job_name}

+
  • Current run - id: {id_current}
    • +
  • Previous run - id: {id_previous}
    • + + +
    + @@ -43,7 +53,7 @@ COLUMNS = ["testcase", "MLD", "MAXIMUM ABS DIFF"] def create_subpage( - html_out, csv_current: str, csv_previous: str, id_current: int, id_previous: int + html_out, csv_current: str, csv_previous: str, id_current: int, id_previous: int, job_name: str ): merged_reports = merge_and_cleanup_mld_reports( csv_current, csv_previous, id_current, id_previous @@ -52,7 +62,7 @@ def create_subpage( tr_from_row(row, id_current, id_previous) for row in merged_reports ) new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format( - id_current=id_current, id_previous=id_previous, table_body=table_body + id_current=id_current, id_previous=id_previous, table_body=table_body, job_name=job_name ) with open(html_out, "w") as f: f.write(new_subpage) @@ -64,8 +74,8 @@ def tr_from_row(row, id_current, id_previous): try: tr.append(TD_TMPL.format(row[c])) except KeyError: - tr.append(TD_TMPL.format(row[f"{c}-{id_current}"])) tr.append(TD_TMPL.format(row[f"{c}-{id_previous}"])) + tr.append(TD_TMPL.format(row[f"{c}-{id_current}"])) return TR_TMPL.format("\n".join(tr)) @@ -122,14 +132,6 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): return merged -create_subpage( - "/tmp/html_out.html", - "/Users/knj/Downloads/mld(1).csv", - "/Users/knj/Downloads/mld(2).csv", - "CURR", - "PREV", -) - if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") @@ -137,6 +139,7 @@ if __name__ == "__main__": parser.add_argument("csv_previous") parser.add_argument("id_current", type=int) parser.add_argument("id_previous", type=int) + parser.add_argument("job_name") args = parser.parse_args() create_subpage( @@ -145,4 +148,5 @@ if __name__ == "__main__": args.csv_previous, args.id_current, args.id_previous, + args.job_name ) -- GitLab From ecae723198c78328971fe747777d99113eb9b43f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 08:56:08 +0200 Subject: [PATCH 32/52] do nothing for BASOP pages for now --- ci/setup_pages.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index c0f8986182..486c525b1b 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -61,11 +61,7 @@ def setup_pages(project_id: int): if project_id == PROJECT_ID_FLOAT_REPO: pathlib.Path("ci/index-pages.html").rename(index_html) elif project_id == PROJECT_ID_BASOP_REPO: - mld_files = [f for f in PUBLIC_FOLDER.iterdir() if f.suffix == ".csv"] - # just to check that there are two files - with open(index_html, "w") as f: - for mld_f in mld_files: - f.write(mld_f.name) + pass else: raise ValueError(f"Invalid project ID: '{project_id}'") -- GitLab From ffbbcc54744169dfb261b6879e5d74ba994135fc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 11:20:10 +0200 Subject: [PATCH 33/52] add colouring based on difference --- ci/basop-pages/create_report_pages.py | 52 +++++++++++++++++++++------ 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 90d1c6d1d9..24ee9ee579 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -14,6 +14,8 @@ SUBPAGE_TMPL_CSS = """ .tbase .tunder{font-weight:bold;text-align:center;text-decoration:underline;vertical-align:top} .tbase .tcenter{font-weight:bold;text-align:center;vertical-align:top} .tbase .tleft{text-align:left;vertical-align:top} +.tbase .tincrease{text-align:left;vertical-align:top;background-color:#ff0000;border-color:inherit;font-weight:bold;} +.tbase .treduce{text-align:left;vertical-align:top;background-color:#00ff00;border-color:inherit;font-weight:bold;} """ @@ -27,6 +29,8 @@ Comparing:
  • Previous run - id: {id_previous}
    • +
    +Table is sorted by Difference in MLD.
    Testcase
    @@ -46,14 +50,21 @@ Comparing:
    """ -TD_TMPL = "{}" +TD_TMPL_NORMAL = "{}" +TD_TMPL_INCREASE = "{}" +TD_TMPL_REDUCE = "{}" TR_TMPL = "{}" COLUMNS = ["testcase", "MLD", "MAXIMUM ABS DIFF"] def create_subpage( - html_out, csv_current: str, csv_previous: str, id_current: int, id_previous: int, job_name: str + html_out, + csv_current: str, + csv_previous: str, + id_current: int, + id_previous: int, + job_name: str, ): merged_reports = merge_and_cleanup_mld_reports( csv_current, csv_previous, id_current, id_previous @@ -62,7 +73,10 @@ def create_subpage( tr_from_row(row, id_current, id_previous) for row in merged_reports ) new_subpage = SUBPAGE_TMPL_CSS + SUBPAGE_TMPL_HTML.format( - id_current=id_current, id_previous=id_previous, table_body=table_body, job_name=job_name + id_current=id_current, + id_previous=id_previous, + table_body=table_body, + job_name=job_name, ) with open(html_out, "w") as f: f.write(new_subpage) @@ -72,10 +86,22 @@ def tr_from_row(row, id_current, id_previous): tr = list() for c in COLUMNS: try: - tr.append(TD_TMPL.format(row[c])) + # this is for the "testcase" column - here we don't compare, just one value is used + tr.append(TD_TMPL_NORMAL.format(row[c])) except KeyError: - tr.append(TD_TMPL.format(row[f"{c}-{id_previous}"])) - tr.append(TD_TMPL.format(row[f"{c}-{id_current}"])) + # this is for all columns where we compare between current and previous run + prev = row[f"{c}-{id_previous}"] + curr = row[f"{c}-{id_current}"] + + # use red background if increase, green if decrease, white if same + td_tmpl = TD_TMPL_NORMAL + if float(curr) > float(prev): + td_tmpl = TD_TMPL_INCREASE + if float(curr) < float(prev): + td_tmpl = TD_TMPL_REDUCE + + tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) + tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) return TR_TMPL.format("\n".join(tr)) @@ -97,13 +123,19 @@ def merge_and_cleanup_mld_reports( ) # TODO: sort on result as well - mld_col = f"MLD-{id_current}" + mld_col_curr = f"MLD-{id_current}" + mld_col_prev = f"MLD-{id_previous}" + # sort based on difference in MLD between current and previous run + # put cases with "None" at the top of the list def sort_func(x): - return float("inf") if x[mld_col] == "None" else float(x[mld_col]) + return ( + float("inf") + if x[mld_col_curr] == "None" or x[mld_col_prev] == "None" + else float(x[mld_col_curr]) - float(x[mld_col_prev]) + ) merged = sorted(merged, key=sort_func, reverse=True) - print(merged) # remove the unecessary whole path from the testcase names for row in merged: @@ -148,5 +180,5 @@ if __name__ == "__main__": args.csv_previous, args.id_current, args.id_previous, - args.job_name + args.job_name, ) -- GitLab From 553a91a0d8e18b53507259ba5c7e8b7890366068 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 12:15:04 +0200 Subject: [PATCH 34/52] only get last ltv job --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 486c525b1b..77e92d252b 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -26,7 +26,7 @@ JOBS_FLOAT_REPO = [ Job("coverage-test-on-main-scheduled", 1), ] JOBS_BASOP_REPO = { - Job("ivas-pytest-mld-long-dec", 2), + Job("ivas-pytest-mld-long-dec", 1), } JOBS_FOR_PROJECT_ID = {PROJECT_ID_FLOAT_REPO: JOBS_FLOAT_REPO, PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO} -- GitLab From e05c384bf60677943e8bb2ad1c3f706103da08e7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 12:44:01 +0200 Subject: [PATCH 35/52] use shutil to avoid cross-device move problems --- ci/setup_pages.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 77e92d252b..557fe02ddf 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -4,6 +4,7 @@ import pathlib import subprocess import sys import csv +import shutil from collections import namedtuple from tempfile import TemporaryDirectory @@ -29,7 +30,10 @@ JOBS_BASOP_REPO = { Job("ivas-pytest-mld-long-dec", 1), } -JOBS_FOR_PROJECT_ID = {PROJECT_ID_FLOAT_REPO: JOBS_FLOAT_REPO, PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO} +JOBS_FOR_PROJECT_ID = { + PROJECT_ID_FLOAT_REPO: JOBS_FLOAT_REPO, + PROJECT_ID_BASOP_REPO: JOBS_BASOP_REPO, +} ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" @@ -59,7 +63,8 @@ def setup_pages(project_id: int): index_html = PUBLIC_FOLDER.joinpath("index.html") if project_id == PROJECT_ID_FLOAT_REPO: - pathlib.Path("ci/index-pages.html").rename(index_html) + src = pathlib.Path("ci/index-pages.html").absolute() + shutil.move(src, index_html) elif project_id == PROJECT_ID_BASOP_REPO: pass else: @@ -117,15 +122,11 @@ def get_artifacts_for_jobs_and_return_num_failed( for artifact in tmp_dir.iterdir(): # this is only needed until the new naming of the csv files is on main print(artifact) + src = tmp_dir.joinpath(artifact).absolute() + dst = PUBLIC_FOLDER.joinpath(artifact) if project_id == PROJECT_ID_BASOP_REPO: - name_with_id = artifact.stem + f"-id_{job_id}" + artifact.suffix - tmp_dir.joinpath(artifact).rename( - PUBLIC_FOLDER.joinpath(name_with_id) - ) - else: - tmp_dir.joinpath(artifact).rename( - PUBLIC_FOLDER.joinpath(artifact) - ) + dst = artifact.stem + f"-id_{job_id}" + artifact.suffix + shutil.move(src, dst) except subprocess.CalledProcessError: print(f"Could not get artifacts for {job.name}") -- GitLab From d2c6fa5aa1d9b936fc0f8b9ddaa2b54f56e9290d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 12:51:37 +0200 Subject: [PATCH 36/52] change branch to search on for testing --- ci/setup_pages.py | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 557fe02ddf..8a971e072c 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -71,28 +71,6 @@ def setup_pages(project_id: int): raise ValueError(f"Invalid project ID: '{project_id}'") -def html_table_from_csv_file(csv_path, html_path): - TABLE_ROW = "{}" - TABLE_HEADER_CELL = "{}" - TABLE_CELL = "{}" - - with open(csv_path) as csv_file: - reader = csv.DictReader(csv_file, delimiter=";") - - with open(html_path, "w") as html: - print("", file=html) - header_row = "".join( - [TABLE_HEADER_CELL.format(x) for x in reader.fieldnames] - ) - print(header_row, file=html) - for csv_row in reader: - html_row = TABLE_ROW.format( - "".join([TABLE_CELL.format(csv_row[x]) for x in reader.fieldnames]) - ) - print(html_row, file=html) - print("
    ", file=html) - - def get_artifacts_for_jobs_and_return_num_failed( jobs: list, project_id: int, success_only: bool ) -> int: @@ -107,8 +85,12 @@ def get_artifacts_for_jobs_and_return_num_failed( for job in jobs: print(os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only) + # TODO: change back before merginG!! + # job_ids = get_job_ids( + # os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only, job.n + # ) job_ids = get_job_ids( - os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only, job.n + "ci/setup_gitlab_pages_for_ci_result_presentation", job.name, project_id, success_only, job.n ) print(f"{job_ids} - {job.name}") -- GitLab From 15fbc82e123cf2afec6e143792053cf1d73135f8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 12:56:28 +0200 Subject: [PATCH 37/52] add debug output --- ci/setup_pages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 8a971e072c..5fb60dfbb1 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -56,6 +56,8 @@ def main(): setup_pages(project_id) + print(list(PUBLIC_FOLDER.iterdir())) + sys.exit(0) -- GitLab From 0a2a1373d145135a7b634def650c778706ef1fea Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 13:01:09 +0200 Subject: [PATCH 38/52] add more debug output --- ci/setup_pages.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 5fb60dfbb1..3795bb8bc0 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -3,7 +3,6 @@ import os import pathlib import subprocess import sys -import csv import shutil from collections import namedtuple from tempfile import TemporaryDirectory @@ -42,6 +41,7 @@ PUBLIC_FOLDER = pathlib.Path("./public") def main(): PUBLIC_FOLDER.mkdir() + print(list(pathlib.Path("./").iterdir())) project_id = int(os.environ["CI_PROJECT_ID"]) jobs = JOBS_FOR_PROJECT_ID[project_id] @@ -64,6 +64,8 @@ def main(): def setup_pages(project_id: int): index_html = PUBLIC_FOLDER.joinpath("index.html") + print(list(PUBLIC_FOLDER.iterdir())) + if project_id == PROJECT_ID_FLOAT_REPO: src = pathlib.Path("ci/index-pages.html").absolute() shutil.move(src, index_html) @@ -105,11 +107,11 @@ def get_artifacts_for_jobs_and_return_num_failed( for artifact in tmp_dir.iterdir(): # this is only needed until the new naming of the csv files is on main - print(artifact) src = tmp_dir.joinpath(artifact).absolute() dst = PUBLIC_FOLDER.joinpath(artifact) if project_id == PROJECT_ID_BASOP_REPO: dst = artifact.stem + f"-id_{job_id}" + artifact.suffix + print(f"{src} -> {dst}") shutil.move(src, dst) except subprocess.CalledProcessError: -- GitLab From d3c5191f8815ab6428ff496f10fd707eceb92eee Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 13:04:12 +0200 Subject: [PATCH 39/52] fix moving --- ci/setup_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 3795bb8bc0..012be6daf5 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -110,7 +110,7 @@ def get_artifacts_for_jobs_and_return_num_failed( src = tmp_dir.joinpath(artifact).absolute() dst = PUBLIC_FOLDER.joinpath(artifact) if project_id == PROJECT_ID_BASOP_REPO: - dst = artifact.stem + f"-id_{job_id}" + artifact.suffix + dst = PUBLIC_FOLDER.joinpath(artifact.stem + f"-id_{job_id}" + artifact.suffix) print(f"{src} -> {dst}") shutil.move(src, dst) -- GitLab From b085c9b32ac93a1688c4fe5cb8c433ef2b5bcc49 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 13:06:54 +0200 Subject: [PATCH 40/52] do not add id suffix when moving files --- ci/setup_pages.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 012be6daf5..4595c21f9c 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -106,11 +106,8 @@ def get_artifacts_for_jobs_and_return_num_failed( tmp_dir = pathlib.Path(tmp_dir) for artifact in tmp_dir.iterdir(): - # this is only needed until the new naming of the csv files is on main src = tmp_dir.joinpath(artifact).absolute() dst = PUBLIC_FOLDER.joinpath(artifact) - if project_id == PROJECT_ID_BASOP_REPO: - dst = PUBLIC_FOLDER.joinpath(artifact.stem + f"-id_{job_id}" + artifact.suffix) print(f"{src} -> {dst}") shutil.move(src, dst) -- GitLab From 134f07e551fee2d622088159ef5c135634075189 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 13:12:01 +0200 Subject: [PATCH 41/52] fix paths --- ci/setup_pages.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 4595c21f9c..f2aa5f51cb 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -36,7 +36,7 @@ JOBS_FOR_PROJECT_ID = { ARTIFACTS = "artifacts.zip" API_URL_BASE = "https://forge.3gpp.org/rep/api/v4/projects/{}/jobs" -PUBLIC_FOLDER = pathlib.Path("./public") +PUBLIC_FOLDER = pathlib.Path("./public").absolute() def main(): @@ -64,8 +64,6 @@ def main(): def setup_pages(project_id: int): index_html = PUBLIC_FOLDER.joinpath("index.html") - print(list(PUBLIC_FOLDER.iterdir())) - if project_id == PROJECT_ID_FLOAT_REPO: src = pathlib.Path("ci/index-pages.html").absolute() shutil.move(src, index_html) @@ -107,7 +105,7 @@ def get_artifacts_for_jobs_and_return_num_failed( for artifact in tmp_dir.iterdir(): src = tmp_dir.joinpath(artifact).absolute() - dst = PUBLIC_FOLDER.joinpath(artifact) + dst = PUBLIC_FOLDER.joinpath(artifact.name) print(f"{src} -> {dst}") shutil.move(src, dst) -- GitLab From cf129ab81417f8f8afff65d69c824237cc4976a8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 14:53:55 +0200 Subject: [PATCH 42/52] cleanup and correct curr/prev order in table --- ci/basop-pages/create_report_pages.py | 2 +- ci/setup_pages.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 24ee9ee579..c1d0a5e9cd 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -40,10 +40,10 @@ Comparing: Max Abs Diff - {id_current} {id_previous} {id_current} {id_previous} + {id_current} {table_body} diff --git a/ci/setup_pages.py b/ci/setup_pages.py index f2aa5f51cb..ae8752933c 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -9,7 +9,6 @@ from tempfile import TemporaryDirectory from get_id_of_last_job_occurence import get_job_ids -PUBLIC_FOL_MAGIC = "-public" PROJECT_ID_FLOAT_REPO = 49 PROJECT_ID_BASOP_REPO = 77 -- GitLab From 960f91cb256f97b052e118cd8c4d1f4617bed49a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 14:58:39 +0200 Subject: [PATCH 43/52] add links to jobs in the html page --- ci/basop-pages/create_report_pages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index c1d0a5e9cd..9542e96894 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -25,8 +25,8 @@ SUBPAGE_TMPL_HTML = """ Comparing:
      -
    • Current run - id: {id_current}
      • -
    • Previous run - id: {id_previous}
      • +
    • Current run - id: {id_current}
      • +
    • Previous run - id: {id_previous}

    -- GitLab From 671853439846e95ee922df8130400f791b410440 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 15:06:21 +0200 Subject: [PATCH 44/52] add main index page for basop pages --- ci/basop-pages/basop_index.html | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 ci/basop-pages/basop_index.html diff --git a/ci/basop-pages/basop_index.html b/ci/basop-pages/basop_index.html new file mode 100644 index 0000000000..dea89f3562 --- /dev/null +++ b/ci/basop-pages/basop_index.html @@ -0,0 +1,22 @@ + + + + + +

    Ivas BASOP code Development

    + +

    Daily long testvector tests

    + + + +

    Test Coverage

    + +
    + tbd... +
    + + -- GitLab From ac6db9f5ad04e526145e22d08b112dd0b5ea4cda Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 15:17:27 +0200 Subject: [PATCH 45/52] Revert "make get_job_id return a list of ids" This reverts commit edb1c25650636ada6be7b7a9062696e2c41d024c. --- ci/get_id_of_last_job_occurence.py | 42 ++++++----------------- ci/setup_pages.py | 55 +++++++++++++++--------------- 2 files changed, 37 insertions(+), 60 deletions(-) diff --git a/ci/get_id_of_last_job_occurence.py b/ci/get_id_of_last_job_occurence.py index 4253c9ab9c..50beac5bb7 100755 --- a/ci/get_id_of_last_job_occurence.py +++ b/ci/get_id_of_last_job_occurence.py @@ -40,15 +40,9 @@ PAGE_SUFFIX = "&page={}" API_URL_TMPL = "https://forge.3gpp.org/rep/api/v4/projects/{}/pipelines" -def get_job_ids( - branch_name: str, job_name: str, project_id: int, success_only: bool, n: int -) -> list[int]: - """ - Find id of last jobs with name for prject with id . - If is true, only jobs with status success are considered. - """ - job_ids = list() - # check last 5000 pipelines max +def get_job_id(branch_name, job_name, project_id, success_only): + job_id = -1 + # check last 500 pipelines max for page in range(100): url_pls = API_URL_TMPL.format(project_id) @@ -70,18 +64,15 @@ def get_job_ids( for job in resp_jobs.json(): include_job = not success_only or job["status"] == "success" if include_job and job["name"] == job_name: - job_ids.append(job["id"]) - - if len(job_ids) == n: - break - - if len(job_ids) == n: + job_id = job["id"] + break + if job_id >= 0: break - if len(job_ids) == n: + if job_id >= 0: break - return job_ids + return job_id if __name__ == "__main__": @@ -89,22 +80,9 @@ if __name__ == "__main__": parser.add_argument("branch_name", help="Name of the branch to search on") parser.add_argument("job_name", help="Name of the job to get the id of") parser.add_argument("project_id", help="ID of project to search in", type=int) - parser.add_argument( - "--success_only", - help="Only include jobs with status 'success'", - action="store_true", - ) + parser.add_argument("--success_only", help="Only include jobs with status 'success'", action="store_true") args = parser.parse_args() - n = 1 - job_ids = get_job_ids( - args.branch_name, args.job_name, args.project_id, args.success_only, n - ) - - try: - job_id = job_ids[0] - except IndexError: - job_id = -1 - + job_id = get_job_id(args.branch_name, args.job_name, args.project_id, args.success_only) print(job_id) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index ae8752933c..955c918395 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -4,29 +4,26 @@ import pathlib import subprocess import sys import shutil -from collections import namedtuple from tempfile import TemporaryDirectory -from get_id_of_last_job_occurence import get_job_ids +from get_id_of_last_job_occurence import get_job_id PROJECT_ID_FLOAT_REPO = 49 PROJECT_ID_BASOP_REPO = 77 -Job = namedtuple("Job", ["name", "n"]) - JOBS_FLOAT_REPO = [ - Job("complexity-stereo-in-stereo-out", 1), - Job("complexity-ism-in-binaural-out", 1), - Job("complexity-sba-hoa3-in-hoa3-out", 1), - Job("complexity-mc-in-7_1_4-out", 1), - Job("complexity-masa-in-7_1_4-out", 1), - Job("complexity-StereoDmxEVS-stereo-in-mono-out", 1), - Job("coverage-test-on-main-scheduled", 1), + "complexity-stereo-in-stereo-out", + "complexity-ism-in-binaural-out", + "complexity-sba-hoa3-in-hoa3-out", + "complexity-mc-in-7_1_4-out", + "complexity-masa-in-7_1_4-out", + "complexity-StereoDmxEVS-stereo-in-mono-out", + "coverage-test-on-main-scheduled", +] +JOBS_BASOP_REPO = [ + "ivas-pytest-mld-long-dec", ] -JOBS_BASOP_REPO = { - Job("ivas-pytest-mld-long-dec", 1), -} JOBS_FOR_PROJECT_ID = { PROJECT_ID_FLOAT_REPO: JOBS_FLOAT_REPO, @@ -87,26 +84,28 @@ def get_artifacts_for_jobs_and_return_num_failed( for job in jobs: print(os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only) # TODO: change back before merginG!! - # job_ids = get_job_ids( - # os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only, job.n + # job_id = get_job_id( + # os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only # ) - job_ids = get_job_ids( - "ci/setup_gitlab_pages_for_ci_result_presentation", job.name, project_id, success_only, job.n + job_id = get_job_id( + "ci/setup_gitlab_pages_for_ci_result_presentation", + job.name, + project_id, + success_only, ) - print(f"{job_ids} - {job.name}") + print(f"{job_id} - {job.name}") try: - for job_id in job_ids: - with TemporaryDirectory() as tmp_dir: - curl_for_artifacts(job_id, tmp_dir) + with TemporaryDirectory() as tmp_dir: + curl_for_artifacts(job_id, tmp_dir) - tmp_dir = pathlib.Path(tmp_dir) + tmp_dir = pathlib.Path(tmp_dir) - for artifact in tmp_dir.iterdir(): - src = tmp_dir.joinpath(artifact).absolute() - dst = PUBLIC_FOLDER.joinpath(artifact.name) - print(f"{src} -> {dst}") - shutil.move(src, dst) + for artifact in tmp_dir.iterdir(): + src = tmp_dir.joinpath(artifact).absolute() + dst = PUBLIC_FOLDER.joinpath(artifact.name) + print(f"{src} -> {dst}") + shutil.move(src, dst) except subprocess.CalledProcessError: print(f"Could not get artifacts for {job.name}") -- GitLab From 052819f4acc67701c556ac490c2fc77fbadbfb4b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 15:26:15 +0200 Subject: [PATCH 46/52] remove last remnants of jobs being namedtuples --- ci/setup_pages.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 955c918395..20a328806d 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -82,19 +82,19 @@ def get_artifacts_for_jobs_and_return_num_failed( failed_count = 0 for job in jobs: - print(os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only) + print(os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) # TODO: change back before merginG!! # job_id = get_job_id( - # os.environ["CI_DEFAULT_BRANCH"], job.name, project_id, success_only + # os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only # ) job_id = get_job_id( "ci/setup_gitlab_pages_for_ci_result_presentation", - job.name, + job, project_id, success_only, ) - print(f"{job_id} - {job.name}") + print(f"{job_id} - {job}") try: with TemporaryDirectory() as tmp_dir: curl_for_artifacts(job_id, tmp_dir) @@ -108,7 +108,7 @@ def get_artifacts_for_jobs_and_return_num_failed( shutil.move(src, dst) except subprocess.CalledProcessError: - print(f"Could not get artifacts for {job.name}") + print(f"Could not get artifacts for {job}") failed_count += 1 return failed_count -- GitLab From 73eac5cc5eb8b20c52b403734f731808163be859 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 15:30:19 +0200 Subject: [PATCH 47/52] for basop repo copy the main page --- ci/setup_pages.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 20a328806d..8be805116c 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -37,7 +37,6 @@ PUBLIC_FOLDER = pathlib.Path("./public").absolute() def main(): PUBLIC_FOLDER.mkdir() - print(list(pathlib.Path("./").iterdir())) project_id = int(os.environ["CI_PROJECT_ID"]) jobs = JOBS_FOR_PROJECT_ID[project_id] @@ -50,23 +49,15 @@ def main(): print("Artifact collection failed for all jobs to check.") sys.exit(1) - setup_pages(project_id) - - print(list(PUBLIC_FOLDER.iterdir())) - - sys.exit(0) - - -def setup_pages(project_id: int): index_html = PUBLIC_FOLDER.joinpath("index.html") - if project_id == PROJECT_ID_FLOAT_REPO: src = pathlib.Path("ci/index-pages.html").absolute() shutil.move(src, index_html) elif project_id == PROJECT_ID_BASOP_REPO: - pass - else: - raise ValueError(f"Invalid project ID: '{project_id}'") + src = pathlib.Path("ci/basop-pages/basop_index.html").absolute() + shutil.move(src, index_html) + + sys.exit(0) def get_artifacts_for_jobs_and_return_num_failed( -- GitLab From 30aa10e0e290ac53b04b89bb8b883cc65cc1d269 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 27 May 2024 15:36:27 +0200 Subject: [PATCH 48/52] change paths in basop main page --- ci/basop-pages/basop_index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/basop-pages/basop_index.html b/ci/basop-pages/basop_index.html index dea89f3562..4c5202ad35 100644 --- a/ci/basop-pages/basop_index.html +++ b/ci/basop-pages/basop_index.html @@ -8,9 +8,9 @@

    Daily long testvector tests

    Test Coverage

    -- GitLab From 9061bb33b0acd3f45db7f0259a807a918d7ee93d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 May 2024 09:00:11 +0200 Subject: [PATCH 49/52] add csv writeout to create_report_pages.py --- ci/basop-pages/create_report_pages.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 9542e96894..1289ed5bb1 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -60,6 +60,7 @@ COLUMNS = ["testcase", "MLD", "MAXIMUM ABS DIFF"] def create_subpage( html_out, + csv_out, csv_current: str, csv_previous: str, id_current: int, @@ -69,6 +70,7 @@ def create_subpage( merged_reports = merge_and_cleanup_mld_reports( csv_current, csv_previous, id_current, id_previous ) + write_out_csv(merged_reports, merged_reports[0].keys(), csv_out) table_body = "\n".join( tr_from_row(row, id_current, id_previous) for row in merged_reports ) @@ -82,6 +84,14 @@ def create_subpage( f.write(new_subpage) +def write_out_csv(data, col_names, outfile): + with open(outfile, "w") as f: + writer = csv.DictWriter(f, col_names, delimiter=";") + writer.writeheader() + for row in data: + writer.writerow(row) + + def tr_from_row(row, id_current, id_previous): tr = list() for c in COLUMNS: @@ -167,6 +177,7 @@ def merge_tables(tbl1, tbl2, suffix1, suffix2, merge_key, other_keys): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("html_out") + parser.add_argument("csv_out") parser.add_argument("csv_current") parser.add_argument("csv_previous") parser.add_argument("id_current", type=int) @@ -176,6 +187,7 @@ if __name__ == "__main__": create_subpage( args.html_out, + args.csv_out, args.csv_current, args.csv_previous, args.id_current, -- GitLab From 1ee436bab0b86b09db48bcb448182538b8500c94 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 May 2024 09:10:02 +0200 Subject: [PATCH 50/52] add merged csv file in page --- ci/basop-pages/create_report_pages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index 1289ed5bb1..c653456f62 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -27,6 +27,7 @@ Comparing:
    -- GitLab From e7221fdbf438542e1365d6af264a8e214557d25d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 May 2024 09:11:18 +0200 Subject: [PATCH 51/52] remove temporary change --- ci/setup_pages.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/ci/setup_pages.py b/ci/setup_pages.py index 8be805116c..4754d09f5c 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -73,17 +73,7 @@ def get_artifacts_for_jobs_and_return_num_failed( failed_count = 0 for job in jobs: - print(os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) - # TODO: change back before merginG!! - # job_id = get_job_id( - # os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only - # ) - job_id = get_job_id( - "ci/setup_gitlab_pages_for_ci_result_presentation", - job, - project_id, - success_only, - ) + job_id = get_job_id( os.environ["CI_DEFAULT_BRANCH"], job, project_id, success_only) print(f"{job_id} - {job}") try: -- GitLab From 3c33c2b27356f00a8983ddf8e7036d858969059c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 May 2024 10:06:54 +0200 Subject: [PATCH 52/52] fix missing value handling + handle new columns --- ci/basop-pages/create_report_pages.py | 58 ++++++++++++++++++--------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/ci/basop-pages/create_report_pages.py b/ci/basop-pages/create_report_pages.py index c653456f62..ca14442edb 100644 --- a/ci/basop-pages/create_report_pages.py +++ b/ci/basop-pages/create_report_pages.py @@ -56,7 +56,11 @@ TD_TMPL_INCREASE = "{}" TD_TMPL_REDUCE = "{}" TR_TMPL = "{}" -COLUMNS = ["testcase", "MLD", "MAXIMUM ABS DIFF"] +# expected columns. actual columns are filtered from the incoming data later, this +# is mainly for controlling the order in the output table +COLUMNS = ["testcase", "Result", "MLD", "MAXIMUM ABS DIFF"] +COLUMNS_GLOBAL = COLUMNS[:1] +COLUMNS_DIFFERENTIAL = COLUMNS[1:] def create_subpage( @@ -95,24 +99,41 @@ def write_out_csv(data, col_names, outfile): def tr_from_row(row, id_current, id_previous): tr = list() - for c in COLUMNS: + + # pre-filter columns to handle case where new columns are added + # only include columns that are there in both data + columns_global = [c for c in COLUMNS_GLOBAL if c in row] + diff_col_tmpl = "{}-{}" + incoming_cols = row.keys() + columns_differential = [ + c + for c in COLUMNS_DIFFERENTIAL + if diff_col_tmpl.format(c, id_current) in incoming_cols + and diff_col_tmpl.format(c, id_previous) in incoming_cols + ] + + for c in columns_global: + # this is currently for the "testcase" column - here we don't compare, just one value is used + tr.append(TD_TMPL_NORMAL.format(row[c])) + for c in columns_differential: + # this is for all columns where we compare between current and previous run + prev = row[f"{c}-{id_previous}"] + curr = row[f"{c}-{id_current}"] + + # use red background if increase, green if decrease, white if same + td_tmpl = TD_TMPL_NORMAL try: - # this is for the "testcase" column - here we don't compare, just one value is used - tr.append(TD_TMPL_NORMAL.format(row[c])) - except KeyError: - # this is for all columns where we compare between current and previous run - prev = row[f"{c}-{id_previous}"] - curr = row[f"{c}-{id_current}"] - - # use red background if increase, green if decrease, white if same - td_tmpl = TD_TMPL_NORMAL if float(curr) > float(prev): td_tmpl = TD_TMPL_INCREASE if float(curr) < float(prev): td_tmpl = TD_TMPL_REDUCE + except ValueError: + # if we land here, one of the cells is not a number, this indicates a crash + # or some error in the scripts, so mark with red as well + td_tmpl = TD_TMPL_INCREASE - tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) - tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) + tr.append(td_tmpl.format(row[f"{c}-{id_previous}"])) + tr.append(td_tmpl.format(row[f"{c}-{id_current}"])) return TR_TMPL.format("\n".join(tr)) @@ -140,11 +161,12 @@ def merge_and_cleanup_mld_reports( # sort based on difference in MLD between current and previous run # put cases with "None" at the top of the list def sort_func(x): - return ( - float("inf") - if x[mld_col_curr] == "None" or x[mld_col_prev] == "None" - else float(x[mld_col_curr]) - float(x[mld_col_prev]) - ) + vals_missing = ["None", ""] + + if x[mld_col_curr] in vals_missing or x[mld_col_prev] in vals_missing: + return float("inf") + + return float(x[mld_col_curr]) - float(x[mld_col_prev]) merged = sorted(merged, key=sort_func, reverse=True) -- GitLab