From 20b46b5536c5e6e7eba0c460acac0223dd10e29d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 17:30:51 +0200 Subject: [PATCH 1/4] add new complexity jobs and increase width of plots --- .gitlab-ci.yml | 140 +++++++++++++++++++------ ci/complexity_measurements/getWmops.sh | 2 +- ci/complexity_measurements/style.css | 26 ++--- ci/setup_pages.py | 16 ++- 4 files changed, 137 insertions(+), 47 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b6a36cbd5..fc9836e0aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2270,9 +2270,13 @@ coverage-test-on-main-scheduled: # still needed as long as no long MASA testvectors are there - .test-job-linux-needs-testv-dir tags: - - test-complexity-measurement + - ivas-linux-fast timeout: 3 hours 30 minutes stage: test + before_script: + - *print-common-info + - *update-ltv-repo + - *complexity-measurements-setup artifacts: name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA" when: always @@ -2280,23 +2284,20 @@ coverage-test-on-main-scheduled: paths: - $CI_JOB_NAME-public -complexity-stereo-in-stereo-out: +complexity-stereo-in-ext-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=stereo - - out_format=stereo + - out_format=ext - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-ism-in-binaural-out: +complexity-ism-in-binaural_room_ir-out: extends: - .complexity-template rules: @@ -2304,35 +2305,44 @@ complexity-ism-in-binaural-out: when: delayed start_in: 1 hour script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=ISM - - out_format=BINAURAL + - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-sba-hoa3-in-hoa3-out: +complexity-ism-in-ext-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX when: delayed - start_in: 2 hours 30 minutes + start_in: 2 hour + script: + - in_format=ISM + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 3 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=HOA3 - - out_format=HOA3 + - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-mc-in-7_1_4-out: +complexity-sba-hoa3-in-binaural_room_ir-out: extends: - .complexity-template rules: @@ -2340,29 +2350,98 @@ complexity-mc-in-7_1_4-out: when: delayed start_in: 4 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup + - in_format=HOA3 + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 5 hours + script: - in_format=MC - - out_format=7_1_4 + - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-masa-in-7_1_4-out: +complexity-mc-in-binaural_room_ir-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX when: delayed - start_in: 7 hours 30 minutes + start_in: 6 hours + script: + - in_format=MC + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 7 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=MASA - - out_format=7_1_4 + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 8 hours + script: + - in_format=MASA + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 9 hours + script: + - in_format=OMASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + when: delayed + start_in: 10 hours + script: + - in_format=OMASA + - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts @@ -2374,11 +2453,8 @@ complexity-StereoDmxEVS-stereo-in-mono-out: rules: - if: $MEASURE_COMPLEXITY_LINUX when: delayed - start_in: 8 hours 30 minutes + start_in: 11 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=StereoDmxEVS - out_format=mono - ret_val=0 diff --git a/ci/complexity_measurements/getWmops.sh b/ci/complexity_measurements/getWmops.sh index c069e2d57b..a3d4710f2c 100755 --- a/ci/complexity_measurements/getWmops.sh +++ b/ci/complexity_measurements/getWmops.sh @@ -35,7 +35,7 @@ if [ $# -ne 2 ]; then fi ivas_format=$1 -output_format=$2 +output_format="$2" date=`date +%Y%m%d` # used for log-file file ending shortDate=`date "+%b %d" | sed -e "s/\ /_/g"` # stored in the log-file diff --git a/ci/complexity_measurements/style.css b/ci/complexity_measurements/style.css index 44fb55de46..764ed0d94f 100755 --- a/ci/complexity_measurements/style.css +++ b/ci/complexity_measurements/style.css @@ -38,7 +38,7 @@ hr { } .graph-container { margin: 0 auto; - width:800px; + width: 1600px; } .message-box { margin-top: 2em; @@ -48,62 +48,62 @@ hr { } #wmops-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops_per_op-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops-48kHz-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops_per_op-48kHz-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops_basop_per_op-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops-graph-basop { height:500px; - width:800px; + width:1600px; float:left; } #conversion_factors_basop_flc { height:500px; - width:800px; + width:1600px; float:left; } #ram-graph { height:500px; - width:800px; + width:1600px; float:left; } #ram-graph-basop { height:500px; - width:800px; + width:1600px; float:left; } #rom-graph { height:500px; - width:800px; + width:1600px; float:left; } #rom-graph-basop{ height:500px; - width:800px; + width:1600px; float:left; } #prom-graph { height:500px; - width:800px; + width:1600px; float:left; } #tooltip { diff --git a/ci/setup_pages.py b/ci/setup_pages.py index f131dfb4e4..07b7c7fe68 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -13,13 +13,27 @@ PROJECT_ID_BASOP_REPO = 77 JOBS_FLOAT_REPO = [ + # old ones no longer running -> replaced by "ext" jobs, remove after some time "complexity-stereo-in-stereo-out", "complexity-ism-in-binaural-out", "complexity-sba-hoa3-in-hoa3-out", "complexity-mc-in-7_1_4-out", "complexity-masa-in-7_1_4-out", - "complexity-StereoDmxEVS-stereo-in-mono-out", "coverage-test-on-main-scheduled", + # current ones + "complexity-stereo-in-ext-out", + "complexity-ism-in-binaural_room_ir-out", + "complexity-ism-in-ext-out", + "complexity-sba-hoa3-in-ext-out", + "complexity-sba-hoa3-in-binaural_room_ir-out", + "complexity-mc-in-ext-out", + "complexity-mc-in-binaural_room_ir-out", + "complexity-masa-in-ext-out", + "complexity-masa-in-binaural-out", + "complexity-omasa-in-ext-out", + "complexity-omasa-in-binaural-out", + # "timeless" jobs (survivors from the old jobs) + "complexity-StereoDmxEVS-stereo-in-mono-out", ] JOBS_BASOP_REPO = [ "ivas-pytest-mld-long-dec", -- GitLab From 9a31c518b4e5dbb2562b4aab31174099a510df64 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 17:34:05 +0200 Subject: [PATCH 2/4] handle first run of complexity job in check_for_changes --- ci/complexity_measurements/check_for_changes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/complexity_measurements/check_for_changes.py b/ci/complexity_measurements/check_for_changes.py index a6839ee59a..35a1d876dc 100644 --- a/ci/complexity_measurements/check_for_changes.py +++ b/ci/complexity_measurements/check_for_changes.py @@ -26,7 +26,12 @@ def check_linewise_logfile(filepath, cols): contents = [line for line in csv.reader(f, delimiter=" ")] curr = contents[-1] - prev = contents[-2] + try: + prev = contents[-2] + except IndexError: + # this is for handling first runs of new complexity jobs -> only one line present + # do not report any changes then + prev = curr change_ratios = [abs(float(curr[i]) / float(prev[i]) - 1) > THRESH for i in cols] changes_found = any(change_ratios) -- GitLab From 23b744034891965d5d64e14961a176e8502fc6f8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 11 Jul 2024 18:03:11 +0200 Subject: [PATCH 3/4] handle zero values that cause division error --- ci/complexity_measurements/check_for_changes.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ci/complexity_measurements/check_for_changes.py b/ci/complexity_measurements/check_for_changes.py index 35a1d876dc..50deb76e08 100644 --- a/ci/complexity_measurements/check_for_changes.py +++ b/ci/complexity_measurements/check_for_changes.py @@ -33,8 +33,20 @@ def check_linewise_logfile(filepath, cols): # do not report any changes then prev = curr - change_ratios = [abs(float(curr[i]) / float(prev[i]) - 1) > THRESH for i in cols] - changes_found = any(change_ratios) + changes_found = False + for c in cols: + if curr[c] == prev[c]: + abs_perc_change = 0 + else: + try: + abs_perc_change = abs(float(curr[c]) / float(prev[c]) - 1) + except ZeroDivisionError: + # in some cases, such as missing instrumentation, values can be zero, so catch that here + # from the first if we know that curr can not be 0 too -> report change + abs_perc_change = THRESH + 1 + if abs_perc_change > THRESH: + changes_found = True + break if changes_found: print("Previous log line:", prev) -- GitLab From 32e76195913e2c06ad40084bf393f53f61799114 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 12 Jul 2024 09:29:50 +0200 Subject: [PATCH 4/4] temporarily disable delayed job starts for testing --- .gitlab-ci.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc9836e0aa..0d5c431553 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2302,8 +2302,8 @@ complexity-ism-in-binaural_room_ir-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 1 hour + # when: delayed + # start_in: 1 hour script: - in_format=ISM - out_format=BINAURAL_ROOM_IR @@ -2317,8 +2317,8 @@ complexity-ism-in-ext-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 2 hour + # when: delayed + # start_in: 2 hour script: - in_format=ISM - out_format=EXT @@ -2332,8 +2332,8 @@ complexity-sba-hoa3-in-ext-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 3 hours + # when: delayed + # start_in: 3 hours script: - in_format=HOA3 - out_format=EXT @@ -2347,8 +2347,8 @@ complexity-sba-hoa3-in-binaural_room_ir-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 4 hours + # when: delayed + # start_in: 4 hours script: - in_format=HOA3 - out_format=BINAURAL_ROOM_IR @@ -2362,8 +2362,8 @@ complexity-mc-in-ext-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 5 hours + # when: delayed + # start_in: 5 hours script: - in_format=MC - out_format=EXT @@ -2377,8 +2377,8 @@ complexity-mc-in-binaural_room_ir-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 6 hours + # when: delayed + # start_in: 6 hours script: - in_format=MC - out_format=BINAURAL_ROOM_IR @@ -2392,8 +2392,8 @@ complexity-masa-in-ext-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 7 hours + # when: delayed + # start_in: 7 hours script: - in_format=MASA - out_format=EXT @@ -2407,8 +2407,8 @@ complexity-masa-in-binaural-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 8 hours + # when: delayed + # start_in: 8 hours script: - in_format=MASA - out_format=BINAURAL @@ -2422,8 +2422,8 @@ complexity-omasa-in-ext-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 9 hours + # when: delayed + # start_in: 9 hours script: - in_format=OMASA - out_format=EXT @@ -2437,8 +2437,8 @@ complexity-omasa-in-binaural-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 10 hours + # when: delayed + # start_in: 10 hours script: - in_format=OMASA - out_format=BINAURAL @@ -2452,8 +2452,8 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 11 hours + # when: delayed + # start_in: 11 hours script: - in_format=StereoDmxEVS - out_format=mono -- GitLab