diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b6a36cbd544295548aef852d4e44574c477fb46..0d5c431553f0b4001862b81a62aa17a6d633c02a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2270,9 +2270,13 @@ coverage-test-on-main-scheduled: # still needed as long as no long MASA testvectors are there - .test-job-linux-needs-testv-dir tags: - - test-complexity-measurement + - ivas-linux-fast timeout: 3 hours 30 minutes stage: test + before_script: + - *print-common-info + - *update-ltv-repo + - *complexity-measurements-setup artifacts: name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA" when: always @@ -2280,89 +2284,164 @@ coverage-test-on-main-scheduled: paths: - $CI_JOB_NAME-public -complexity-stereo-in-stereo-out: +complexity-stereo-in-ext-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=stereo - - out_format=stereo + - out_format=ext - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-ism-in-binaural-out: +complexity-ism-in-binaural_room_ir-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 1 hour + # when: delayed + # start_in: 1 hour script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=ISM - - out_format=BINAURAL + - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-sba-hoa3-in-hoa3-out: +complexity-ism-in-ext-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 2 hours 30 minutes + # when: delayed + # start_in: 2 hour + script: + - in_format=ISM + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-sba-hoa3-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + # when: delayed + # start_in: 3 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=HOA3 - - out_format=HOA3 + - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-mc-in-7_1_4-out: +complexity-sba-hoa3-in-binaural_room_ir-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 4 hours + # when: delayed + # start_in: 4 hours + script: + - in_format=HOA3 + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-mc-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + # when: delayed + # start_in: 5 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=MC - - out_format=7_1_4 + - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val -complexity-masa-in-7_1_4-out: +complexity-mc-in-binaural_room_ir-out: extends: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 7 hours 30 minutes + # when: delayed + # start_in: 6 hours + script: + - in_format=MC + - out_format=BINAURAL_ROOM_IR + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + # when: delayed + # start_in: 7 hours + script: + - in_format=MASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-masa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + # when: delayed + # start_in: 8 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=MASA - - out_format=7_1_4 + - out_format=BINAURAL + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-ext-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + # when: delayed + # start_in: 9 hours + script: + - in_format=OMASA + - out_format=EXT + - ret_val=0 + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? + - *complexity-measurements-prepare-artifacts + - exit $ret_val + +complexity-omasa-in-binaural-out: + extends: + - .complexity-template + rules: + - if: $MEASURE_COMPLEXITY_LINUX + # when: delayed + # start_in: 10 hours + script: + - in_format=OMASA + - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$? - *complexity-measurements-prepare-artifacts @@ -2373,12 +2452,9 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - when: delayed - start_in: 8 hours 30 minutes + # when: delayed + # start_in: 11 hours script: - - *print-common-info - - *update-ltv-repo - - *complexity-measurements-setup - in_format=StereoDmxEVS - out_format=mono - ret_val=0 diff --git a/ci/complexity_measurements/check_for_changes.py b/ci/complexity_measurements/check_for_changes.py index a6839ee59a53f97c7816ce29e95836ab9302549b..50deb76e08202f6b83f2f5ee2afe7499af2cc866 100644 --- a/ci/complexity_measurements/check_for_changes.py +++ b/ci/complexity_measurements/check_for_changes.py @@ -26,10 +26,27 @@ def check_linewise_logfile(filepath, cols): contents = [line for line in csv.reader(f, delimiter=" ")] curr = contents[-1] - prev = contents[-2] + try: + prev = contents[-2] + except IndexError: + # this is for handling first runs of new complexity jobs -> only one line present + # do not report any changes then + prev = curr - change_ratios = [abs(float(curr[i]) / float(prev[i]) - 1) > THRESH for i in cols] - changes_found = any(change_ratios) + changes_found = False + for c in cols: + if curr[c] == prev[c]: + abs_perc_change = 0 + else: + try: + abs_perc_change = abs(float(curr[c]) / float(prev[c]) - 1) + except ZeroDivisionError: + # in some cases, such as missing instrumentation, values can be zero, so catch that here + # from the first if we know that curr can not be 0 too -> report change + abs_perc_change = THRESH + 1 + if abs_perc_change > THRESH: + changes_found = True + break if changes_found: print("Previous log line:", prev) diff --git a/ci/complexity_measurements/getWmops.sh b/ci/complexity_measurements/getWmops.sh index c069e2d57b16ada134eba8f1fa9a9eaab5f9c2a5..a3d4710f2cc0b222cc49d52f1b094041a50643ac 100755 --- a/ci/complexity_measurements/getWmops.sh +++ b/ci/complexity_measurements/getWmops.sh @@ -35,7 +35,7 @@ if [ $# -ne 2 ]; then fi ivas_format=$1 -output_format=$2 +output_format="$2" date=`date +%Y%m%d` # used for log-file file ending shortDate=`date "+%b %d" | sed -e "s/\ /_/g"` # stored in the log-file diff --git a/ci/complexity_measurements/style.css b/ci/complexity_measurements/style.css index 44fb55de46125fbad0e32bceb11b3395f355fbd1..764ed0d94ff13958cbb612872e9bd9054e193387 100755 --- a/ci/complexity_measurements/style.css +++ b/ci/complexity_measurements/style.css @@ -38,7 +38,7 @@ hr { } .graph-container { margin: 0 auto; - width:800px; + width: 1600px; } .message-box { margin-top: 2em; @@ -48,62 +48,62 @@ hr { } #wmops-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops_per_op-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops-48kHz-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops_per_op-48kHz-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops_basop_per_op-graph { height:500px; - width:800px; + width:1600px; float:left; } #wmops-graph-basop { height:500px; - width:800px; + width:1600px; float:left; } #conversion_factors_basop_flc { height:500px; - width:800px; + width:1600px; float:left; } #ram-graph { height:500px; - width:800px; + width:1600px; float:left; } #ram-graph-basop { height:500px; - width:800px; + width:1600px; float:left; } #rom-graph { height:500px; - width:800px; + width:1600px; float:left; } #rom-graph-basop{ height:500px; - width:800px; + width:1600px; float:left; } #prom-graph { height:500px; - width:800px; + width:1600px; float:left; } #tooltip { diff --git a/ci/setup_pages.py b/ci/setup_pages.py index f131dfb4e45550f5e5503fa9169cf44fb8694e87..07b7c7fe68a76c1f1b3575d887ab352603136137 100755 --- a/ci/setup_pages.py +++ b/ci/setup_pages.py @@ -13,13 +13,27 @@ PROJECT_ID_BASOP_REPO = 77 JOBS_FLOAT_REPO = [ + # old ones no longer running -> replaced by "ext" jobs, remove after some time "complexity-stereo-in-stereo-out", "complexity-ism-in-binaural-out", "complexity-sba-hoa3-in-hoa3-out", "complexity-mc-in-7_1_4-out", "complexity-masa-in-7_1_4-out", - "complexity-StereoDmxEVS-stereo-in-mono-out", "coverage-test-on-main-scheduled", + # current ones + "complexity-stereo-in-ext-out", + "complexity-ism-in-binaural_room_ir-out", + "complexity-ism-in-ext-out", + "complexity-sba-hoa3-in-ext-out", + "complexity-sba-hoa3-in-binaural_room_ir-out", + "complexity-mc-in-ext-out", + "complexity-mc-in-binaural_room_ir-out", + "complexity-masa-in-ext-out", + "complexity-masa-in-binaural-out", + "complexity-omasa-in-ext-out", + "complexity-omasa-in-binaural-out", + # "timeless" jobs (survivors from the old jobs) + "complexity-StereoDmxEVS-stereo-in-mono-out", ] JOBS_BASOP_REPO = [ "ivas-pytest-mld-long-dec",