Loading .gitlab-ci.yml +81 −24 Original line number Diff line number Diff line Loading @@ -45,10 +45,12 @@ variables: ENCODER_TEST: "" DELTA_ODG: "" COMPARE_DMX: "" SPLIT_COMPARISON: "" SKIP_REGRESSION_CHECK: "" FAILED_TESTCASES_LIST: "failed-testcases.txt" ERRORS_TESTCASES_LIST: "errors-testcases.txt" PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" CUT_COMMIT_FILE: "CuT-git-sha.txt" MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" Loading Loading @@ -522,10 +524,13 @@ stages: variables: # keep "mld" in artifact name for backwards compatibility reasons CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - *print-common-info - *update-scripts-repo Loading @@ -545,14 +550,16 @@ stages: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - *build-and-create-float-ref-outputs - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" - summary_args="MLD DIFF SSNR ODG" - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi - *build-and-create-float-ref-outputs # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then Loading @@ -566,10 +573,24 @@ stages: - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG - mkdir $IMAGES_ARTIFACT_NAME - for MEASURE in $summary_args;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $summary_args ### create histograms - if [ "$SPLIT_COMPARISON" = "true" ]; then - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT - else - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - fi # first for "whole" files comparison - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - if [ "$SPLIT_COMPARISON" = "true" ]; then - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT - else # touch files to suppress warning for missing artifacts - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT - fi - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) Loading Loading @@ -609,9 +630,12 @@ stages: - report.html - $PAGES_HTML_ARTIFACT_NAME - $CSV_ARTIFACT_NAME - $CSV_ARTIFACT_SPLIT - $MERGED_CSV_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_SPLIT - $IMAGES_ARTIFACT_NAME - $IMAGES_ARTIFACT_SPLIT expose_as: "pytest compare results" reports: junit: Loading Loading @@ -753,7 +777,7 @@ stages: - *print-common-info # create empty files for all artifacts to suppress warnings in case of no regressions found or all is BE - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAXIMUM_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAXIMUM_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAX_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAX_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv - mkdir $IMAGES_ARTIFACT_NAME - set -euxo pipefail Loading Loading @@ -803,9 +827,9 @@ stages: # Store branch outputs for comparison - mv tests/dut tests/dut_branch # create the summary based on the branch - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME # create the summary based on the branch only - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT ### run main now - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME Loading Loading @@ -869,12 +893,12 @@ stages: - $MERGE_TARGET_COMMIT_FILE - regressions_crashes.csv - regressions_MLD.csv - regressions_MAXIMUM_ABS_DIFF.csv - regressions_MAX_ABS_DIFF.csv - regressions_MIN_SSNR.csv - regressions_MIN_ODG.csv - improvements_crashes.csv - improvements_MLD.csv - improvements_MAXIMUM_ABS_DIFF.csv - improvements_MAX_ABS_DIFF.csv - improvements_MIN_SSNR.csv - improvements_MIN_ODG.csv expose_as: "pytest compare results" Loading Loading @@ -967,8 +991,8 @@ stages: # create summary - mkdir $IMAGES_ARTIFACT_NAME - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py report-diff.csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE --diff; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - exit 0 Loading Loading @@ -1209,6 +1233,7 @@ build-codec-linux-make: - .build-job-linux tags: - ivas-basop-linux timeout: "10 minutes" script: - *print-common-info - *activate-Werror-linux Loading Loading @@ -1260,6 +1285,7 @@ build-codec-sanitizers-linux: - .rules-basis tags: - ivas-basop-linux timeout: "10 minutes" script: - *update-scripts-repo - *print-common-info Loading @@ -1277,7 +1303,7 @@ build-codec-windows-msbuild: when: never extends: - .build-job-windows timeout: "7 minutes" timeout: "10 minutes" tags: - ivas-windows script: Loading Loading @@ -1797,6 +1823,7 @@ ivas-pytest-compare_ref-long-enc: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec: Loading @@ -1808,6 +1835,7 @@ ivas-pytest-compare_ref-long-dec: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev-10: Loading @@ -1819,6 +1847,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev-10: Loading @@ -1830,6 +1859,7 @@ ivas-pytest-compare_ref-long-dec-lev-10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev+10: Loading @@ -1841,6 +1871,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev+10: Loading @@ -1852,6 +1883,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-smoke-test-saturation: Loading Loading @@ -2208,9 +2240,8 @@ ivas-pytest-on-merge-request: # this is a testing/maintenance mechanism to force getting the log history from a specific job id # see below in the concrete complexity jobs - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - ls - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - public_dir="$CI_JOB_NAME-public" # if is needed to catch case when no artifact is there (first run), similarly as above - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/; fi Loading @@ -2221,8 +2252,8 @@ ivas-pytest-on-merge-request: ### 1.5.part: get the corresponding measurement from ivas-float-update - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi Loading @@ -2235,8 +2266,6 @@ ivas-pytest-on-merge-request: - mkdir $public_dir/logs # first move logs - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") - echo $log_files - ls wmops/logs - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done # copy index page blueprint - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html Loading @@ -2244,7 +2273,16 @@ ivas-pytest-on-merge-request: - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html # do separately here to avoid overwrite complaints by mv - mv -f ci/complexity_measurements/style.css ${public_dir}/ - ls $public_dir .complexity-measurements-report-summary: &complexity-measurements-report-summary - *print-results-banner - if [ $ret_val -eq 0 ]; then - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." - elif [ $ret_val -eq 123 ]; then - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." - else - echo -e "Something went wrong in running the codec. Likely some modes were crashing." - fi .complexity-template: extends: Loading @@ -2262,7 +2300,6 @@ ivas-pytest-on-merge-request: - *complexity-measurements-setup # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present) - rm -rf COMPLEXITY/logs - which coan allow_failure: exit_codes: - 123 Loading @@ -2286,6 +2323,7 @@ complexity-stereo-in-stereo-out: - out_format=stereo - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-ism-in-binaural-out: Loading @@ -2303,6 +2341,7 @@ complexity-ism-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-ism-in-binaural_room_ir-out: Loading @@ -2320,6 +2359,7 @@ complexity-ism-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-ism-in-ext-out: Loading @@ -2337,6 +2377,7 @@ complexity-ism-in-ext-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-sba-hoa3-in-hoa3-out: Loading @@ -2354,6 +2395,7 @@ complexity-sba-hoa3-in-hoa3-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-sba-hoa3-in-binaural-out: Loading @@ -2371,6 +2413,7 @@ complexity-sba-hoa3-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-sba-hoa3-in-binaural_room_ir-out: Loading @@ -2388,6 +2431,7 @@ complexity-sba-hoa3-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-mc-in-7_1_4-out: Loading @@ -2405,6 +2449,7 @@ complexity-mc-in-7_1_4-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-mc-in-binaural-out: Loading @@ -2422,6 +2467,7 @@ complexity-mc-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-mc-in-binaural_room_ir-out: Loading @@ -2439,6 +2485,7 @@ complexity-mc-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-masa-in-ext-out: Loading @@ -2456,6 +2503,7 @@ complexity-masa-in-ext-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-masa-in-binaural-out: Loading @@ -2473,6 +2521,7 @@ complexity-masa-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-masa-in-hoa3-out: Loading @@ -2490,6 +2539,7 @@ complexity-masa-in-hoa3-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val # complexity-omasa-in-ext-out: Loading @@ -2507,6 +2557,7 @@ complexity-masa-in-hoa3-out: # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - *complexity-measurements-report-summary # - exit $ret_val complexity-omasa-in-binaural-out: Loading @@ -2524,6 +2575,7 @@ complexity-omasa-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-omasa-in-hoa3-out: Loading @@ -2541,6 +2593,7 @@ complexity-omasa-in-hoa3-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-StereoDmxEVS-stereo-in-mono-out: Loading @@ -2558,6 +2611,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val # complexity-osba-in-ext-out: Loading @@ -2573,6 +2627,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - *complexity-measurements-report-summary # - exit $ret_val complexity-osba-in-binaural-out: Loading @@ -2590,6 +2645,7 @@ complexity-osba-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-osba-in-binaural_room_ir-out: Loading @@ -2607,6 +2663,7 @@ complexity-osba-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val # job that sets up gitlab pages website Loading Loading
.gitlab-ci.yml +81 −24 Original line number Diff line number Diff line Loading @@ -45,10 +45,12 @@ variables: ENCODER_TEST: "" DELTA_ODG: "" COMPARE_DMX: "" SPLIT_COMPARISON: "" SKIP_REGRESSION_CHECK: "" FAILED_TESTCASES_LIST: "failed-testcases.txt" ERRORS_TESTCASES_LIST: "errors-testcases.txt" PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" CUT_COMMIT_FILE: "CuT-git-sha.txt" MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" Loading Loading @@ -522,10 +524,13 @@ stages: variables: # keep "mld" in artifact name for backwards compatibility reasons CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv" MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv" PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - *print-common-info - *update-scripts-repo Loading @@ -545,14 +550,16 @@ stages: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - *build-and-create-float-ref-outputs - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" - summary_args="MLD DIFF SSNR ODG" - REPORT_ARG="" - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi - *build-and-create-float-ref-outputs # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then Loading @@ -566,10 +573,24 @@ stages: - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG - mkdir $IMAGES_ARTIFACT_NAME - for MEASURE in $summary_args;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $summary_args ### create histograms - if [ "$SPLIT_COMPARISON" = "true" ]; then - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT - else - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME - fi # first for "whole" files comparison - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - if [ "$SPLIT_COMPARISON" = "true" ]; then - python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT - else # touch files to suppress warning for missing artifacts - touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT - fi - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) Loading Loading @@ -609,9 +630,12 @@ stages: - report.html - $PAGES_HTML_ARTIFACT_NAME - $CSV_ARTIFACT_NAME - $CSV_ARTIFACT_SPLIT - $MERGED_CSV_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_NAME - $SUMMARY_HTML_ARTIFACT_SPLIT - $IMAGES_ARTIFACT_NAME - $IMAGES_ARTIFACT_SPLIT expose_as: "pytest compare results" reports: junit: Loading Loading @@ -753,7 +777,7 @@ stages: - *print-common-info # create empty files for all artifacts to suppress warnings in case of no regressions found or all is BE - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAXIMUM_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAXIMUM_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAX_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAX_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv - mkdir $IMAGES_ARTIFACT_NAME - set -euxo pipefail Loading Loading @@ -803,9 +827,9 @@ stages: # Store branch outputs for comparison - mv tests/dut tests/dut_branch # create the summary based on the branch - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME # create the summary based on the branch only - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT ### run main now - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME Loading Loading @@ -869,12 +893,12 @@ stages: - $MERGE_TARGET_COMMIT_FILE - regressions_crashes.csv - regressions_MLD.csv - regressions_MAXIMUM_ABS_DIFF.csv - regressions_MAX_ABS_DIFF.csv - regressions_MIN_SSNR.csv - regressions_MIN_ODG.csv - improvements_crashes.csv - improvements_MLD.csv - improvements_MAXIMUM_ABS_DIFF.csv - improvements_MAX_ABS_DIFF.csv - improvements_MIN_SSNR.csv - improvements_MIN_ODG.csv expose_as: "pytest compare results" Loading Loading @@ -967,8 +991,8 @@ stages: # create summary - mkdir $IMAGES_ARTIFACT_NAME - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py report-diff.csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE --diff; done - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT - exit 0 Loading Loading @@ -1209,6 +1233,7 @@ build-codec-linux-make: - .build-job-linux tags: - ivas-basop-linux timeout: "10 minutes" script: - *print-common-info - *activate-Werror-linux Loading Loading @@ -1260,6 +1285,7 @@ build-codec-sanitizers-linux: - .rules-basis tags: - ivas-basop-linux timeout: "10 minutes" script: - *update-scripts-repo - *print-common-info Loading @@ -1277,7 +1303,7 @@ build-codec-windows-msbuild: when: never extends: - .build-job-windows timeout: "7 minutes" timeout: "10 minutes" tags: - ivas-windows script: Loading Loading @@ -1797,6 +1823,7 @@ ivas-pytest-compare_ref-long-enc: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec: Loading @@ -1808,6 +1835,7 @@ ivas-pytest-compare_ref-long-dec: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=1.0 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev-10: Loading @@ -1819,6 +1847,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev-10: Loading @@ -1830,6 +1859,7 @@ ivas-pytest-compare_ref-long-dec-lev-10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=0.3162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-enc-lev+10: Loading @@ -1841,6 +1871,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: - DUT_DECODER_PATH=./$REF_DECODER_PATH - TEST_SUITE="$LONG_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-pytest-compare_ref-long-dec-lev+10: Loading @@ -1852,6 +1883,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE="$LONG_TEST_SUITE" - LEVEL_SCALING=3.162 - SPLIT_COMPARISON="true" <<: *ivas-pytest-anchor ivas-smoke-test-saturation: Loading Loading @@ -2208,9 +2240,8 @@ ivas-pytest-on-merge-request: # this is a testing/maintenance mechanism to force getting the log history from a specific job id # see below in the concrete complexity jobs - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - ls - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - public_dir="$CI_JOB_NAME-public" # if is needed to catch case when no artifact is there (first run), similarly as above - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/; fi Loading @@ -2221,8 +2252,8 @@ ivas-pytest-on-merge-request: ### 1.5.part: get the corresponding measurement from ivas-float-update - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi Loading @@ -2235,8 +2266,6 @@ ivas-pytest-on-merge-request: - mkdir $public_dir/logs # first move logs - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") - echo $log_files - ls wmops/logs - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done # copy index page blueprint - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html Loading @@ -2244,7 +2273,16 @@ ivas-pytest-on-merge-request: - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html # do separately here to avoid overwrite complaints by mv - mv -f ci/complexity_measurements/style.css ${public_dir}/ - ls $public_dir .complexity-measurements-report-summary: &complexity-measurements-report-summary - *print-results-banner - if [ $ret_val -eq 0 ]; then - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." - elif [ $ret_val -eq 123 ]; then - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." - else - echo -e "Something went wrong in running the codec. Likely some modes were crashing." - fi .complexity-template: extends: Loading @@ -2262,7 +2300,6 @@ ivas-pytest-on-merge-request: - *complexity-measurements-setup # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present) - rm -rf COMPLEXITY/logs - which coan allow_failure: exit_codes: - 123 Loading @@ -2286,6 +2323,7 @@ complexity-stereo-in-stereo-out: - out_format=stereo - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-ism-in-binaural-out: Loading @@ -2303,6 +2341,7 @@ complexity-ism-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-ism-in-binaural_room_ir-out: Loading @@ -2320,6 +2359,7 @@ complexity-ism-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-ism-in-ext-out: Loading @@ -2337,6 +2377,7 @@ complexity-ism-in-ext-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-sba-hoa3-in-hoa3-out: Loading @@ -2354,6 +2395,7 @@ complexity-sba-hoa3-in-hoa3-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-sba-hoa3-in-binaural-out: Loading @@ -2371,6 +2413,7 @@ complexity-sba-hoa3-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-sba-hoa3-in-binaural_room_ir-out: Loading @@ -2388,6 +2431,7 @@ complexity-sba-hoa3-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-mc-in-7_1_4-out: Loading @@ -2405,6 +2449,7 @@ complexity-mc-in-7_1_4-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-mc-in-binaural-out: Loading @@ -2422,6 +2467,7 @@ complexity-mc-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-mc-in-binaural_room_ir-out: Loading @@ -2439,6 +2485,7 @@ complexity-mc-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-masa-in-ext-out: Loading @@ -2456,6 +2503,7 @@ complexity-masa-in-ext-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-masa-in-binaural-out: Loading @@ -2473,6 +2521,7 @@ complexity-masa-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-masa-in-hoa3-out: Loading @@ -2490,6 +2539,7 @@ complexity-masa-in-hoa3-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val # complexity-omasa-in-ext-out: Loading @@ -2507,6 +2557,7 @@ complexity-masa-in-hoa3-out: # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - *complexity-measurements-report-summary # - exit $ret_val complexity-omasa-in-binaural-out: Loading @@ -2524,6 +2575,7 @@ complexity-omasa-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-omasa-in-hoa3-out: Loading @@ -2541,6 +2593,7 @@ complexity-omasa-in-hoa3-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-StereoDmxEVS-stereo-in-mono-out: Loading @@ -2558,6 +2611,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val # complexity-osba-in-ext-out: Loading @@ -2573,6 +2627,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - *complexity-measurements-report-summary # - exit $ret_val complexity-osba-in-binaural-out: Loading @@ -2590,6 +2645,7 @@ complexity-osba-in-binaural-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val complexity-osba-in-binaural_room_ir-out: Loading @@ -2607,6 +2663,7 @@ complexity-osba-in-binaural_room_ir-out: - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - *complexity-measurements-report-summary - exit $ret_val # job that sets up gitlab pages website Loading
