Commit 6e0bb410 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/update-ci-on-main-pc' into 'main-pc' 

[CI] update CI setup with changes from main

See merge request !1473
parents aaa01f05 5eca6c4a

.gitlab-ci.yml

+81 −24
Original line number Diff line number Diff line
@@ -45,10 +45,12 @@ variables: 
  ENCODER_TEST: ""

  DELTA_ODG: ""

  COMPARE_DMX: ""

  SPLIT_COMPARISON: ""

  SKIP_REGRESSION_CHECK: ""

  FAILED_TESTCASES_LIST: "failed-testcases.txt"

  ERRORS_TESTCASES_LIST: "errors-testcases.txt"

  PYTEST_CACHE_ARTIFACT: "pytest_cache.zip"

  MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG"

  FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"

  CUT_COMMIT_FILE: "CuT-git-sha.txt"

  MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"
@@ -522,10 +524,13 @@ stages: 
  variables:

    # keep "mld" in artifact name for backwards compatibility reasons

    CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

    CSV_ARTIFACT_SPLIT: "mld--split--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

    MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv"

    PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html"

    SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"

    SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html"

    IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"

    IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME"

  script:

    - *print-common-info

    - *update-scripts-repo
@@ -545,14 +550,16 @@ stages: 
    - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then

    -   BUILD_WITH_DEBUG_MODE_INFO="true"

    - fi

    - *build-and-create-float-ref-outputs



    - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")

    - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"

    - summary_args="MLD DIFF SSNR ODG"

    - REPORT_ARG=""

    - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi

    - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi

    - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi

    - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi



    - *build-and-create-float-ref-outputs



    # DMX comparison only in manual job with no other metrics

    - if [ "$COMPARE_DMX" = "true" ]; then
@@ -566,10 +573,24 @@ stages: 
    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in $summary_args;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME --measures $summary_args

    ### create histograms

    - if [ "$SPLIT_COMPARISON" = "true" ]; then

    -   python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT

    - else

    -   python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME

    - fi



    # first for "whole" files comparison

    - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT



    - if [ "$SPLIT_COMPARISON" = "true" ]; then

    -   python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT --write-out-histograms

    -   python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT

    - else

    # touch files to suppress warning for missing artifacts

    -   touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT

    - fi



    - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

    -    id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)
@@ -609,9 +630,12 @@ stages: 
      - report.html

      - $PAGES_HTML_ARTIFACT_NAME

      - $CSV_ARTIFACT_NAME

      - $CSV_ARTIFACT_SPLIT

      - $MERGED_CSV_ARTIFACT_NAME

      - $SUMMARY_HTML_ARTIFACT_NAME

      - $SUMMARY_HTML_ARTIFACT_SPLIT

      - $IMAGES_ARTIFACT_NAME

      - $IMAGES_ARTIFACT_SPLIT

    expose_as: "pytest compare results"

    reports:

      junit:
@@ -753,7 +777,7 @@ stages: 
    - *print-common-info



    # create empty files for all artifacts to suppress warnings in case of no regressions found or all is BE

    - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAXIMUM_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAXIMUM_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv

    - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAX_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAX_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv

    - mkdir $IMAGES_ARTIFACT_NAME



    - set -euxo pipefail
@@ -803,9 +827,9 @@ stages: 
    # Store branch outputs for comparison

    - mv tests/dut tests/dut_branch



    # create the summary based on the branch

    - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME

    # create the summary based on the branch only

    - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT



    ### run main now

    - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
@@ -869,12 +893,12 @@ stages: 
      - $MERGE_TARGET_COMMIT_FILE

      - regressions_crashes.csv

      - regressions_MLD.csv

      - regressions_MAXIMUM_ABS_DIFF.csv

      - regressions_MAX_ABS_DIFF.csv

      - regressions_MIN_SSNR.csv

      - regressions_MIN_ODG.csv

      - improvements_crashes.csv

      - improvements_MLD.csv

      - improvements_MAXIMUM_ABS_DIFF.csv

      - improvements_MAX_ABS_DIFF.csv

      - improvements_MIN_SSNR.csv

      - improvements_MIN_ODG.csv

    expose_as: "pytest compare results"
@@ -967,8 +991,8 @@ stages: 


    # create summary

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in MLD DIFF SSNR ODG;do python3 scripts/create_histogram_summary.py report-diff.csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE --diff; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME

    - python3 scripts/create_histograms.py report-diff.csv $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT --write-out-histograms --no-bins

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT



    - exit 0
@@ -1209,6 +1233,7 @@ build-codec-linux-make: 
    - .build-job-linux

  tags:

    - ivas-basop-linux

  timeout: "10 minutes"

  script:

    - *print-common-info

    - *activate-Werror-linux
@@ -1260,6 +1285,7 @@ build-codec-sanitizers-linux: 
    - .rules-basis

  tags:

    - ivas-basop-linux

  timeout: "10 minutes"

  script:

    - *update-scripts-repo

    - *print-common-info
@@ -1277,7 +1303,7 @@ build-codec-windows-msbuild: 
      when: never

  extends:

    - .build-job-windows

  timeout: "7 minutes"

  timeout: "10 minutes"

  tags:

    - ivas-windows

  script:
@@ -1797,6 +1823,7 @@ ivas-pytest-compare_ref-long-enc: 
    - DUT_DECODER_PATH=./$REF_DECODER_PATH

    - TEST_SUITE="$LONG_TEST_SUITE_ENCODER"

    - LEVEL_SCALING=1.0

    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor

  

ivas-pytest-compare_ref-long-dec:
@@ -1808,6 +1835,7 @@ ivas-pytest-compare_ref-long-dec: 
    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=1.0

    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor

  

ivas-pytest-compare_ref-long-enc-lev-10:
@@ -1819,6 +1847,7 @@ ivas-pytest-compare_ref-long-enc-lev-10: 
    - DUT_DECODER_PATH=./$REF_DECODER_PATH

    - TEST_SUITE="$LONG_TEST_SUITE_ENCODER"

    - LEVEL_SCALING=0.3162

    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor

  

ivas-pytest-compare_ref-long-dec-lev-10:
@@ -1830,6 +1859,7 @@ ivas-pytest-compare_ref-long-dec-lev-10: 
    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=0.3162

    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor



ivas-pytest-compare_ref-long-enc-lev+10:
@@ -1841,6 +1871,7 @@ ivas-pytest-compare_ref-long-enc-lev+10: 
    - DUT_DECODER_PATH=./$REF_DECODER_PATH

    - TEST_SUITE="$LONG_TEST_SUITE_ENCODER"

    - LEVEL_SCALING=3.162

    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor

  

ivas-pytest-compare_ref-long-dec-lev+10:
@@ -1852,6 +1883,7 @@ ivas-pytest-compare_ref-long-dec-lev+10: 
    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=3.162

    - SPLIT_COMPARISON="true"

  <<: *ivas-pytest-anchor



ivas-smoke-test-saturation:
@@ -2208,9 +2240,8 @@ ivas-pytest-on-merge-request: 
  # this is a testing/maintenance mechanism to force getting the log history from a specific job id

  # see below in the concrete complexity jobs

  - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi

  - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - ls

  - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - public_dir="$CI_JOB_NAME-public"

  # if is needed to catch case when no artifact is there (first run), similarly as above

  - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/;  fi
@@ -2221,8 +2252,8 @@ ivas-pytest-on-merge-request: 
  ### 1.5.part: get the corresponding measurement from ivas-float-update

  - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id

  - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip

  - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true

  - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip

  - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true

  # add file to arguments only if the artifact could be retrieved to prevent error later.

  - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi
@@ -2235,8 +2266,6 @@ ivas-pytest-on-merge-request: 
  - mkdir $public_dir/logs

  # first move logs

  - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g")

  - echo $log_files

  - ls wmops/logs

  - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done

  # copy index page blueprint

  - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html
@@ -2244,7 +2273,16 @@ ivas-pytest-on-merge-request: 
  - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html

  # do separately here to avoid overwrite complaints by mv

  - mv -f ci/complexity_measurements/style.css ${public_dir}/

  - ls $public_dir



.complexity-measurements-report-summary: &complexity-measurements-report-summary

  - *print-results-banner

  - if [ $ret_val -eq 0 ]; then

  -   echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected."

  - elif [ $ret_val -eq 123 ]; then

  -   echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured."

  - else

  -   echo -e "Something went wrong in running the codec. Likely some modes were crashing."

  - fi



.complexity-template:

  extends:
@@ -2262,7 +2300,6 @@ ivas-pytest-on-merge-request: 
    - *complexity-measurements-setup

    # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present)

    - rm -rf COMPLEXITY/logs

    - which coan

  allow_failure:

    exit_codes:

      - 123
@@ -2286,6 +2323,7 @@ complexity-stereo-in-stereo-out: 
    - out_format=stereo

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-ism-in-binaural-out:
@@ -2303,6 +2341,7 @@ complexity-ism-in-binaural-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-ism-in-binaural_room_ir-out:
@@ -2320,6 +2359,7 @@ complexity-ism-in-binaural_room_ir-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-ism-in-ext-out:
@@ -2337,6 +2377,7 @@ complexity-ism-in-ext-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-sba-hoa3-in-hoa3-out:
@@ -2354,6 +2395,7 @@ complexity-sba-hoa3-in-hoa3-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-sba-hoa3-in-binaural-out:
@@ -2371,6 +2413,7 @@ complexity-sba-hoa3-in-binaural-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-sba-hoa3-in-binaural_room_ir-out:
@@ -2388,6 +2431,7 @@ complexity-sba-hoa3-in-binaural_room_ir-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-mc-in-7_1_4-out:
@@ -2405,6 +2449,7 @@ complexity-mc-in-7_1_4-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-mc-in-binaural-out:
@@ -2422,6 +2467,7 @@ complexity-mc-in-binaural-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-mc-in-binaural_room_ir-out:
@@ -2439,6 +2485,7 @@ complexity-mc-in-binaural_room_ir-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-masa-in-ext-out:
@@ -2456,6 +2503,7 @@ complexity-masa-in-ext-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-masa-in-binaural-out:
@@ -2473,6 +2521,7 @@ complexity-masa-in-binaural-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-masa-in-hoa3-out:
@@ -2490,6 +2539,7 @@ complexity-masa-in-hoa3-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



# complexity-omasa-in-ext-out:
@@ -2507,6 +2557,7 @@ complexity-masa-in-hoa3-out: 
#     - ret_val=0

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val



complexity-omasa-in-binaural-out:
@@ -2524,6 +2575,7 @@ complexity-omasa-in-binaural-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-omasa-in-hoa3-out:
@@ -2541,6 +2593,7 @@ complexity-omasa-in-hoa3-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-StereoDmxEVS-stereo-in-mono-out:
@@ -2558,6 +2611,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



# complexity-osba-in-ext-out:
@@ -2573,6 +2627,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: 
#     - ret_val=0

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val



complexity-osba-in-binaural-out:
@@ -2590,6 +2645,7 @@ complexity-osba-in-binaural-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



complexity-osba-in-binaural_room_ir-out:
@@ -2607,6 +2663,7 @@ complexity-osba-in-binaural_room_ir-out: 
    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



# job that sets up gitlab pages website