Commit f809f4c0 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'unify-complexity-measurements' into 'main' 

Unify complexity measurement CI code

See merge request !60
parents 6be4f848 c16f8e31

includes/complexity-jobs.yml

0 → 100644
+457 −0
Original line number Diff line number Diff line 
# ---------------------------------------------------------------

# Complexity measurement jobs

# ---------------------------------------------------------------



# create necessary environment

.complexity-measurements-setup:

  &complexity-measurements-setup

  ### 1. part: this is boilerplate code to make the gitlab pages presentation work

  - mkdir -p wmops/logs



  - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id



  # this is a testing/maintenance mechanism to force getting the log history from a specific job id

  # see below in the concrete complexity jobs

  - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi



  - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - public_dir="$CI_JOB_NAME-public"



  # if is needed to catch case when no artifact is there (first run), similarly as above

  # 1. check for public_dir being there as this might not be the case when artifact download failed

  # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts

  - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then

  -   mv $public_dir/* wmops/

  # check here if we have the split-by-levels files present - if not, fake them up with the existing global one

  # this is needed for the first run with split graphs on a branch where the global version did run previously

  # NOTE: checking only for level_1 file here as this should already be sufficient

  # NOTE2: also not chechking for RAM for same reason

  -   wmops_all_global="wmops/log_wmops_all.txt"

  -   ram_all_global="wmops/log_ram_all.txt"

  -   if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then

  -     declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw")

  -     for suffix in "${suffixes[@]}"; do

  -       cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt

  -       cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt

  -     done

  -   fi

  - fi



  - rm artifacts.zip

  - rm -rf $public_dir



  ### 1.5.part: get the corresponding measurement from ivas-codec main in BASOP

  # only do this for ivas-basop

  - if [ "$CI_PROJECT_ID" == "$PROJECT_ID_BASOP" ]; then

  # find id of same name job in float repo on branch main

  -   job_id=$(python3 ci/get_id_of_last_job_occurence.py main $CI_JOB_NAME $PROJECT_ID_FLOAT)

  -   echo $job_id

  -   curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$PROJECT_ID_FLOAT/jobs/$job_id/artifacts" --output artifacts_ref.zip

  -   unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true

  # add file to arguments only if the artifact could be retrieved to prevent error later.

  -   if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi

  - fi



 # prepare artifacts -> move to public directory

.complexity-measurements-prepare-artifacts:

  &complexity-measurements-prepare-artifacts

  - public_dir="$CI_JOB_NAME-public"

  - mkdir $public_dir

  - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/

  # move logfiles for links

  - mkdir $public_dir/logs

  # first move logs

  - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g")

  - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done

  - if [ "$CI_PROJECT_ID" == "$PROJECT_ID_FLOAT" ]; then mv wmops/logs/latest_WMOPS.csv $public_dir/logs/; fi

  # copy index page blueprint

  - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html

  # patch the format in the title

  - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html

  # do separately here to avoid overwrite complaints by mv

  - mv -f ci/complexity_measurements/style.css ${public_dir}/



.complexity-measurements-report-summary: &complexity-measurements-report-summary

  - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh

  - if [ $ret_val -eq 0 ]; then

  -   echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected."

  - elif [ $ret_val -eq 123 ]; then

  -   echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured."

  - else

  -   echo -e "Something went wrong in running the codec. Likely some modes were crashing."

  - fi



.complexity-template:

  extends:

    - .test-job-linux

  tags:

    # TODO: align runner tags between projects

    - ivas-linux-fast

  timeout: 3 hours 30 minutes

  stage: test

  variables:

    ret_val: 0

    GET_WMOPS_ARGS: ""

  before_script:

    - !reference [.job-linux, before_script]

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    - |

      if [ "$CI_PROJECT_ID" == "$PROJECT_ID_BASOP" ]; then

        bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

        GET_WMOPS_ARGS="mem_only"

      fi



    # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present)

    - rm -rf COMPLEXITY/logs



    - set -euxo pipefail

    - *complexity-measurements-setup

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"

    when: always

    expire_in: 2 week

    paths:

      - $CI_JOB_NAME-public

      - COMPLEXITY/logs



complexity-stereo-in-ext-out:

  extends:

    - .complexity-template

  rules:

    - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=stereo

    - out_format=ext

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - *complexity-measurements-report-summary

    - exit $ret_val



# complexity-ism-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 1 hour

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=ISM

#     - out_format=BINAURAL

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-ism-in-binaural_room_ir-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 2 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=ISM

#     - out_format=BINAURAL_ROOM_IR

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-ism-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 3 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=ISM

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-sba-hoa3-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 4 hours 30 minutes

#   script:

#     - in_format=HOA3

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-sba-hoa3-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 5 hours 30 minutes

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=HOA3

#     - out_format=BINAURAL

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-sba-hoa3-in-binaural_room_ir-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 6 hours 30 minutes

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=HOA3

#     - out_format=BINAURAL_ROOM_IR

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-mc-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 7 hours 30 minutes

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=MC

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-mc-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 10 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=MC

#     - out_format=BINAURAL

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-mc-in-binaural_room_ir-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 12 hours 30 minutes

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=MC

#     - out_format=BINAURAL_ROOM_IR

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-masa-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 15 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=MASA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-masa-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 16 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=MASA

#     - out_format=BINAURAL

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-masa-in-hoa3-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 17 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=MASA

#     - out_format=HOA3

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-omasa-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 18 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=OMASA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-omasa-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 20 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=OMASA

#     - out_format=BINAURAL

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-omasa-in-hoa3-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 22 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=OMASA

#     - out_format=HOA3

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-StereoDmxEVS-stereo-in-mono-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 24 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=StereoDmxEVS

#     - out_format=mono

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-osba-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 25 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=OSBA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-osba-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 28 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=OSBA

#     - out_format=BINAURAL

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

#

# complexity-osba-in-binaural_room_ir-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MANUAL_PIPELINE_TYPE == 'complexity' || $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 31 hours

#   variables:

#     JOB_ID_INJECT: ""

#   script:

#     - in_format=OSBA

#     - out_format=BINAURAL_ROOM_IR

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - *complexity-measurements-report-summary

#     - exit $ret_val

includes/default-variables-basop.yml

+0 −1
Original line number Diff line number Diff line 
variables:

  EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop"

  # TODO: cleanup and use global var defined in ivas-codec-ci

  FLOAT_REF_BRANCH: "ivas-float-update"

  FLOAT_REF_BRANCH_MERGE_SOURCE: ""

  SCRIPTS_DIR: "/usr/local/scripts"

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv"

includes/default-variables-float.yml

+1 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ variables: 
      - 'check-clipping'

      - 'test-branch-vs-input-passthrough'

      - 'coverage'

      - 'complexity'



  PYTEST_ARGS: ""

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv"

includes/default-variables.yml

+2 −0
Original line number Diff line number Diff line
@@ -15,3 +15,5 @@ variables: 
  BASOP_CI_SCRIPTS_BRANCH: "main"

  # overwrite this via .gitlab-ci.yml in the respective other repo to disable "(model from file)" testcases

  DISABLE_HRTF: "false"

  PROJECT_ID_FLOAT: "49"

  PROJECT_ID_BASOP: "77"

includes/rules.yml

+3 −0
Original line number Diff line number Diff line
@@ -19,6 +19,9 @@ 
      when: never

    - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

      when: never

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity'

      variables:

        IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH'

    - when: on_success



.rules-merge-request-to-main: