diff --git a/main-basop.yml b/main-basop.yml index b03a9983a8baf37e6a71cfdce0ea65ce8c33d9f0..bdca57dab278781b972c966b16336d9ab04609f6 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -181,6 +181,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' variables: IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'runner-speed-test' + variables: + IVAS_PIPELINE_NAME: 'Runner speed test: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' @@ -1264,8 +1267,8 @@ ivas-pytest-on-merge-request: expose_as: "pytest ivas results" reports: junit: - - $REPORT_XML + - $REPORT_XML split-rendering-pytest-on-merge-request: extends: - .test-job-linux-needs-testv-dir @@ -2543,6 +2546,125 @@ backup-long-term-job-logs: - exit 1 - fi +# Run +runner-speed-test: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "runner-speed-test" + stage: test + parallel: + matrix: + - RUNNER: +# - test-nokia-basop-linux-runner-2 +# - test-nokia-basop-linux-runner-3 +# - test-dolby-linux-runner +# - test-dolby-linux-runner-2 +# - test-ericsson-linux-runner +# - test-ericsson-linux-runner-3 + - test-ericsson-linux-runner-5 +# - test-ericsson-linux-runner-7 +# - test-ericsson-linux-runner-8 +# - test-ericsson-linux-runner-9 +# - test-ericsson-linux-runner-10 +# - test-ericsson-linux-sprinter-1 +# - test-fhg-basop-runner-3 +# - test-fhg-basop-runner-4 + tags: + - ${RUNNER} + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./$REF_ENCODER_PATH + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + - FILTERED_JOB_NAME=${CI_JOB_NAME// /-} + - FILTERED_JOB_NAME=${FILTERED_JOB_NAME//\[/} + - FILTERED_JOB_NAME=${FILTERED_JOB_NAME//\]/} + - FILTERED_JOB_NAME=${FILTERED_JOB_NAME//:/-} + - export CI_JOB_NAME=$FILTERED_JOB_NAME + <<: *ivas-pytest-anchor + + + +# --------------------------------------------------------------- +# Complexity measurement jobs +# --------------------------------------------------------------- + +# create necessary environment +.complexity-measurements-setup: &complexity-measurements-setup + ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work + - mkdir -p wmops/logs + + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + # this is a testing/maintenance mechanism to force getting the log history from a specific job id + # see below in the concrete complexity jobs + - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip + - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html + - public_dir="$CI_JOB_NAME-public" + + # if is needed to catch case when no artifact is there (first run), similarly as above + # 1. check for public_dir being there as this might not be the case when artifact download failed + # 2. check for public dir not being empty - handle job failures in prev job that happen after the dir is created. In that case, the empty dir is in the artifacts + - if [ -d $public_dir ] && [ ! -z "$( ls -A $public_dir )" ]; then + - mv $public_dir/* wmops/ + # check here if we have the split-by-levels files present - if not, fake them up with the existing global one + # this is needed for the first run with split graphs on a branch where the global version did run previously + # NOTE: checking only for level_1 file here as this should already be sufficient + # NOTE2: also not chechking for RAM for same reason + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi + - fi + + - rm artifacts.zip + - rm -rf $public_dir + + ### 1.5.part: get the corresponding measurement from ivas-float-update + # only do this on main and not for the complexity meaurements run on ivas-float-update + - if [ "$CI_COMMIT_REF_NAME" == "main" ]; then + - job_id=$(python3 ci/get_id_of_last_job_occurence.py $FLOAT_REF_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) + - echo $job_id + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip + - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true + # add file to arguments only if the artifact could be retrieved to prevent error later. + - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi + - fi + +# prepare artifacts -> move to public directory +.complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts + - public_dir="$CI_JOB_NAME-public" + - mkdir $public_dir + - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ + # move logfiles for links + - mkdir $public_dir/logs + # first move logs + - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g") + - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done + - if [ "$CI_COMMIT_REF_NAME" == "$FLOAT_REF_BRANCH" ]; then mv wmops/logs/latest_WMOPS.csv $public_dir/logs/; fi + # copy index page blueprint + - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html + # patch the format in the title + - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html + # do separately here to avoid overwrite complaints by mv + - mv -f ci/complexity_measurements/style.css ${public_dir}/ + +.complexity-measurements-report-summary: &complexity-measurements-report-summary + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/print-results-banner.sh + - if [ $ret_val -eq 0 ]; then + - echo -e "No crashes occured.\nNo changes in complexity or memory usage (>1%) detected." + - elif [ $ret_val -eq 123 ]; then + - echo -e "Changes in complexity or memory usage (>1%) detected!!!\nNo crashes occured." + - else + - echo -e "Something went wrong in running the codec. Likely some modes were crashing." + - fi + +.complexity-template: ivas-conformance-linux: extends: - .job-linux