diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c75ef1404cc1fba2da68207376ff981cae74756..1aea80f40d6c89aa2190919d7e13bc593d6186aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,6 +29,12 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing variables: IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs' + variables: + IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' + variables: + IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' variables: IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' @@ -279,7 +285,7 @@ workflow: - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID) - echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous" - - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip - unzip artifacts.zip -d previous_artifacts # This wildcard thingy relies on only one csv file being present per job - file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv" @@ -839,6 +845,8 @@ clang-format-check: # ensure that codec builds on linux build-codec-linux-make: rules: + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs @@ -856,6 +864,8 @@ build-codec-linux-make: # ensure that codec builds on linux with instrumentation active build-codec-linux-instrumented-make: rules: + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main @@ -873,6 +883,8 @@ build-codec-linux-instrumented-make: build-codec-linux-debugging-make: rules: + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main @@ -891,6 +903,8 @@ build-codec-linux-debugging-make: build-codec-windows-msbuild: rules: + - if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' || $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' || $UPDATE_PAGES + when: never - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs @@ -1634,6 +1648,120 @@ voip-be-on-merge-request: - make -j >> /dev/null - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py +# To inject backup copy in manual trigger: +# - Store logs backup in accessible folder, e.g. /usr/local/backup +# - Set MANUAL_PIPELINE_TRIGGER to long-term-logs +# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5 +# - Set LOGS_BACKUP_SOURCE_DIR to source folder, e.g. /usr/local/backup +ivas-long-term-job-logs: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs" + - if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs" + tags: + - $RUNNER_TAG + stage: maintenance + timeout: "25 minutes" + script: + - !reference [ .job-linux, before_script ] + - set -euxo pipefail + - echo "Running on RUNNER_TAG = $RUNNER_TAG" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + # Clean up logs + - if [ -d "logs" ]; then rm -rf "logs"; fi + # Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set + - if [ "$LOGS_BACKUP_SOURCE_DIR" != "" ]; then + - cp_ret=0 + - cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ || cp_ret=$? + - if [ "$cp_ret" != 0 ]; then + - echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!" + - exit 1 + - fi + - fi + + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID) + - echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous" + + - if [ "$id_previous" != "-1" ]; then + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + # If there is overlap between injected log and new log, use the new logs. + - unzip -o artifacts.zip + - else + # create logs dir if it doesn't exist (should only happen in first run) + - echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!" + - mkdir logs + - fi + + # create folder for today's results + - TODAY=$(date +'%Y-%m-%d') + - mkdir -p logs/$TODAY + + # Aggregate job logs + - job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10" + - for job_name in $job_names; do + - echo "Getting job logs for $job_name" + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID) + - echo "Job ID from variables - "$job_name", Job ID from script - $id_previous" + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip -d previous_artifacts + - mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY + - rm artifacts.zip + - rm -r previous_artifacts + - done + - ls logs + - exit 0 + + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 4 weeks + when: always + paths: + - logs + expose_as: "ivas long term job logs results" + +# To store backup copy: +# - Prepare accessible folder for backup, e.g. /usr/local/backup +# - Set MANUAL_PIPELINE_TRIGGER to backup-long-term-logs +# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5 +# - Set LOGS_BACKUP_TARGET_DIR to source folder, e.g. /usr/local/backup +backup-long-term-job-logs: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs" + - if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs" + tags: + - $RUNNER_TAG + stage: maintenance + timeout: "25 minutes" + script: + - !reference [ .job-linux, before_script ] + - set -euxo pipefail + - echo "Running on RUNNER_TAG = $RUNNER_TAG" + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + + # Find ID of last run of ivas-long-term-job-logs + - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID) + + - if [ "$id_previous" != "-1" ]; then + # Unzip artifacts to recover past logs dir + - curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip + - unzip artifacts.zip + - else + # create logs dir if it doesn't exist + - mkdir logs + - fi + + # Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set + - if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then + - cp -r logs $LOGS_BACKUP_TARGET_DIR + - exit 0 + - else + - echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!" + - exit 1 + - fi + # --------------------------------------------------------------- # Complexity measurement jobs @@ -1649,7 +1777,7 @@ voip-be-on-merge-request: # this is a testing/maintenance mechanism to force getting the log history from a specific job id # see below in the concrete complexity jobs - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi - - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - public_dir="$CI_JOB_NAME-public" @@ -1679,7 +1807,7 @@ voip-be-on-merge-request: ### 1.5.part: get the corresponding measurement from ivas-float-update - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip + - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi diff --git a/.gitlab-ci/variables.yml b/.gitlab-ci/variables.yml index d91866b8623ebbe6a7f890afb9a3ba87d187bf8b..96ef1f5076cfa2fd17dd770b210f1d75d79f323b 100644 --- a/.gitlab-ci/variables.yml +++ b/.gitlab-ci/variables.yml @@ -46,6 +46,9 @@ variables: FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" CUT_COMMIT_FILE: "CuT-git-sha.txt" MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" + RUNNER_TAG: "ivas-basop-linux" + LOGS_BACKUP_SOURCE_DIR: "" + LOGS_BACKUP_TARGET_DIR: "" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -65,3 +68,5 @@ variables: - 'voip-be-test' - 'renderer-framesize-be' - 'peaq-enc-passthrough' + - 'long-term-logs' + - 'backup-long-term-logs'