Commit c9dbeded authored by norvell's avatar norvell
Browse files

A few fixes for long term logs job

parent e6348466

.gitlab-ci.yml

+13 −10
Original line number Diff line number Diff line
@@ -282,7 +282,7 @@ workflow: 
    - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

    -    id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)

    -    echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous"

    -    curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

    -    curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

    -    unzip artifacts.zip -d previous_artifacts

    # This wildcard thingy relies on only one csv file being present per job

    -    file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv"
@@ -1662,24 +1662,27 @@ ivas-long-term-job-logs: 
      # handle case where the job is running for the first time. e.g where the job was not found.

      id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)

      echo "Job ID from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"

      curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

      

      if [ "$id_previous" != "-1" ]; then

        unzip artifacts.zip -d logs

        # Unzip artifacts to recover past logs dir

        curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

        unzip artifacts.zip

      else

        # create logs dir if it doesn't exist

        mkdir logs

      fi



      # create logs dir if it doesn't exist

      # create folder for today's results

      TODAY=$(date +'%Y-%m-%d')

      mkdir -p logs/$TODAY



      # Aggregate job logs

      

      job=("ivas-pytest-compare_ref-long-enc" "ivas-pytest-compare_ref-long-enc-lev+10" "ivas-pytest-compare_ref-long-enc-lev-10" "ivas-pytest-compare_ref-long-dec" "ivas-pytest-compare_ref-long-dec-lev+10" "ivas-pytest-compare_ref-long-dec-lev-10" "ivas-pytest-compare_ref-long-fx-fx" "ivas-pytest-compare_ref-long-fx-fx-lev+10" "ivas-pytest-compare_ref-long-fx-fx-lev-10")

      for job_name in "${job[@]}"; do

      jobs="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10"

      for job in $jobs; do

        echo "Getting job logs for: $job"

        id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID)

        echo "Job ID from variables - "$job_name", Job ID from script - $id_previous"

        curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

        curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

        unzip artifacts.zip -d previous_artifacts

        # This wildcard thingy relies on only one csv file being present per job

        mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY
@@ -1717,7 +1720,7 @@ ivas-long-term-job-logs: 
  # this is a testing/maintenance mechanism to force getting the log history from a specific job id

  # see below in the concrete complexity jobs

  - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi

  - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip -qq artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - public_dir="$CI_JOB_NAME-public"
@@ -1747,7 +1750,7 @@ ivas-long-term-job-logs: 
  ### 1.5.part: get the corresponding measurement from ivas-float-update

  - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id

  - curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip

  - curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip

  - unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" || true

  # add file to arguments only if the artifact could be retrieved to prevent error later.

  - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi