Commit f6a2f4a5 authored by norvell's avatar norvell
Browse files

Add '-' in log jobs for consistency. Fix job name in backup-long-term-job-logs

parent 56a2530e

.gitlab-ci.yml

+59 −65
Original line number Diff line number Diff line
@@ -1671,52 +1671,49 @@ ivas-long-term-job-logs: 
  timeout: "25 minutes"

  script:

    - !reference [ .job-linux, before_script ]

    - |

      set -euxo pipefail

      echo "Running on RUNNER_TAG: $RUNNER_TAG"

      bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

      bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

    - set -euxo pipefail

    - echo "Running on RUNNER_TAG: $RUNNER_TAG"

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh



    # Clean up logs

      if [ -d "logs" ]; then rm -rf "logs"; fi



    - if [ -d "logs" ]; then rm -rf "logs"; fi

    # Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set

      if [ -d "$LOGS_BACKUP_SOURCE_DIR" ]; then

        cp -r $LOGS_BACKUP_SOURCE_DIR/logs .

      fi

    - if [ -d "$LOGS_BACKUP_SOURCE_DIR" ]; then

    -   cp -r $LOGS_BACKUP_SOURCE_DIR/logs .

    - fi



    # handle case where the job is running for the first time. e.g where the job was not found.

      id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)

      echo "Job ID from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"

    - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)

    - echo "Job ID from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"



      if [ "$id_previous" != "-1" ]; then

    - if [ "$id_previous" != "-1" ]; then

        # Unzip artifacts to recover past logs dir

        curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

        unzip artifacts.zip

      else

    -   curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

    -   unzip artifacts.zip

    - else

        # create logs dir if it doesn't exist

        mkdir logs

      fi

    -   mkdir logs

    - fi



    # create folder for today's results

      TODAY=$(date +'%Y-%m-%d')

      mkdir -p logs/$TODAY

    - TODAY=$(date +'%Y-%m-%d')

    - mkdir -p logs/$TODAY



    # Aggregate job logs

      job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10"

      for job_name in $job_names; do

        echo "Getting job logs for: $job_name"

        id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID)

        echo "Job ID from variables - "$job_name", Job ID from script - $id_previous"

        curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

        unzip artifacts.zip -d previous_artifacts

        # This wildcard thingy relies on only one csv file being present per job

        mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY

        rm artifacts.zip

        rm -r previous_artifacts

      done

      ls logs

      exit 0

    - job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10"

    - for job_name in $job_names; do

    -   echo "Getting job logs for: $job_name"

    -   id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID)

    -   echo "Job ID from variables - "$job_name", Job ID from script - $id_previous"

    -   curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

    -   unzip artifacts.zip -d previous_artifacts

    -   mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY

    -   rm artifacts.zip

    -   rm -r previous_artifacts

    - done

    - ls logs

    - exit 0



  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
@@ -1741,34 +1738,31 @@ backup-long-term-job-logs: 
  timeout: "25 minutes"

  script:

    - !reference [ .job-linux, before_script ]

    - |

      set -euxo pipefail

      echo "Running on RUNNER_TAG: $RUNNER_TAG"

      bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

      bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

    - set -euxo pipefail

    - echo "Running on RUNNER_TAG: $RUNNER_TAG"

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh



      id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)

      echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"

    # Find ID of last run of ivas-long-term-job-logs

    - id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID)



      if [ "$id_previous" != "-1" ]; then

    - if [ "$id_previous" != "-1" ]; then

        # Unzip artifacts to recover past logs dir

        curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

        unzip artifacts.zip

      else

    -   curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip

    -   unzip artifacts.zip

    - else

        # create logs dir if it doesn't exist

        mkdir logs

      fi

    -   mkdir logs

    - fi



    # Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set

      if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then

        cp -r logs $LOGS_BACKUP_TARGET_DIR/logs

        exit 0;

      else

        echo "Error: LOGS_BACKUP_TARGET_DIR not set for backup job!"

        exit -1;

      fi



      

    - if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then

    -   cp -r logs $LOGS_BACKUP_TARGET_DIR

    -   exit 0;

    - else

    -   echo "Error: LOGS_BACKUP_TARGET_DIR not set for backup job!"

    -   exit -1;

    - fi





# ---------------------------------------------------------------