rework + more verbose debug output

default:

  # by default, we want all jobs to be interruptible, and we need to configure this explicitly

  interruptible: true

stages:

workflow:

  rules:

    # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  stage: test

  tags:

    - ivas-linux

  variables:

    MERGED_RESPONSE: "merged.json"

    BASELINE_RESPONSE: "baseline.json"

    ARTIFACTS_MERGED_YAML: "merged.yml"

    ARTIFACTS_BASELINE_YAML: "baseline.yml"

    ARTIFACTS_DIFF_YAML: "diff.yml"

    REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec"

    REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop"

  parallel:

      - REPO:

        - ivas-codec

        - ivas-basop

  script:

    - set -euxo pipefail

    - if [ "$REPO" == "ivas-codec" ]; then repo_url=$REPO_URL_FLOAT; patch_yml="main-float.yml"; fi

    - if [ "$REPO" == "ivas-basop" ]; then repo_url=$REPO_URL_BASOP; patch_yml="main-basop-yml";fi

    # get the CI file from the respective repo without lengthy clone

    - test_yml="$REPO-ci.yml"

    - curl -sS --url "$repo_url/./raw/main/.gitlab-ci.yml?inline=false" --output $test_yml

    - cat $test_yml

    # get the "baseline" yaml for comparison. this is always the thing we have on the MR target branch (usually main)

    - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

    - target_sha=$(git rev-parse HEAD)

    - base_yml="base-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME-ci.yml"

    # patch the yaml file to include the stuff we have locally in this branch

    - cp $test_yml $base_yml

    - |

      sed -i".bak" '/^include:/,$d' $base_yml

    - cat $base_yml

      if [ "$REPO" == "ivas-codec" ]; then

        repo_url=$REPO_URL_FLOAT

        patch_yml="main-float.yml"

      else

        repo_url=$REPO_URL_BASOP

        patch_yml="main-basop.yml"

      fi

    # artifact names

    - MERGED_RESPONSE="${REPO}-merged.json"

    - BASELINE_RESPONSE="${REPO}-baseline.json"

    - ARTIFACTS_MERGED_YAML="${REPO}-merged.yml"

    - ARTIFACTS_BASELINE_YAML="${REPO}-baseline.yml"

    - ARTIFACTS_DIFF_YAML="${REPO}-diff.yml"

    - TEST_YML="${REPO}-ci.yml"

    - BASE_YML="${REPO}-base-ci.yml"

    # Get CI file from the respective repo

    - curl -sS --url "${repo_url}/-/raw/main/.gitlab-ci.yml?inline=false" --output $TEST_YML

    - echo "=== Original CI file ==="

    - cat $TEST_YML

    - |

      echo -e "include:\n  - local: ${patch_yml}\n" >> $base_yml

    - cat $base_yml

    - head -n 50 $patch_yml

    # lint patched yaml

      echo "Target branch SHA: $CI_MERGE_REQUEST_TARGET_BRANCH_SHA"

    - |

      echo "Current commit SHA: $CI_COMMIT_SHA"

    - git fetch origin

    - git checkout $CI_COMMIT_REF_NAME

    - cp $TEST_YML $BASE_YML

    - sed -i '/^include:/,$d' $BASE_YML

    - |

      echo -e "include:\n  - local: ${patch_yml}" >> $BASE_YML

    - echo "=== Baseline CI file (target branch) ==="

    - cat $BASE_YML

    # Lint baseline with target branch ref

    - |

      jq -n --arg yaml "$(<$base_yml)" '{ content: $yaml }' \

      jq -n --arg yaml "$(<$BASE_YML)" '{ content: $yaml }' \

      | curl -sS -X POST \

             --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${target_sha}" \

             --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" \

             -H "Content-Type: application/json" \

             -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

             --output $BASELINE_RESPONSE \

             --data-binary @-

    - cat $BASELINE_RESPONSE | jq

    - git checkout $CI_COMMIT_REF_NAME

    # patch the yaml file to include the stuff we have locally in this branch

    - sed -i '/^include:/,$d' $TEST_YML

    - |

      sed -i".bak" '/^include:/,$d' $test_yml

    - cat $test_yml

    - |

      echo -e "include:\n  - local: ${patch_yml}\n" >> $test_yml

    - cat $test_yml

      echo -e "include:\n  - local: ${patch_yml}" >> $TEST_YML

    - echo "=== Current CI file (feature branch) ==="

    - cat $TEST_YML

    # Show what the local file looks like

    - echo "=== First 50 lines of ${patch_yml} ==="

    - head -n 50 $patch_yml

    # lint patched yaml

    # Lint current with feature branch ref

    - |

      jq -n --arg yaml "$(<$test_yml)" '{ content: $yaml }' \

      jq -n --arg yaml "$(<$TEST_YML)" '{ content: $yaml }' \

      | curl -sS -X POST \

             --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&ref=${CI_COMMIT_SHA}" \

             -H "Content-Type: application/json" \

             -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

             --output $MERGED_RESPONSE \

             --data-binary @-

    - cat $MERGED_RESPONSE | jq

    - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML

    - jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML

    - echo "=== Computing diff ==="

    - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML || true

    # Check validity

    - baseline_valid=$(jq -r ".valid" $BASELINE_RESPONSE)

    - merged_valid=$(jq -r ".valid" $MERGED_RESPONSE)

    - |

      echo "Baseline valid: $baseline_valid"

    - |

      echo "Merged valid: $merged_valid"

    # Show errors if invalid

    - |

      if [ "$merged_valid" != "true" ]; then

        echo "=== MERGED CONFIG IS INVALID ==="

        jq -r ".errors" $MERGED_RESPONSE

        exit 1

      fi

    - |

    # extract actual merged yaml from the json responses and get diff

    - cat $BASELINE_RESPONSE | jq -r ".merged_yaml" > $ARTIFACTS_BASELINE_YAML

    - cat $MERGED_RESPONSE | jq -r ".merged_yaml" > $ARTIFACTS_MERGED_YAML

    - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML

      if [ "$baseline_valid" != "true" ]; then

        echo "=== WARNING: BASELINE CONFIG IS INVALID ==="

        jq -r ".errors" $BASELINE_RESPONSE

      fi

    # fail job if merged yaml is not valid

    - if [ $(jq ".valid" $MERGED_RESPONSE) != "true" ]; then exit 1; fi

  artifacts:

    when: always

    access: all

    expire_in: "5 days"

    paths:

      - $ARTIFACTS_BASELINE_YAML

      - $ARTIFACTS_MERGED_YAML

      - $ARTIFACTS_DIFF_YAML

      - $BASELINE_RESPONSE

      - $MERGED_RESPONSE

      - "*-baseline.yml"

      - "*-merged.yml"

      - "*-diff.yml"

      - "*-baseline.json"

      - "*-merged.json"