Merge branch 'add-ci-for-validation-of-ci-configs' into 'main'

default:

  interruptible: true

stages:

  - test

workflow:

  rules:

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

# Get the merged CI config for ivas-codec and ivas-basop repos based on the current branch and check for it to be valid.

check-merged-ci-config-is-valid:

  stage: test

  tags:

    - ivas-linux

  variables:

    REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec"

    REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop"

  parallel:

    matrix:

      - REPO:

        - ivas-codec

        - ivas-basop

  script:

    - set -euo pipefail

    - |

      if [ "$REPO" == "ivas-codec" ]; then

        repo_url=$REPO_URL_FLOAT

        patch_yml="main-float.yml"

      else

        repo_url=$REPO_URL_BASOP

        patch_yml="main-basop.yml"

      fi

    # artifact names

    - MERGED_RESPONSE="${REPO}-merged.json"

    - BASELINE_RESPONSE="${REPO}-baseline.json"

    - ARTIFACTS_MERGED_YAML="${REPO}-merged.yml"

    - ARTIFACTS_BASELINE_YAML="${REPO}-baseline.yml"

    - ARTIFACTS_DIFF_YAML="${REPO}-diff.yml"

    - TEST_YML="${REPO}-ci.yml"

    - BASE_YML="${REPO}-base-ci.yml"

    # Approach for validating the CI configs:

    # Due to intricacies of gitlab API token perissions/setup, the setup is a bit convoluted. We can't

    # just call the the lint API endpoint of the repective repos, but instead need to

    #   - get the CI file from the respective repo (which includes from this repo's main)

    #   - patch the include section so that it includes the correct local file in this repo (main-float.yml or main-basop-yml)

    #   - call CI lint in THIS repo with passing the patched file -> this should have the same result as the other repo included remotely

    #   - to get the merged yaml back, we need to pass dry_run: true, BUT this causes CI lint to actually run a pipeline which fails

    #     because there are no push pipeline defined...

    #   - so now we need to use the merged yaml file that we got back to again run the static lint and then finally we get a result for the whole thing being valid

    # Get CI file from the respective repo

    - curl -sS --url "${repo_url}/-/raw/main/.gitlab-ci.yml?inline=false" --output $TEST_YML

    - cp $TEST_YML $BASE_YML

    - sed -i '/^include:/,$d' $BASE_YML

    - |

      echo -e "include:\n  - local: ${patch_yml}" >> $BASE_YML

    # Get merged baseline with target branch ref

    - |

      jq -Rs '{ dry_run: true, content: . }' < $BASE_YML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $BASELINE_RESPONSE \

         --data-binary @-

    # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...)

    - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML

    - |

      jq -Rs '{ content: . }' < $ARTIFACTS_BASELINE_YAML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $BASELINE_RESPONSE \

         --data-binary @-

    - sed -i '/^include:/,$d' $TEST_YML

    - |

      echo -e "include:\n  - local: ${patch_yml}" >> $TEST_YML

    # Get current with feature branch ref

    - |

      jq -Rs '{ dry_run: true, content: . }' < $TEST_YML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_REF_NAME}" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $MERGED_RESPONSE \

         --data-binary @-

    # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...)

    - jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML

    - |

      jq -Rs '{ content: . }' <$ARTIFACTS_MERGED_YAML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $MERGED_RESPONSE \

         --data-binary @-

    # find semantic diff between merged yaml with main and with this branch

    - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML || true

    # Check validity

    - baseline_valid=$(jq -r ".valid" $BASELINE_RESPONSE)

    - merged_valid=$(jq -r ".valid" $MERGED_RESPONSE)

    - |

      if [ "$baseline_valid" != "true" ]; then

        echo "=== WARNING: BASELINE CONFIG IS INVALID ==="

        jq -r ".errors" $BASELINE_RESPONSE

      fi

    - |

      if [ "$merged_valid" != "true" ]; then

        echo "=== MERGED CONFIG IS INVALID ==="

        jq -r ".errors" $MERGED_RESPONSE

        exit 1

      fi

  artifacts:

    when: always

    access: all

    expire_in: "5 days"

    paths:

      - "*-baseline.yml"

      - "*-merged.yml"

      - "*-diff.yml"

      - "*-baseline.json"

      - "*-merged.json"