CI/CD for SA5

# CI/CD:

#

# RESOLVE triggered by:

#  - any (branch commit)

# VALIDATION triggered by:

#  - any (branch commit)

#  

# GENERATION triggered by:

#  - merge request

#   

variables:

  LOGFILE_YANG: "yang-validation.log"

  LOGFILE_LINT: "yang-linting.log"

  LOGFILE_OPENAPI: "openapi-validation.log"

stages:

  - resolve

  - validation

  - generation

Resolve:

  stage: resolve

  only:

    - pushes

    - merge_requests

  script:

    - echo 'Resolve'

    - git clone https://forge.3gpp.org/rep/reinaortega/3gpp-scripts.git scripts/

  after_script:

    - ls

  artifacts:

    paths:

      - scripts/

    expire_in: 5 mins #It should be deleted after last stage of this pipeline

YANG Strict:

  stage: validation

  only:

    - pushes

  needs:

    - job: Resolve

      artifacts: true

  script:

    - echo 'YANG Strict'

    - ./scripts/sa5/validate-data-models.sh .   

  after_script:

    - ls

  artifacts:

    paths:

      - ${LOGFILE_YANG}

YANG 3GPP Checks:

  stage: validation

  only:

    - pushes

  needs:

    - job: Resolve

      artifacts: true

  script:

    - echo 'YANG 3GPP Checks'

    - ./scripts/sa5/lint-data-models.sh .

  artifacts:

    paths:

      - ${LOGFILE_LINT}

Validate OpenAPI:

  stage: validation

  only:

    - pushes

  needs:

    - job: Resolve

      artifacts: true

  script:

    - echo 'Validate OpenAPI'

    - ./scripts/sa5/validate-all.sh ./OpenAPI

  artifacts:

    paths:

      - ./OpenAPI/${LOGFILE_OPENAPI}

CR contribution:

  stage: generation

  when: on_success

  only:

    - merge_requests

  dependencies:

    - Resolve

  script:

    - echo 'CR contribution'

    - ./scripts/sa5/generate_changemarks.sh ${CI_PROJECT_URL} ${CI_MERGE_REQUEST_PROJECT_ID} ${CI_MERGE_REQUEST_IID}    

  artifacts:

    paths:

      - ./*.docx

 No newline at end of file

#!/bin/bash

DOCKER_IMAGE=forge.3gpp.org:5050/tools/3gpp-scripts/forgelib:v1.0.0

echo "\n------ Checking for docker image --------"

docker pull "$DOCKER_IMAGE"

echo "------ Removing previous outputs --------"

rm *.docx

echo "------ Parsing repo URL --------"

HOST_URL=$(echo $1 | cut -d'/' -f 1-3)

PROJECT_NAME=$(echo $1 | cut -d'/' -f 5- | cut -d'.' -f 1)

echo "HOST URL:" $HOST_URL

echo "PROJECT NAME:" $PROJECT_NAME

echo "PROJECT ID:" $2

echo "MERGE IID:" $3

echo "\n------ Generating change marks --------"

docker container run --rm -v$(pwd):/tmp/ "$DOCKER_IMAGE" forgelib-changedocs -vv --outPath=/tmp/ "$HOST_URL" "$2" "$PROJECT_NAME" "$3"

exit 0

#!/bin/bash

# Copyright ETSI 2020

LOGFILE=yang-linting.log

DOCKER_IMG=etsiforge/pyang:2.4.0-3gpp-d410f57

echo "------ Switching to $1 folder --------"

cd $1

echo "------ Content of the folder ------"

ls

echo "------ Checking for previous logs ------"

if [ -f "$LOGFILE" ] ; then

	echo "Found previous log. Removing it"

	rm "$LOGFILE"

fi

echo "------ Linting modified YANG files ------"

# If there are no YANG, JSON or XML files, then simply exit

ls | grep -q 'yang'

found_data_models=$?

if [ ! $found_data_models ] ; then 

    echo "-- No YANG files."

    exit 0

fi

echo "------ pyYANG Version ------"

docker run --rm -v $(pwd)/yang-models:/yang "${DOCKER_IMG}" pyang --version

echo "------ Linting modified YANG files ------"

#  Display all  errors and warnings to the user. Errors are allowed.

docker pull -q "${DOCKER_IMG}"

# Linting only yang files that have changed in last commit

files_changed=`git log --raw -n 1 |grep "^:.*yang-models/.*yang"| cut -d '/' -f 2`

echo $files_changed

external_files_changed=`git log --raw -n 1 |grep "^:.*yang-models/external-yams/.*yang"| cut -d '/' -f 2`

echo $external_files_modified

if [ -d "changed-yang-models" ]; then

        echo "Removing changed-yang-models dir"

        rm -rf "changed-yang-models"

fi

mkdir changed-yang-models

mkdir changed-yang-models/external-yams

if [ ! -z "$files_changed" ] ; then

        cd yang-models

        cp $files_changed ../changed-yang-models/

        cd ..

fi

if [ ! -z "$external_files_changed" ] ; then

        cd yang-models/external-yams

        cp $external_files_changed ../changed-yang-models/external-yams

        cd ../..

fi

if [ ! -z "$files_changed" ] || [ ! -z "$external_files_changed" ] ; then

	docker run --rm -v $(pwd):/yang "${DOCKER_IMG}" "/bin/sh" -c 'pyang --3gpp -p yang-models -p yang-models/external-yams changed-yang-models/*.yang' 2>>$LOGFILE >>$LOGFILE

	ret=$?        

else

    echo "-- No YANG files modified."

    exit 0

fi

#Validate all yang files

#docker run --rm -v $(pwd)/yang-models:/yang "${DOCKER_IMG}" "/bin/sh" -c 'pyang --3gpp -p external-yams _3gpp-*.yang' 2>>$LOGFILE >>$LOGFILE

#ret=$?

echo "Output of Linting commad saved to $LOGFILE."

echo "------ Linting result ------"

echo $ret

if [ $ret == 1 ] ; then

	echo "Some issues were collected by the linter, but they are treated as warnings and are not failing the validation."

	echo "To see the full output of the linter, download the archived artifact named '$LOGFILE'."

fi

# exit $ret

exit 0

#!/bin/bash

# Copyright ETSI 2019-2021

# See: https://forge.etsi.org/etsi-forge-copyright-statement.txt

LOGFILE=openapi-validation.log

DOCKER_IMAGE=etsiforge/swagger-cli:4.0.3

echo -e "\n------ Switching to $1 folder --------"

cd $1

echo "------ Checking for previous logs ------"

if [ -f "$LOGFILE" ] ; then

    echo "Found previous log. Removing it"

    rm "$LOGFILE"

fi

if [ -x "$(command -v docker)" ] ; then

    RUN_IN_DOCKER=0

    echo "Using dockerized validator ($DOCKER_IMAGE)"

    docker pull "$DOCKER_IMAGE"

	#docker pull -q "$DOCKER_IMAGE" --> Pull quietly only from Ubuntu 20.04

else

    RUN_IN_DOCKER=1

    if [ ! -x "$(command -v swagger-cli)" ] ; then

        echo "Validator swagger-cli not found. Quitting."

        exit 1

    else

        echo "Using local validator ($(which swagger-cli))"

    fi

fi

function validate {

    echo -e "----  Validating $i:  "    

    if [ $RUN_IN_DOCKER == 0 ] ; then

	   bname=$(basename "$1")

       docker run --rm -v $(dirname $(realpath $1)):"/specs" $DOCKER_IMAGE swagger-cli validate "/specs/$bname"

    else

        swagger-cli validate "$1"

    fi

}

echo -e "\n------ Validating all YAML files (may takes several minutes) ------"

# If there are no YAML file, simply exit

ls | grep -q yaml

found_yaml=$?

if [ ! $found_yaml ] ; then 

    echo "-- No YAML files."

    exit 0

fi

fres=0

LOG=""

ERR=""

for i in ./*.yaml ;  do

    MSG=$(validate "$i" 2>&1)

    res=$?

    if [ ! $res == 0 ] ; then 

        ERR="$ERR$MSG\n"

    fi 

    LOG="$LOG$MSG\n"

    fres=$(($fres||$res))

done

echo -e "\n-- Final validator returns $fres." | tee -a $LOGFILE

if [ $fres == 0 ] ; then

   echo "No errors found, all files validate the OpenAPI definition. Below the complete log." | tee -a $LOGFILE

else

   echo "Some errors found, see below the details." | tee -a $LOGFILE

fi

if [ ! "$ERR" == "" ] ; then 

    echo -e "\n---- Errors detected ----" | tee -a $LOGFILE

    echo -e $ERR  | tee -a $LOGFILE

fi

echo -e "\n---- Complete log of validation ----" | tee -a $LOGFILE

echo -e $LOG | tee -a $LOGFILE

echo -e "\n------ Content of the folder ------" | tee -a $LOGFILE

ls | tee -a $LOGFILE

# Exit code needed for jenkins to know the verdict of the build

exit $fres

#!/bin/bash

# Copyright ETSI 2020

LOGFILE=yang-validation.log

DOCKER_IMG=etsiforge/pyang:2.4.0-3gpp-d410f57

echo "------ Switching to $1 folder --------"

cd $1

echo "------ Content of the folder ------"

ls

echo "------ Checking for previous logs ------"

if [ -f "$LOGFILE" ] ; then

	echo "Found previous log. Removing it"

	rm "$LOGFILE"

fi

echo "------ Validating modified YANG files ------"

# If there are no YANG, JSON or XML files, then simply exit

ls | grep -q 'yang'

found_data_models=$?

if [ ! $found_data_models ] ; then 

    echo "-- No YANG files."

    exit 0

fi

# This must run error-free and must display all warnings to the user

docker pull -q "${DOCKER_IMG}"

#Validate all yang files

docker run --rm -v $(pwd)/yang-models:/yang "${DOCKER_IMG}" "/bin/sh" -c 'pyang --strict -p external-yams *.yang' 2>>$LOGFILE >>$LOGFILE

ret=$?

cat $LOGFILE

if [ $ret == 0 ] ; then

    echo "No errors found by pyang --strict validation."

fi

exit $ret