Commit f0f70b0c authored by Jan Kiene's avatar Jan Kiene
Browse files

finish anchor for merge request jobs

parent 6952a410

.gitlab-ci.yml

+51 −10
Original line number Diff line number Diff line
@@ -21,6 +21,10 @@ variables: 
  TESTCASE_TIMEOUT_STV: 900

  TESTCASE_TIMEOUT_LTV: 2400

  TESTCASE_TIMEOUT_STV_SANITIZERS: 1800

  CI_REGRESSION_THRESH_MLD: 0.1

  CI_REGRESSION_THRESH_MAX_ABS_DIFF: 5

  CI_REGRESSION_THRESH_SSNR: -1

  CI_REGRESSION_THRESH_ODG: -0.05

  GIT_CLEAN_FLAGS: -ffdxq

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger
@@ -252,7 +256,7 @@ stages: 


    - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME



    - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
@@ -286,7 +290,6 @@ stages: 
      - $PAGES_HTML_ARTIFACT_NAME

      - $CSV_ARTIFACT_NAME

      - $MERGED_CSV_ARTIFACT_NAME

      - images

      - $SUMMARY_HTML_ARTIFACT_NAME

      - $IMAGES_ARTIFACT_NAME

    expose_as: "pytest compare results"
@@ -298,6 +301,15 @@ stages: 
  stage: test

  needs: ["build-codec-linux-make"]

  timeout: "300 minutes"

  variables:

    XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml"

    XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml"

    HTML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html"

    HTML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html"

    CSV_BRANCH: "scores-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv"

    CSV_MAIN: "scores-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.csv"

    IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"

    SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"

  script:

    - *print-common-info

    - *update-scripts-repo
@@ -318,20 +330,49 @@ stages: 
    # this per default builds the branch and the reference and creates the reference outputs

    - *build-and-create-reference-outputs

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors_branch=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit.xml scores_branch.csv

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH



    # create the summary based on the branch

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME



    ### run main now

     git checkout main

    - git checkout main

    - make clean

    - make -j

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors_main=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit.xml scores_main.csv

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN



    ### compare the two csv files for regressions

    - python3 ci/basop-pages/create_report_pages.py /dev/null scores_merged.csv scores_branch.csv scores_main.csv branch main $CI_JOB_NAME

    - regressions_found=$(python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN



    - if [ $regressions_found != 0 ]; then echo "Detected regression wrt to main!"; exit $EXIT_CODE_FAIL; fi

    - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi

    - exit 0

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 1 week

    when: always

    paths:

      - $XML_REPORT_BRANCH

      - $XML_REPORT_MAIN

      - $HTML_REPORT_BRANCH

      - $HTML_REPORT_MAIN

      - $CSV_BRANCH

      - $CSV_MAIN

      - $SUMMARY_HTML_ARTIFACT_NAME

      - $IMAGES_ARTIFACT_NAME

    expose_as: "pytest compare results"

    reports:

      junit:

        - report-junit.xml





.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor