Commit 77aab356 authored by Jan Kiene's avatar Jan Kiene
Browse files

add csv reports to the comparison jobs

parent 73f41f76

main-basop.yml

+38 −17
Original line number Diff line number Diff line
@@ -1241,6 +1241,10 @@ renderer-pytest-on-merge-request: 
  tags:

    - ivas-linux

  stage: compare

  variables:

    REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml"

    REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html"

    REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
@@ -1262,17 +1266,19 @@ renderer-pytest-on-merge-request: 


    - exit_code=0

    - testcase_timeout=60

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?



    ### Run test using branch scripts and input

    - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - *merge-request-comparison-check



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV



  allow_failure:

    exit_codes:

      - 123
@@ -1281,12 +1287,13 @@ renderer-pytest-on-merge-request: 
    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

      - $REPORT_XML

      - $REPORT_HTML

      - $REPORT_CSV

    expose_as: "pytest renderer results"

    reports:

      junit:

        - report-junit.xml

        - $REPORT_XML



ivas-pytest-on-merge-request:

  extends:
@@ -1295,6 +1302,10 @@ ivas-pytest-on-merge-request: 
    - .rules-merge-request-no-draft

  stage: compare

  timeout: "40 minutes"

  variables:

    REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml"

    REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html"

    REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

  tags:

    - ivas-linux

  script:
@@ -1327,11 +1338,13 @@ ivas-pytest-on-merge-request: 
    ### run pytest

    - exit_code=0

    - testcase_timeout=600

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout -k "not model" || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout -k "not model" || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - *merge-request-comparison-check



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV



  allow_failure:

    exit_codes:

      - 123
@@ -1340,12 +1353,13 @@ ivas-pytest-on-merge-request: 
    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

      - $REPORT_XML

      - $REPORT_HTML

      - $REPORT_CSV

    expose_as: "pytest ivas results"

    reports:

      junit:

        - report-junit.xml

        - $REPORT_XML



#TODO: re-enable after merge to main is done

.split-rendering-pytest-on-merge-request:
@@ -1355,6 +1369,10 @@ ivas-pytest-on-merge-request: 
    - .rules-merge-request-no-draft

  timeout: "60 minutes"

  stage: compare

  variables:

    REPORT_XML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.xml"

    REPORT_HTML: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.html"

    REPORT_CSV: "report--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
@@ -1380,7 +1398,7 @@ ivas-pytest-on-merge-request: 
    # remporary solution to make it work with and without https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/merge_requests/2181

    - mv $DUT_POST_RENDERER_PATH ISAR_post_rend_tmp

    - cp $REF_POST_RENDERER_PATH $DUT_POST_RENDERER_PATH

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -q --log-level ERROR -n auto -rA tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - mv ISAR_post_rend_tmp $DUT_POST_RENDERER_PATH



    ### Run test using scripts and input from main
@@ -1391,11 +1409,13 @@ ivas-pytest-on-merge-request: 
    - echo "$comp_args"



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/split_rendering/test_split_rendering.py $comp_args --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - *merge-request-comparison-check



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV



  allow_failure:

    exit_codes:

      - 123
@@ -1404,12 +1424,13 @@ ivas-pytest-on-merge-request: 
    expire_in: 2 week

    when: always

    paths:

      - report-junit.xml

      - report.html

      - $REPORT_XML

      - $REPORT_HTML

      - $REPORT_CSV

    expose_as: "pytest split rendering results"

    reports:

      junit:

        - report-junit.xml

        - $REPORT_XML



# test that runs all modes with 1s input signals

codec-smoke-test: