Commit f0151119 authored by Jan Kiene's avatar Jan Kiene
Browse files

cleanup use of -v for pytest calls

parent fe6a902d

main-basop.yml

+4 −4
Original line number Diff line number Diff line
@@ -363,7 +363,7 @@ workflow: 
    - mv $DUT_RENDERER_PATH IVAS_rend



    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml $comp_args -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    ### create histograms
@@ -491,7 +491,7 @@ workflow: 


    - exit_code=0

    - rm -rf .pytest_cache || true

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?

    - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?



    - if [ $exit_code -ne 0 ]; then

    -   exit_code=$EXIT_CODE_NON_BE
@@ -628,7 +628,7 @@ workflow: 
    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$?

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh

    # create dut outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH

    # Store branch outputs for later comparison
@@ -643,7 +643,7 @@ workflow: 
    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$?

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh

    # create merge-target outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN



    ### compare the two csv files for regressions

main-float.yml

+12 −12
Original line number Diff line number Diff line
@@ -1261,7 +1261,7 @@ ivas-conformance: 
    # Reference creation

    - python scripts/prepare_combined_format_inputs.py

    - $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer_short/test_renderer.py", "tests/split_rendering/test_split_rendering.py"

    - python -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

    - python -m pytest -q $TEST_SET -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

    - python scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



    # Copy input data and output ref data
@@ -1357,7 +1357,7 @@ ivas-conformance-linux: 
    # Reference creation

    - python3 scripts/prepare_combined_format_inputs.py

    - TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py"

    - python3 -m pytest -q $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

    - python3 -m pytest -q $TEST_SET -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

    - python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



    # Copy input data and output ref data
@@ -2231,11 +2231,11 @@ coverage-test-on-main-scheduled-stv: 
    # codec smoke test

    - bash ci/smoke_test.sh coverage

    # default renderer framesize is 20ms

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only || true

    # run renderer smoke test - needed for REFVEC tests

    - python3 -m pytest -n auto -q --tb=no tests/renderer/test_renderer.py

    # run renderer test only with REF creation stage - no CUT needed since executables are identical
@@ -2270,16 +2270,16 @@ coverage-test-on-main-scheduled-ltv: 
    COVERAGE_TITLE: "long test vectors"

  script:

    # -- Add extended coverage of EVS test and long test vectors

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # default renderer framesize is 20ms

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -n auto -q --tb=no --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    # for different renderer framesizes, only the decoder needs to be tested, so use --decoder_only

    # need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 5 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -n auto -q --tb=no --param_file scripts/config/self_test_ltv.prm --dut_decoder_path ./IVAS_dec --decoder_only --dut_fr 10 || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -n auto -q --tb=no --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv || true

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -n auto -q --tb=no --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv || true

    # run renderer smoke test - needed for REFVEC tests

    - ls -altr scripts/testv

    # LTV smoketest is the same as REF creation stage for renderer

snippets/basop/build-and-create-float-ref-outputs.sh

+1 −1
Original line number Diff line number Diff line
@@ -25,4 +25,4 @@ if [ "$COMPARE_DMX" = "true" ]; then 
  enc_dmx_arg="--compare_enc_dmx"

fi



python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?
 
python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

snippets/ivas-conformance.sh

+5 −3
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ cp IVAS_rend IVAS_rend_ref 
cp ISAR_post_rend ISAR_post_rend_ref



# Reference creation

python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html

python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt



# Copy input data and output ref data
@@ -64,5 +64,7 @@ cd - 
bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE"



# Check for failures

if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi

if [ $exit_code -ne 0 ]; then

  echo "Test failures encountered"

  exit $EXIT_CODE_FAIL

fi