Commit 1a91d0ab authored by Jan Kiene's avatar Jan Kiene
Browse files

support two different float references in regression jobs

parent d6247872

.gitlab-ci.yml

+22 −7
Original line number Diff line number Diff line
@@ -570,22 +570,37 @@ workflow: 
    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-source

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source



    # need to restore cache again

    # set pytest args for ref creation

    - enc_stats_arg=""

    - if [ "$ENCODER_TEST" = "true" ]; then

    -   enc_stats_arg="--enc_stats"

    - fi

    - enc_dmx_arg=""

    - if [ "$COMPARE_DMX" = "true" ]; then

    -   enc_dmx_arg="--compare_enc_dmx"

    - fi



    ### ----- run pytest for branch (dut) first -----

    # create float reference outputs with corresponding branch

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$?

    - *overwrite-pytest-cache-with-artifact

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    # create dut outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH



    # Store branch outputs for comparison

    # Store branch outputs for later comparison

    - mv tests/dut tests/dut_branch



    # create the summary based on the branch only

    - python3 scripts/create_histograms.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

    - *overwrite-pytest-cache-with-artifact



    # need to restore cache again

    ### ----- run pytest for merge target now -----

    # create float reference outputs with corresponding branch

    - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$?

    - *overwrite-pytest-cache-with-artifact

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    # create merge-target outputs

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN



    ### compare the two csv files for regressions

.gitlab-ci/variables.yml

+4 −0
Original line number Diff line number Diff line
@@ -30,6 +30,10 @@ variables: 
  MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target"

  MERGE_TARGET_RENDERER_PATH: "./IVAS_rend_merge_target"

  MERGE_TARGET_POST_RENDERER_PATH: "./ISAR_post_rend_merge_target"

  MERGE_SOURCE_FLOAT_REF_ENCODER_PATH: "./IVAS_cod_merge_source_float_ref"

  MERGE_SOURCE_FLOAT_REF_DECODER_PATH: "./IVAS_dec_merge_source_float_ref"

  MERGE_SOURCE_FLOAT_REF_RENDERER_PATH: "./IVAS_rend_merge_source_float_ref"

  MERGE_SOURCE_FLOAT_REF_POST_RENDERER_PATH: "./ISAR_post_rend_merge_source_float_ref"

  # These path variables are used for building the binaries

  # They should never be overwritten!

  DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_dut"