diff --git a/includes/default-variables.yml b/includes/default-variables.yml index 13e8811ba36b2958b46f0ff4858987f341afe314..6fedaa93d294205909a9eb1b03a87903c8d95b89 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -28,3 +28,11 @@ variables: # workaround for job failures due to forge being slow to respond when uploading artifacts GIT_HTTP_LOW_SPEED_LIMIT: "0" GIT_HTTP_LOW_SPEED_TIME: "60" + # set to DEBUG for more expressive logs + IVAS_CI_PYTHON_LOG_LEVEL: "INFO" + # default names for make build log files + MAKE_BUILD_LOG_FILE_DEFAULT: "make-build-log" + MAKE_BUILD_LOG_FILE_FLOAT_REF: "make-build-log-float-ref" + MAKE_BUILD_LOG_FILE_FLOAT_REF_MERGE_SOURCE: "make-build-log-float-ref-merge-source" + MAKE_BUILD_LOG_FILE_MERGE_TARGET: "make-build-log-merge-target" + MAKE_BUILD_LOG_FILE_DUT: "make-build-log-dut" diff --git a/main-basop.yml b/main-basop.yml index 68696efc39bdd91a7cb0e7a51770df9cb0bdbb46..38a7c899303e8cd99640d40dca0f90860f7bd96a 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -487,7 +487,7 @@ workflow: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source - exit_code_target=0 - - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + - python3 -m pytest $TEST_SUITE --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? - exit_code=0 - rm -rf .pytest_cache || true @@ -625,7 +625,7 @@ workflow: ### ----- run pytest for branch (dut) first ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create dut outputs - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true @@ -640,7 +640,7 @@ workflow: ### ----- run pytest for merge target now ----- # create float reference outputs with corresponding branch - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH || exit_code=$? - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/overwrite-pytest-cache-with-artifact.sh # create merge-target outputs - python3 -m pytest --tb=no -q $TEST_SUITE --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --dut_encoder_path $MERGE_TARGET_ENCODER_PATH --dut_decoder_path $MERGE_TARGET_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true @@ -742,8 +742,8 @@ workflow: ### run pytest - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH --compare_to_input || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH --compare_to_input || exit_code=$? - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv @@ -1143,6 +1143,7 @@ renderer-pytest-on-merge-request: - $REPORT_XML - $REPORT_HTML - $REPORT_CSV + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "pytest renderer results" reports: junit: @@ -1194,7 +1195,7 @@ ivas-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1 + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -1202,7 +1203,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=600 - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV @@ -1220,6 +1221,7 @@ ivas-pytest-on-merge-request: - $REPORT_XML - $REPORT_HTML - $REPORT_CSV + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "pytest ivas results" reports: junit: @@ -1293,6 +1295,7 @@ split-rendering-pytest-on-merge-request: - $REPORT_XML - $REPORT_HTML - $REPORT_CSV + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "pytest split rendering results" reports: junit: @@ -1329,7 +1332,7 @@ split-rendering-pytest-on-merge-request: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - make clean - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - python3 tests/create_short_testvectors.py --cut_len 1.0 # TODO: run full test again once the custom binary files are supported - python3 -m pytest -k "not diff_from_rom and not test_binary_file" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html @@ -1337,6 +1340,7 @@ split-rendering-pytest-on-merge-request: paths: - report.html - report-junit.xml + - $MAKE_BUILD_LOG_FILE_DEFAULT when: always name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" expose_as: "logs-hrtf-loading" @@ -1372,7 +1376,7 @@ split-rendering-pytest-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1389,6 +1393,7 @@ split-rendering-pytest-on-merge-request: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "interop test results" reports: junit: @@ -1404,7 +1409,7 @@ split-rendering-smoke-test: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=10 - python3 -m pytest -q -n auto -rA --tb=no --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout --html=report.html --self-contained-html artifacts: @@ -1430,7 +1435,7 @@ rtpdump-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=60 - python3 -m pytest -q --tb=no -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/rtp/test_rtp.py --testcase_timeout=$testcase_timeout artifacts: @@ -1440,6 +1445,7 @@ rtpdump-test: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "rtpdump pytest results" reports: junit: @@ -1472,7 +1478,7 @@ renderer-smoke-test: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - make -j IVAS_rend + - make -j IVAS_rend 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=60 - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: @@ -1496,8 +1502,8 @@ renderer-smoke-test: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=asan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - - cmake --build cmake-build -- -j + - make clean + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=180 - python3 -m pytest -q -n auto --tb=no --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout @@ -1507,6 +1513,7 @@ renderer-smoke-test: when: always paths: - report-junit.xml + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "renderer asan pytest results" reports: junit: @@ -1524,8 +1531,8 @@ renderer-smoke-test: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=msan -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true - - cmake --build cmake-build -- -j + - make clean + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=180 - python3 -m pytest -q -n auto --tb=no --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout @@ -1535,6 +1542,7 @@ renderer-smoke-test: when: always paths: - report-junit.xml + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "renderer msan pytest results" reports: junit: @@ -1554,7 +1562,7 @@ renderer-smoke-test: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - python3 scripts/basop_create_ignorelist_for_ubsan.py - - make -j CLANG=3 IGNORELIST=1 + - make -j CLANG=3 IGNORELIST=1 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=180 - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1,log_path=usan_log_catchall python3 -m pytest -q -n auto --tb=no --junit-xml=report-junit.xml tests/renderer_short/test_renderer.py --testcase_timeout=$testcase_timeout - grep_exit_code=0 @@ -1568,6 +1576,7 @@ renderer-smoke-test: when: always paths: - report-junit.xml + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "renderer usan pytest results" reports: junit: @@ -1614,17 +1623,17 @@ renderer-smoke-test: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi - make clean - - make -j $make_args + - make -j $make_args 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs # TODO: was there for ivas-float-update, but not for main -> takes too long - # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + # - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi @@ -2046,7 +2055,7 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - make_args="$make_args IGNORELIST=1" - fi - make clean - - make -j $make_args + - make -j $make_args 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS" # disable per-testcase timeout for msan to evaluate what is going on that it takes so long - if [[ $CLANG_NUM = 1 ]]; then @@ -2054,7 +2063,7 @@ ivas-pytest-compare_ref-long-fx-fx-lev+10: - fi # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) - - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec + - python3 -m pytest $TEST_SUITE --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml $testcase_timeout_arg --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec ivas-pytest-msan: extends: @@ -2083,6 +2092,7 @@ ivas-pytest-msan: - report-junit.xml - report.html - msan-errors.csv + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Sanitizer selftest results" reports: junit: @@ -2114,6 +2124,7 @@ ivas-pytest-asan: - report-junit.xml - report.html - asan-errors.csv + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Sanitizer selftest results" reports: junit: @@ -2145,6 +2156,7 @@ ivas-pytest-usan: - report-junit.xml - report.html - usan-errors.csv + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Sanitizer selftest results" reports: junit: @@ -2206,7 +2218,7 @@ test-long-self-test: ### build main (ref) binaries - make clean - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - mv ./IVAS_cod ./IVAS_cod_ref - mv ./IVAS_dec ./IVAS_dec_ref @@ -2216,18 +2228,17 @@ test-long-self-test: ### build branch binaries - make clean - - make -j - + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT ### prepare pytest # create references - exit_code_ref=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --update_ref 1 --param_file scripts/config/self_test_ltv.prm --use_ltv --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv.prm --use_ltv --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml $comp_args --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -2242,6 +2253,7 @@ test-long-self-test: paths: - report-junit-ltv.xml - report-ltv.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "test-long-self-test results" reports: junit: @@ -2302,12 +2314,12 @@ coverage-test-on-main-scheduled: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh # Build DuT binaries with GCOV - make clean >> /dev/null - - make GCOV=1 -j + - make GCOV=1 -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in ref creation call - testcase_timeout=$TESTCASE_TIMEOUT_LTV - exit_code=0 - - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer_short --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$? + - python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer_short --update_ref 1 --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec || exit_code=$? - lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage # remove apps and lib_util files from coverage @@ -2324,6 +2336,7 @@ coverage-test-on-main-scheduled: - coverage_enc_dec_rend.info - coverage_enc_dec_rend - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Coverage result" reports: junit: @@ -2349,14 +2362,14 @@ be-2-evs-26444: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h - - make -j >> /dev/null + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT # copy over to never change the testvector dir - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test - mkdir -p ./evs_be_test/output/decoded ./evs_be_test/output/bitstreams - exit_code=0 - - python3 -m pytest tests/test_26444.py -v --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? + - python3 -m pytest tests/test_26444.py --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto || exit_code=$? - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_FAIL; fi - exit 0 @@ -2367,6 +2380,7 @@ be-2-evs-26444: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "EVS 26444 result" reports: junit: @@ -2411,13 +2425,13 @@ voip-be-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - make clean - - make -j > build_log.txt + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py artifacts: when: always expire_in: "5 days" paths: - - build_log.txt + - $MAKE_BUILD_LOG_FILE_DEFAULT check-be-between-renderer-framesizes: extends: @@ -2432,16 +2446,16 @@ check-be-between-renderer-framesizes: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh - make clean - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - exit_code=0 # NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh) - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml || exit_code=$? - exit_code5=0 - exit_code10=0 - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true @@ -2468,6 +2482,7 @@ check-be-between-renderer-framesizes: - report-10ms.html - report-junit-20ms.xml - report-20ms.html + - $MAKE_BUILD_LOG_FILE_DEFAULT reports: junit: - report-junit-5ms.xml @@ -2680,7 +2695,7 @@ ivas-conformance-linux: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - - make -j # To be substituted with reference platform build + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/ivas-conformance.sh artifacts: @@ -2699,6 +2714,7 @@ ivas-conformance-linux: - Readme_IVAS_ISAR_post_rend.txt - $COVERAGE_OUTPUT_FILE - $COVERAGE_OUTPUT_DIR + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml diff --git a/main-float.yml b/main-float.yml index 75d74fa0e6bda65fef8e36665cd64701d5fe6b31..34a5260473fb7ea9eb8ec659bda0d90b103ca9be 100644 --- a/main-float.yml +++ b/main-float.yml @@ -51,6 +51,7 @@ variables: COVERAGE_OUTPUT_FILE_LTV: "coverage-ltv.info" COVERAGE_OUTPUT_FILE_CONFORMANCE: "coverage-conformance.info" COVERAGE_OUTPUT_FILE_MERGED: "coverage-merged.info" + MAKE_BUILD_LOG_FILE_REFERENCE: "make-build-log-ref" # This sets when pipelines are created. Jobs have more specific rules to restrict them. workflow: @@ -120,7 +121,7 @@ workflow: - mkdir build - cd build - cmake .. - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - mv IVAS_rend ../IVAS_rend_test @@ -141,7 +142,7 @@ workflow: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-debugging-macro.sh - cd build - cmake .. - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_REFERENCE - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - mv IVAS_rend ../IVAS_rend_ref @@ -167,7 +168,7 @@ workflow: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - make clean - - make -j CLANG=$CLANG_NUM + - make -j CLANG=$CLANG_NUM 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=$SELFTEST_SANITY_TIMEOUT - export UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1,print_stacktrace=1 @@ -178,12 +179,12 @@ workflow: - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi # run encoder and decoder with 20ms renderer framesize first, use reference creation mode - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$? # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active # we are only interested in runtime errors from the sanitizers and ignore the diffs - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? - - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$? + - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$? - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi after_script: @@ -206,6 +207,7 @@ workflow: - errors-20ms.csv - errors-10ms.csv - errors-5ms.csv + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Sanitizer selftest results" reports: junit: @@ -470,18 +472,18 @@ pytest-compare-20ms-and-5ms-rendering: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh - make clean - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT ### prepare pytest - cp IVAS_cod IVAS_cod_ref - cp IVAS_dec IVAS_dec_ref # create references - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --update_ref 1 ### run pytest - exit_code=0 - exit_code5=0 - exit_code10=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only || exit_code5=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only || exit_code10=$? - zero_errors5=$(cat report-junit-5ms.xml | grep -c 'errors="0"') || true - zero_errors10=$(cat report-junit-10ms.xml | grep -c 'errors="0"') || true - zero_errors=1 @@ -501,6 +503,7 @@ pytest-compare-20ms-and-5ms-rendering: - report-5ms.html - report-junit-10ms.xml - report-10ms.html + - $MAKE_BUILD_LOG_FILE_DEFAULT reports: junit: - report-junit-5ms.xml @@ -514,9 +517,9 @@ renderer-smoke-test: needs: ["build-codec-linux-clang-make"] stage: test script: - - make -j IVAS_rend + - make -j IVAS_rend 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=60 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml --html=report.html tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --log-level ERROR --junit-xml=report-junit.xml --html=report.html tests/renderer/test_renderer.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -524,6 +527,7 @@ renderer-smoke-test: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "renderer make pytest results" reports: junit: @@ -632,6 +636,8 @@ renderer-pytest-on-merge-request: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT + - $MAKE_BUILD_LOG_FILE_REFERENCE expose_as: "pytest renderer results" reports: junit: @@ -645,9 +651,9 @@ split-rendering-smoke-test: needs: ["build-codec-linux-clang-make"] stage: test script: - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=10 - - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml --html=report.html tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout + - python3 -m pytest -q -n auto -rA --log-level ERROR --junit-xml=report-junit.xml --html=report.html tests/split_rendering/test_split_rendering.py --testcase_timeout=$testcase_timeout artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week @@ -655,6 +661,7 @@ split-rendering-smoke-test: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "split rendering make pytest results" reports: junit: @@ -668,7 +675,7 @@ split-rendering-voip-be-to-binaural: needs: ["build-codec-linux-clang-make"] stage: test script: - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=30 - python3 -m pytest -q -n auto -rA --junit-xml=report-junit.xml --html=report.html tests/split_rendering/test_voip_be_splitrend_vs_binaural.py --testcase_timeout=$testcase_timeout artifacts: @@ -678,6 +685,7 @@ split-rendering-voip-be-to-binaural: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "VoIP split rendering vs BINAURAL make pytest results" reports: junit: @@ -691,7 +699,7 @@ rtpdump-test: needs: ["build-codec-linux-clang-make"] stage: test script: - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - testcase_timeout=15 - python3 -m pytest -q --tb=no -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/rtp/test_rtp.py --testcase_timeout=$testcase_timeout artifacts: @@ -701,6 +709,7 @@ rtpdump-test: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "rtpdump pytest results" reports: junit: @@ -746,7 +755,7 @@ split-rendering-pytest-on-merge-request: - echo "Building reference codec at commit $target_commit" # build reference binaries - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_REFERENCE - mv IVAS_cod IVAS_cod_ref - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref @@ -763,7 +772,7 @@ split-rendering-pytest-on-merge-request: - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts - git checkout $source_branch_commit_sha - make clean - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT ### Run test using scripts and input from main - if [ $ref_using_main == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts @@ -785,6 +794,8 @@ split-rendering-pytest-on-merge-request: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT + - $MAKE_BUILD_LOG_FILE_REFERENCE expose_as: "pytest split rendering results" reports: junit: @@ -821,7 +832,7 @@ ivas-pytest-on-merge-request: ### prepare pytest # create references - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1 + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" --update_ref 1 ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi @@ -829,7 +840,7 @@ ivas-pytest-on-merge-request: ### run pytest - exit_code=0 - testcase_timeout=60 - - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -844,6 +855,8 @@ ivas-pytest-on-merge-request: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT + - $MAKE_BUILD_LOG_FILE_REFERENCE expose_as: "pytest ivas results" reports: junit: @@ -877,7 +890,7 @@ ivas-interop-on-merge-request: - exit_code2=0 # set timeout for individual testcase runs to 60 seconds - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$? - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi @@ -894,6 +907,8 @@ ivas-interop-on-merge-request: paths: - report-junit.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT + - $MAKE_BUILD_LOG_FILE_REFERENCE expose_as: "interop test results" reports: junit: @@ -924,14 +939,14 @@ evs-pytest-on-merge-request: ### prepare pytest # create references - testcase_timeout=60 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest for EVS cases - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check @@ -946,6 +961,8 @@ evs-pytest-on-merge-request: paths: - report-junit-evs.xml - report.html + - $MAKE_BUILD_LOG_FILE_DEFAULT + - $MAKE_BUILD_LOG_FILE_REFERENCE expose_as: "pytest evs results" reports: junit: @@ -961,8 +978,13 @@ voip-be-on-merge-request: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - make clean - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py + artifacts: + when: always + expire_in: "5 days" + paths: + - $MAKE_BUILD_LOG_FILE_DEFAULT clang-format-check: extends: @@ -1035,13 +1057,14 @@ check-bitexactness-hrtf-rom-and-file: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - cmake . - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - python3 tests/create_short_testvectors.py --cut_len 1.0 - python3 -m pytest tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html artifacts: paths: - report.html - report-junit.xml + - $MAKE_BUILD_LOG_FILE_DEFAULT when: always name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" expose_as: "logs-hrtf-loading" @@ -1057,13 +1080,14 @@ check-bitexactness-ext-and-transport-format: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - cmake . - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - python3 tests/create_short_testvectors.py --cut_len 1.0 - python3 -m pytest tests/test_be_for_ext_outputs.py --html=report.html --junit-xml=report-junit.xml --self-contained-html artifacts: paths: - report.html - report-junit.xml + - $MAKE_BUILD_LOG_FILE_DEFAULT when: always name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--ext-sanity-check" expose_as: "logs-ext-sanity-check" @@ -1182,14 +1206,14 @@ be-2-evs-windows: # - mv IVAS_dec_test IVAS_dec # # create references # - testcase_timeout=60 -# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --update_ref 1 --testcase_timeout=$testcase_timeout # # ### re-checkout the latest commit here, if ref_using_main is set # - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi # # ### run pytest # - exit_code=0 -# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? +# - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? # - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi # - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true # - if [ $exit_code -ne 0 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi @@ -1394,7 +1418,7 @@ ivas-conformance-linux: # Create GCOV execs for coverage analysis - make clean - - make GCOV=1 -j + - make GCOV=1 -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - cp IVAS_cod testvec/bin - cp IVAS_dec testvec/bin @@ -1431,6 +1455,7 @@ ivas-conformance-linux: - Readme_IVAS_ISAR_post_rend.txt - $COVERAGE_OUTPUT_FILE - $COVERAGE_OUTPUT_DIR + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "Draft IVAS conformance -- Linux" reports: junit: report-junit.xml @@ -1483,11 +1508,11 @@ test-long-self-test: # create references - exit_code_ref=0 - testcase_timeout=300 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$? ### run pytest self-test using long test vectors - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv --mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$? - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1519,9 +1544,9 @@ check-clipping: script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/enable-debugging-macro.sh - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - tests/scale_pcm.py ./scripts/testv/ $SCALE_FACTOR # Default: 3.162 (+10 dB). Can be set in manual trigger. - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec - python3 scripts/parse_xml_report.py report-junit.xml report.csv --clipping artifacts: name: "check-clipping--sha-$CI_COMMIT_SHORT_SHA--results" @@ -1531,6 +1556,7 @@ check-clipping: - report-junit.xml - report.html - report.csv + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "check-clipping results" reports: junit: @@ -1548,10 +1574,10 @@ test-branch-vs-input-passthrough: - 123 script: - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - - make -j + - make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT - exit_code=0 - - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$? + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py report-junit.xml report.csv - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi @@ -1566,6 +1592,7 @@ test-branch-vs-input-passthrough: - report-junit.xml - report.html - report.csv + - $MAKE_BUILD_LOG_FILE_DEFAULT expose_as: "test-branch-vs-input-passthrough results" reports: junit: diff --git a/snippets/basop/build-binaries.sh b/snippets/basop/build-binaries.sh index 7a0f23ba81f55a1eebafe18311e06fd9ae73aeda..64ecc54e75f7bfc19095210ca0a354cf622d30fa 100755 --- a/snippets/basop/build-binaries.sh +++ b/snippets/basop/build-binaries.sh @@ -13,6 +13,7 @@ float-ref) ivas_rend_filename="./$REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" isar_post_rend_filename="./$REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$FLOAT_REF_COMMIT_FILE" + make_log_file="$MAKE_BUILD_LOG_FILE_FLOAT_REF" ;; float-ref-merge-source) ref_to_check_out="$MERGE_SOURCE_FLOAT_REF_COMMIT" @@ -22,6 +23,7 @@ float-ref-merge-source) ivas_rend_filename="./$MERGE_SOURCE_FLOAT_REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" isar_post_rend_filename="./$MERGE_SOURCE_FLOAT_REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$MERGE_SOURCE_FLOAT_REF_COMMIT_FILE" + make_log_file="$MAKE_BUILD_LOG_FILE_FLOAT_REF_MERGE_SOURCE" ;; merge-target) ref_to_check_out="$MERGE_TARGET_COMMIT" @@ -31,6 +33,7 @@ merge-target) ivas_rend_filename="./$MERGE_TARGET_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" isar_post_rend_filename="./$MERGE_TARGET_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$MERGE_TARGET_COMMIT_FILE" + make_log_file="$MAKE_BUILD_LOG_FILE_MERGE_TARGET" ;; dut) ref_to_check_out="$CI_COMMIT_SHA" @@ -40,6 +43,7 @@ dut) ivas_rend_filename="./$DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" isar_post_rend_filename="./$DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY" commit_file="$CUT_COMMIT_FILE" + make_log_file="$MAKE_BUILD_LOG_FILE_DUT" ;; *) echo "Error: invalid mode: $mode" @@ -73,7 +77,7 @@ git checkout "$ref_to_check_out" bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh make clean -make -j "$(nproc)" +make -j "$(nproc)" 2>&1 >$make_log_file # avoid errors in mv when renaming to same file mv IVAS_cod "${CI_PROJECT_DIR}/${ivas_cod_filename}" diff --git a/snippets/copy-ltv-files-to-testv-dir.sh b/snippets/copy-ltv-files-to-testv-dir.sh index 19ee80ae40a98babd0d8fe858773bfe5fe7bff38..f0e6bf8e025cfe59fdd3917f49648f8f4390c575 100755 --- a/snippets/copy-ltv-files-to-testv-dir.sh +++ b/snippets/copy-ltv-files-to-testv-dir.sh @@ -1,7 +1,5 @@ #! /bin/bash -set -euxo pipefail - cd "${CI_PROJECT_DIR}" cp "$LTV_DIR"/*.wav scripts/testv/ diff --git a/snippets/merge-request-comparison-setup-codec.sh b/snippets/merge-request-comparison-setup-codec.sh index 622a4f5b589ae184b148acbc51b98c088863214b..ebec099379e03e718e4de8cc54cf2b6b39a7a9dc 100644 --- a/snippets/merge-request-comparison-setup-codec.sh +++ b/snippets/merge-request-comparison-setup-codec.sh @@ -6,8 +6,7 @@ mv $MERGE_TARGET_ENCODER_PATH $REF_ENCODER_PATH mv $MERGE_TARGET_DECODER_PATH $REF_DECODER_PATH mv $MERGE_TARGET_RENDERER_PATH $REF_RENDERER_PATH -# NOTE: temporary solution for merge to main -if [ -f $MERGE_TARGET_POST_RENDERER_PATH ]; then mv $MERGE_TARGET_POST_RENDERER_PATH $REF_POST_RENDERER_PATH; fi +mv $MERGE_TARGET_POST_RENDERER_PATH $REF_POST_RENDERER_PATH make clean -make -j +make -j 2>&1 >$MAKE_BUILD_LOG_FILE_DEFAULT