Loading .gitlab-ci.yml +225 −1 Original line number Diff line number Diff line Loading @@ -20,7 +20,11 @@ variables: - 'default' - 'test-be-release' - 'test-long-self-test' - 'compare-ref-vs-cut' REF_COMMIT: description: "Ref commit id or ref release tag. To be used by Manual pipelines: 'compare-ref-vs-cut'" CUT_COMMIT: description: "Cut commit id or cut release tag. To be used by Manual pipeline:'compare-ref-vs-cut'" default: interruptible: true # Make all jobs by default interruptible Loading Loading @@ -50,6 +54,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'compare-ref-vs-cut' variables: IVAS_PIPELINE_NAME: 'BE comparison pipeline: $REF_COMMIT vs $CUT_COMMIT' - if: $CI_PIPELINE_SOURCE == 'trigger' stages: Loading Loading @@ -180,6 +187,8 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec when: never - if: $MANUAL_PIPELINE_TYPE == 'compare-ref-vs-cut' # Skip all the normal jobs when testing manually against release codec when: never - when: on_success .rules-merge-request: Loading Loading @@ -1164,6 +1173,221 @@ codec-comparison-on-main-push: # Manual jobs # --------------------------------------------------------------- .print-common-info-ref-vs-cut: &print-common-info-ref-vs-cut - | echo "Printing common information for build job." echo "REF_COMMIT: $REF_COMMIT." echo "CUT_COMMIT: $CUT_COMMIT." echo "Current job is run on commit $CI_COMMIT_SHA" echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" .rules-compare-commits: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'compare-ref-vs-cut' && $REF_COMMIT && $CUT_COMMIT build-codec-linux-make-ref-vs-cut: extends: - .build-job-with-check-for-warnings - .rules-compare-commits script: - *print-common-info-ref-vs-cut - git checkout $REF_COMMIT - make -j 2>&1 | tee $BUILD_OUTPUT - mv IVAS_cod IVAS_cod_ref - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref # - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? - git checkout $CUT_COMMIT - make -j 2>&1 | tee $BUILD_OUTPUT - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? # use scripts and tests from $CI_COMMIT_SHA - git checkout $CI_COMMIT_SHA # compare bit exactness between target and source branch ivas-pytest-ref-vs-cut: extends: - .test-job-linux - .rules-compare-commits stage: compare needs: ["build-codec-linux-make-ref-vs-cut"] timeout: "10 minutes" script: - *print-common-info-ref-vs-cut # set non-be flag - non_be_flag=1 ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --ref_encoder_path IVAS_cod_ref --ref_decoder_path IVAS_dec_ref -m create_ref - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --ref_encoder_path IVAS_cod_ref --ref_decoder_path IVAS_dec_ref -m create_ref_part2 ### run pytest - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA--cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week when: always paths: - report-junit.xml - report.html expose_as: "pytest ivas results ref vs cut" reports: junit: - report-junit.xml # compare renderer bitexactness between target and source branch renderer-pytest-ref-vs-cut: extends: - .test-job-linux - .rules-compare-commits needs: ["build-codec-linux-make-ref-vs-cut"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" stage: compare script: - *print-common-info-ref-vs-cut # set non-be flag - non_be_flag=1 # run test - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA-cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week when: always paths: - report-junit.xml - report.html expose_as: "pytest renderer results ref vs cut" reports: junit: - report-junit.xml evs-pytest-cut: extends: - .test-job-linux - .rules-compare-commits stage: compare needs: ["build-codec-linux-make-ref-vs-cut"] timeout: "10 minutes" script: - *print-common-info-ref-vs-cut # non-be flag to 0 - non_be_flag=0 ### prepare pytest # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --ref_encoder_path IVAS_cod_ref --ref_decoder_path IVAS_dec_ref -m create_ref --param_file scripts/config/self_test_evs.prm ### run pytest for EVS cases - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA-cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week when: always paths: - report-junit-evs.xml - report.html expose_as: "pytest evs results" reports: junit: - report-junit-evs.xml # compare split-rendering bitexactness between target and source branch split-rendering-pytest-ref-vs-cut: extends: - .test-job-linux - .rules-compare-commits needs: ["build-codec-linux-make-ref-vs-cut"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" stage: compare script: - *print-common-info-ref-vs-cut - make clean - rm *IVAS_cod_ref || true - rm *IVAS_dec_ref || true - rm *IVAS_rend_ref || true # set non-be tag - non_be_flag=1 - git checkout $REF_COMMIT # build reference binaries - *enable-split-rendering - make -j INCLUDE_SPLIT=1 - mv IVAS_cod IVAS_cod_ref - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts # build cut binaries - git checkout $CUT_COMMIT - make clean - *enable-split-rendering - make -j INCLUDE_SPLIT=1 ### Run test using scripts and input from $CI_COMMIT_SHA - git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - git checkout $CI_COMMIT_SHA # run test - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA-cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 2 week when: always paths: - report-junit.xml - report.html expose_as: "pytest split rendering results" reports: junit: - report-junit.xml test-be-to-release: stage: test tags: Loading Loading
.gitlab-ci.yml +225 −1 Original line number Diff line number Diff line Loading @@ -20,7 +20,11 @@ variables: - 'default' - 'test-be-release' - 'test-long-self-test' - 'compare-ref-vs-cut' REF_COMMIT: description: "Ref commit id or ref release tag. To be used by Manual pipelines: 'compare-ref-vs-cut'" CUT_COMMIT: description: "Cut commit id or cut release tag. To be used by Manual pipeline:'compare-ref-vs-cut'" default: interruptible: true # Make all jobs by default interruptible Loading Loading @@ -50,6 +54,9 @@ workflow: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test' variables: IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH' - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'compare-ref-vs-cut' variables: IVAS_PIPELINE_NAME: 'BE comparison pipeline: $REF_COMMIT vs $CUT_COMMIT' - if: $CI_PIPELINE_SOURCE == 'trigger' stages: Loading Loading @@ -180,6 +187,8 @@ stages: when: never - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec when: never - if: $MANUAL_PIPELINE_TYPE == 'compare-ref-vs-cut' # Skip all the normal jobs when testing manually against release codec when: never - when: on_success .rules-merge-request: Loading Loading @@ -1164,6 +1173,221 @@ codec-comparison-on-main-push: # Manual jobs # --------------------------------------------------------------- .print-common-info-ref-vs-cut: &print-common-info-ref-vs-cut - | echo "Printing common information for build job." echo "REF_COMMIT: $REF_COMMIT." echo "CUT_COMMIT: $CUT_COMMIT." echo "Current job is run on commit $CI_COMMIT_SHA" echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" .rules-compare-commits: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'compare-ref-vs-cut' && $REF_COMMIT && $CUT_COMMIT build-codec-linux-make-ref-vs-cut: extends: - .build-job-with-check-for-warnings - .rules-compare-commits script: - *print-common-info-ref-vs-cut - git checkout $REF_COMMIT - make -j 2>&1 | tee $BUILD_OUTPUT - mv IVAS_cod IVAS_cod_ref - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref # - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? - git checkout $CUT_COMMIT - make -j 2>&1 | tee $BUILD_OUTPUT - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? # use scripts and tests from $CI_COMMIT_SHA - git checkout $CI_COMMIT_SHA # compare bit exactness between target and source branch ivas-pytest-ref-vs-cut: extends: - .test-job-linux - .rules-compare-commits stage: compare needs: ["build-codec-linux-make-ref-vs-cut"] timeout: "10 minutes" script: - *print-common-info-ref-vs-cut # set non-be flag - non_be_flag=1 ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --ref_encoder_path IVAS_cod_ref --ref_decoder_path IVAS_dec_ref -m create_ref - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --ref_encoder_path IVAS_cod_ref --ref_decoder_path IVAS_dec_ref -m create_ref_part2 ### run pytest - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA--cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week when: always paths: - report-junit.xml - report.html expose_as: "pytest ivas results ref vs cut" reports: junit: - report-junit.xml # compare renderer bitexactness between target and source branch renderer-pytest-ref-vs-cut: extends: - .test-job-linux - .rules-compare-commits needs: ["build-codec-linux-make-ref-vs-cut"] # TODO: set reasonable timeout, will most likely take less timeout: "20 minutes" stage: compare script: - *print-common-info-ref-vs-cut # set non-be flag - non_be_flag=1 # run test - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer_be_comparison.py || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA-cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week when: always paths: - report-junit.xml - report.html expose_as: "pytest renderer results ref vs cut" reports: junit: - report-junit.xml evs-pytest-cut: extends: - .test-job-linux - .rules-compare-commits stage: compare needs: ["build-codec-linux-make-ref-vs-cut"] timeout: "10 minutes" script: - *print-common-info-ref-vs-cut # non-be flag to 0 - non_be_flag=0 ### prepare pytest # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --ref_encoder_path IVAS_cod_ref --ref_decoder_path IVAS_dec_ref -m create_ref --param_file scripts/config/self_test_evs.prm ### run pytest for EVS cases - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA-cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 1 week when: always paths: - report-junit-evs.xml - report.html expose_as: "pytest evs results" reports: junit: - report-junit-evs.xml # compare split-rendering bitexactness between target and source branch split-rendering-pytest-ref-vs-cut: extends: - .test-job-linux - .rules-compare-commits needs: ["build-codec-linux-make-ref-vs-cut"] # TODO: set reasonable timeout, will most likely take less timeout: "30 minutes" stage: compare script: - *print-common-info-ref-vs-cut - make clean - rm *IVAS_cod_ref || true - rm *IVAS_dec_ref || true - rm *IVAS_rend_ref || true # set non-be tag - non_be_flag=1 - git checkout $REF_COMMIT # build reference binaries - *enable-split-rendering - make -j INCLUDE_SPLIT=1 - mv IVAS_cod IVAS_cod_ref - mv IVAS_dec IVAS_dec_ref - mv IVAS_rend IVAS_rend_ref - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts # build cut binaries - git checkout $CUT_COMMIT - make clean - *enable-split-rendering - make -j INCLUDE_SPLIT=1 ### Run test using scripts and input from $CI_COMMIT_SHA - git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts - git checkout $CI_COMMIT_SHA # run test - exit_code=0 - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering_be_comparison.py || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "compare-$CI_PIPELINE_IID--ref-sha-$CI_COMMIT_SHORT_SHA-cut-sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" expire_in: 2 week when: always paths: - report-junit.xml - report.html expose_as: "pytest split rendering results" reports: junit: - report-junit.xml test-be-to-release: stage: test tags: Loading
