diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4843bd3bf05df7c7f29c629673017eddf0174547..b3f455f0281200b79c7d83f49f42c65578735892 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,6 +101,7 @@ stages: - prevalidate - build - test + - compare - deploy # --------------------------------------------------------------- @@ -117,6 +118,24 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" +.mr-fetch-target-branch: &mr-fetch-target-branch + # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching + # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later + - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true + # needed when depth is lower than the number of commits in the branch + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +.mr-get-target-commit: &mr-get-target-commit # compare to last target branch commit before pipeline was created + - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) + +.get-commits-behind-count: &get-commits-behind-count + - echo $CI_COMMIT_SHA + - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) + +.check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs + - if [ $commits_behind_count -ne 0 ]; then echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way.\nMain might have changed during your pipeline run. Run 'git merge origin/main' to update."; exit 1; fi + .build-reference-binaries: &build-reference-binaries - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries @@ -167,6 +186,13 @@ stages: - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + .update-ltv-repo: &update-ltv-repo - cd $LTV_DIR - git pull @@ -180,6 +206,45 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile + +# --------------------------------------------------------------- +# Job templates +# --------------------------------------------------------------- + +# When designing templates, try not to use too much inheritance and +# if multiple templates and extended on, remember that on conflict, +# latest overwrites the parameter. + +# templates for rules +.rules-basis: + rules: + - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) + when: never + - if: $CI_PIPELINE_SOURCE == 'trigger' # Don't run triggered pipeline by default + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-be-release' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'check-clipping' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + when: never + - when: on_success + +.rules-merge-request: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" @@ -683,6 +748,94 @@ build-codec-linux-instrumented-make: - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP - make -j -C $INSTR_DIR +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +lc3-wrapper-unit-test: + extends: + - .test-job-linux + - .rules-merge-request + needs: ["build-codec-linux-make"] + stage: test + script: + - make -j + - scripts/split_rendering/lc3plus/ivas_lc3plus_unit_test + +# compare split-rendering bitexactness between target and source branch +split-rendering-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "30 minutes" + stage: compare + script: + - *print-common-info + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[split*[ -]*non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + # store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + - *mr-fetch-target-branch + - *mr-get-target-commit + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + # build reference binaries + - make -j + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + # back to source branch + - git restore lib_com/options.h # Revert changes back before checking out another branch to avoid conflicts + - git checkout $source_branch_commit_sha + - make clean + - make -j + + ### Run test using scripts and input from main + - if [ $ref_using_target == 1 ]; then git restore lib_com/options.h; fi # Revert changes back before checking out another branch to avoid conflicts + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--job-$CI_JOB_NAME--results" + expire_in: 2 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest split rendering results" + reports: + junit: + - report-junit.xml + # --------------------------------------------------------------- # Short test jobs that run in merge request pipelines # ---------------------------------------------------------------