diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 894265e419f91605e36cf5aa61a291e25dade876..a00af3162ca5bd2d4cfa2c49b9ec16c48e60e2ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,76 +1,13 @@ variables: - TESTV_DIR: "/usr/local/testv" - LTV_DIR: "/usr/local/ltv" - EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" - FLOAT_REF_BRANCH: "ivas-float-update" - BUILD_OUTPUT: "build_output.txt" - SCRIPTS_DIR: "/usr/local/scripts" - EXIT_CODE_NON_BE: 123 - EXIT_CODE_FAIL: 1 - LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" - LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" - SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" - SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" - LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" - TEST_SUITE: "" - # These path variables are used by the pytest calls. - # They can be overwritten in the job templates to e.g. only test encoder or decoder in the chain - DUT_ENCODER_PATH: "./IVAS_cod" - DUT_DECODER_PATH: "./IVAS_dec" - REF_ENCODER_PATH: "./IVAS_cod_ref" - REF_DECODER_PATH: "./IVAS_dec_ref" - MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target" - MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target" - # These path variables are used for building the binaries - # They should never be overwritten! - REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_ref" - REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_ref" - MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_merge_target" - MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" - LEVEL_SCALING: "1.0" - IVAS_PIPELINE_NAME: '' - BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" - PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" - TESTCASE_TIMEOUT_STV: 900 - TESTCASE_TIMEOUT_LTV: 2400 - TESTCASE_TIMEOUT_LTV_SANITIZERS: 10800 - CI_REGRESSION_THRESH_MLD: "0.1" - CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" - CI_REGRESSION_THRESH_SSNR: "-1" - CI_REGRESSION_THRESH_ODG: "-0.05" - GIT_CLEAN_FLAGS: -ffdxq - INSTR_DIR: "scripts/c-code_instrument" - BUILD_WITH_DEBUG_MODE_INFO: "" - ENCODER_TEST: "" - DELTA_ODG: "" - COMPARE_DMX: "" - SPLIT_COMPARISON: "" - SKIP_REGRESSION_CHECK: "" - FAILED_TESTCASES_LIST: "failed-testcases.txt" - ERRORS_TESTCASES_LIST: "errors-testcases.txt" - PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" - MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" - FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" - CUT_COMMIT_FILE: "CuT-git-sha.txt" - MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" - MANUAL_PIPELINE_TYPE: - description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." - value: 'default' - options: - - 'default' - - 'pytest-compare' - - 'pytest-compare-enc-dmx' - - 'pytest-compare-long' - - 'pytest-compare-to-input' - - 'pytest-saturation-smoke-test' - - 'evs-26444' - - 'sanitizer' - - 'pytest-renderer' - - 'complexity' - - 'coverage' - - 'voip-be-test' - - 'peaq-enc-passthrough' + # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this + # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF 4eb4c0dfbdc845280a9994b5f7540f69c737537b +include: + - local: .gitlab-ci/variables.yml + - project: ivas-codec-pc/ivas-codec-ci + ref: *IVAS_CODEC_CI_REF + file: main.yml default: interruptible: true # Make all jobs by default interruptible @@ -128,16 +65,6 @@ workflow: variables: IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' - -stages: - - .pre - - prevalidate - - build - - check-be - - test - - postvalidate - - deploy - # --------------------------------------------------------------- # Generic script anchors # --------------------------------------------------------------- @@ -145,13 +72,6 @@ stages: # These can be used later on to do common tasks # Prints useful information for every job and should be used at the beginning of each job -.print-common-info: &print-common-info - - | - echo "Printing common information for build job." - echo "Current job is run on commit $CI_COMMIT_SHA" - echo "Commit time was $CI_COMMIT_TIMESTAMP" - date | xargs echo "System time is" - .print-common-info-windows: &print-common-info-windows - | echo "Printing common information for build job." @@ -159,107 +79,6 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression -.activate-debug-mode-info-if-set: &activate-debug-mode-info-if-set - - if [ "$BUILD_WITH_DEBUG_MODE_INFO" = "true" ]; then - - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h - - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUG_MODE_INFO\ *\)\*\//\1/g" lib_com/options.h - - fi - -.build-float-ref-binaries: &build-float-ref-binaries - - git rev-parse HEAD > $CUT_COMMIT_FILE - - current_commit_sha=$(git rev-parse HEAD) - ### build reference binaries - - git checkout $FLOAT_REF_BRANCH - - git pull origin $FLOAT_REF_BRANCH - - *activate-debug-mode-info-if-set - - make clean - - make -j >> /dev/null - - mv ./IVAS_cod ./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY - - mv ./IVAS_dec ./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY - - mv ./IVAS_rend ./IVAS_rend_ref - ### Return to current branch - - git restore . - - git rev-parse HEAD > $FLOAT_REF_COMMIT_FILE - - git checkout $current_commit_sha - -.build-merge-target-binaries: &build-merge-target-binaries - - current_commit_sha=$(git rev-parse HEAD) - ### build merge target binaries - - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - *activate-debug-mode-info-if-set - - make clean - - make -j >> /dev/null - - mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY - - mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY - - mv ./IVAS_rend ./IVAS_rend_merge_target - ### Return to current branch - - git restore . - - git rev-parse HEAD > $MERGE_TARGET_COMMIT_FILE - - git checkout $current_commit_sha - -.build-float-ref-and-dut-binaries: &build-float-ref-and-dut-binaries -### build reference binaries - - *build-float-ref-binaries -### build dut binaries - - *activate-debug-mode-info-if-set - - make clean - - make -j >> /dev/null - -.build-and-create-float-ref-outputs: &build-and-create-float-ref-outputs - - *build-float-ref-and-dut-binaries - - ### prepare pytest - # create short test vectors - - python3 tests/create_short_testvectors.py - # create references - - exit_code=0 - - - enc_stats_arg="" - - if [ "$ENCODER_TEST" = "true" ]; then - - enc_stats_arg="--enc_stats" - - fi - - - enc_dmx_arg="" - - if [ "$COMPARE_DMX" = "true" ]; then - - enc_dmx_arg="--compare_enc_dmx" - - fi - - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? - -.update-scripts-repo: &update-scripts-repo - - cd $SCRIPTS_DIR - - sed -i '/fetch/d' .git/config # Remove all fetch lines to clean out dead links - - git remote set-branches --add origin $BASOP_CI_BRANCH_PC_REPO # Add currently used branch - - git fetch - - git restore . # Just as a precaution - - git checkout $BASOP_CI_BRANCH_PC_REPO - - git pull origin $BASOP_CI_BRANCH_PC_REPO - - cd - - - cp -r $SCRIPTS_DIR/ci . - - cp -r $SCRIPTS_DIR/scripts . - - cp -r $SCRIPTS_DIR/tests . - - cp $SCRIPTS_DIR/pytest.ini . - -.apply-testv-scaling: &apply-testv-scaling - - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING" - - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING - -.update-ltv-repo: &update-ltv-repo - - cd $LTV_DIR - - git pull - - cd - - -.get-commits-behind-count: &get-commits-behind-count - - echo $CI_COMMIT_SHA - - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) - -.copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir - - cp "$LTV_DIR"/*.wav scripts/testv/ - - cp "$LTV_DIR"/*.met scripts/testv/ - - cp "$LTV_DIR"/*.csv scripts/testv/ - .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile @@ -335,22 +154,18 @@ stages: # Job templates # --------------------------------------------------------------- -# templates to define stages and platforms -.test-job-linux: - tags: - - ivas-basop-linux - -.build-job-linux: - stage: build - timeout: "2 minutes" +# override for centrally defined job to make use of the basop runners +.job-linux: tags: - ivas-basop-linux -.build-job-windows: - stage: build - timeout: "4 minutes" - tags: - - ivas-windows +# custom variant of this template, we need to update the scripts repo before every build +.test-job-linux: + extends: + - .job-linux + before_script: + - !reference [.job-linux, before_script] + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh .print-results-banner: &print-results-banner - set +x @@ -361,7 +176,7 @@ stages: .test-job-linux-needs-testv-dir: extends: .test-job-linux before_script: - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -381,11 +196,11 @@ stages: IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME" script: - set -euxo pipefail - - *print-common-info - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - testcase_timeout=$TESTCASE_TIMEOUT_STV @@ -393,12 +208,13 @@ stages: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ]; then - - *apply-testv-scaling + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -408,7 +224,7 @@ stages: - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; fi - - *build-and-create-float-ref-outputs + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh # DMX comparison only in manual job with no other metrics - if [ "$COMPARE_DMX" = "true" ]; then @@ -491,15 +307,30 @@ stages: - report-junit.xml .check-up-to-date-in-comparison-jobs: &check-up-to-date-in-comparison-jobs - - *get-commits-behind-count - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" if [ $commits_behind_count -ne 0 ]; then set +x echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run. Checking bitexactness or testing for regressions now can result in meaningless results. Run\n\t git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." exit 1 fi -.check-be-to-target-anchor: &check-be-to-target-anchor +.check-be-job: + extends: + - .test-job-linux + before_script: + - !reference [ .test-job-linux, before_script ] + - rm -rf tests/dut tests/ref + variables: + USE_LTV: 0 + DUT_ENCODER_PATH: ./IVAS_cod_ref + DUT_DECODER_PATH: ./IVAS_dec_ref + MERGE_TARGET_ENCODER_PATH: ./IVAS_cod_ref + MERGE_TARGET_DECODER_PATH: ./IVAS_dec_ref + +.check-be-to-target-job: + extends: + - .check-be-job stage: check-be needs: ["build-codec-linux-make"] timeout: "300 minutes" @@ -508,16 +339,16 @@ stages: HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" PYTEST_LOG_TARGET_BRANCH: "pytest-log-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME.txt" script: - - *print-common-info + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - set -euxo pipefail - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - python3 tests/create_short_testvectors.py - if [ $USE_LTV -eq 1 ]; then - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - testcase_timeout=$TESTCASE_TIMEOUT_STV @@ -527,11 +358,11 @@ stages: - python3 scripts/prepare_combined_format_inputs.py - if [ $LEVEL_SCALING != "1.0" ];then - - *apply-testv-scaling + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - - *build-float-ref-binaries - - *build-merge-target-binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh merge-target - make clean - make -j >> /dev/null - *check-up-to-date-in-comparison-jobs @@ -610,8 +441,13 @@ stages: - unzip $PYTEST_CACHE_ARTIFACT - fi -.check-regressions-pytest-anchor: &check-regressions-pytest-anchor +.check-regressions-pytest-job: + extends: + - .check-be-job stage: test + needs: + - job: "check-be-to-target-short-enc-0db" + artifacts: true timeout: "300 minutes" variables: XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" @@ -623,7 +459,7 @@ stages: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: - - *print-common-info + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh # create empty files for all artifacts to suppress warnings in case of no regressions found or all is BE - touch $XML_REPORT_BRANCH $XML_REPORT_MAIN $HTML_REPORT_BRANCH $HTML_REPORT_MAIN $CSV_BRANCH $CSV_MAIN $SUMMARY_HTML_ARTIFACT_NAME $FLOAT_REF_COMMIT_FILE $CUT_COMMIT_FILE $MERGE_TARGET_COMMIT_FILE regressions_crashes.csv regressions_MLD.csv regressions_MAX_ABS_DIFF.csv regressions_MIN_SSNR.csv regressions_MIN_ODG.csv improvements_crashes.csv improvements_MLD.csv improvements_MAX_ABS_DIFF.csv improvements_MIN_SSNR.csv improvements_MIN_ODG.csv @@ -641,11 +477,11 @@ stages: - exit 0 - fi - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - testcase_timeout=$TESTCASE_TIMEOUT_STV @@ -653,7 +489,7 @@ stages: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ];then - - *apply-testv-scaling + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi # check MR title for flag that allows regressions to be mergable @@ -665,7 +501,7 @@ stages: ### run branch first # this per default builds the branch and the reference and creates the reference outputs - - *build-and-create-float-ref-outputs + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-and-create-float-ref-outputs.sh - *check-up-to-date-in-comparison-jobs # need to restore cache again - *overwrite-pytest-cache-with-artifact @@ -758,6 +594,8 @@ stages: .ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor + extends: + - .job-linux stage: test needs: ["build-codec-linux-make"] timeout: "600 minutes" @@ -769,12 +607,14 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' && $IVAS_PYTEST_MSAN - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "sanitizer" script: - - *print-common-info - - *update-scripts-repo - - *copy-ltv-files-to-testv-dir + - !reference [ .job-linux, before_script ] + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - python3 ci/remove_unsupported_testcases.py $PRM_FILES - - *build-float-ref-binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref - set -euxo pipefail - make_args="CLANG=$CLANG_NUM" - if [[ $CLANG_NUM == 3 ]]; then @@ -802,6 +642,8 @@ stages: - report-junit.xml .ivas-pytest-compare-to-input-anchor: &ivas-pytest-compare-to-input-anchor + extends: + - .job-linux stage: test needs: ["build-codec-linux-make"] timeout: "360 minutes" @@ -809,11 +651,13 @@ stages: SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" script: - - *print-common-info - - *update-scripts-repo + - !reference [ .job-linux, before_script ] + + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - testcase_timeout=$TESTCASE_TIMEOUT_LTV - else - testcase_timeout=$TESTCASE_TIMEOUT_STV @@ -821,9 +665,9 @@ stages: - python3 ci/remove_unsupported_testcases.py $PRM_FILES - if [ $LEVEL_SCALING != "1.0" ];then - - *apply-testv-scaling + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - - *build-float-ref-and-dut-binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}") - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING" @@ -896,14 +740,15 @@ uninterruptible: branch-is-up-to-date-with-target-pre: extends: + - .job-linux - .rules-merge-request stage: prevalidate needs: [] tags: - ivas-basop-linux script: - - *get-commits-behind-count - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" if [ $commits_behind_count -ne 0 ]; then echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." exit 1 @@ -911,13 +756,14 @@ branch-is-up-to-date-with-target-pre: branch-is-up-to-date-with-target-post: extends: + - .job-linux - .rules-merge-request stage: postvalidate tags: - ivas-basop-linux script: - - *get-commits-behind-count - | + commits_behind_count="$(bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/get-commits-behind-count.sh)" if [ $commits_behind_count -ne 0 ]; then echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." exit 1 @@ -938,7 +784,7 @@ clang-format-check: needs: [] timeout: "5 minutes" script: - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch - > @@ -992,7 +838,7 @@ build-codec-linux-make: - .build-job-linux timeout: "10 minutes" script: - - *print-common-info + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - *activate-Werror-linux - make -j @@ -1009,8 +855,8 @@ build-codec-linux-instrumented-make: - .build-job-linux timeout: "10 minutes" script: - - *print-common-info - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - bash scripts/prepare_instrumentation.sh -m MEM_ONLY - make -j -C $INSTR_DIR @@ -1028,8 +874,8 @@ build-codec-linux-debugging-make: variables: BUILD_WITH_DEBUG_MODE_INFO: "true" script: - - *print-common-info - - *activate-debug-mode-info-if-set + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/activate-debug-mode-info-if-set.sh - make -j build-codec-windows-msbuild: @@ -1057,178 +903,100 @@ build-codec-windows-msbuild: ### jobs that check for bitexactness of fx encoder and decoder check-be-to-target-short-enc-0db: extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=1.0 - - rm -rf tests/dut tests/ref - <<: *check-be-to-target-anchor + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # -/-0dB + LEVEL_SCALING: "1.0" check-be-to-target-short-enc-+10db: extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=3.162 - - rm -rf tests/dut tests/ref - <<: *check-be-to-target-anchor + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # +10dB + LEVEL_SCALING: "3.162" check-be-to-target-short-enc--10db: extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=0.3162 - - rm -rf tests/dut tests/ref - <<: *check-be-to-target-anchor + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # -10dB + LEVEL_SCALING: "0.3162" check-be-to-target-short-dec-0db: extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref - - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=1.0 - - rm -rf tests/dut tests/ref - <<: *check-be-to-target-anchor + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +/-0dB + LEVEL_SCALING: "1.0" check-be-to-target-short-dec-+10db: extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref - - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=3.162 - - rm -rf tests/dut tests/ref - <<: *check-be-to-target-anchor + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +10dB + LEVEL_SCALING: "3.162" check-be-to-target-short-dec--10db: extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref - - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 - - rm -rf tests/dut tests/ref - <<: *check-be-to-target-anchor + - .check-be-to-target-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # -10dB + LEVEL_SCALING: "0.3162" ### jobs that check for regressions on non-BE testcases check-regressions-short-enc-0db: - stage: test - needs: - - job: "check-be-to-target-short-enc-0db" - artifacts: true extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=1.0 - - rm -rf tests/dut tests/ref - <<: *check-regressions-pytest-anchor + - .check-regressions-pytest-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # +/-0dB + LEVEL_SCALING: "1.0" check-regressions-short-enc-+10db: - stage: test - needs: - - job: "check-be-to-target-short-enc-+10db" - artifacts: true extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=3.162 - - rm -rf tests/dut tests/ref - <<: *check-regressions-pytest-anchor + - .check-regressions-pytest-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # +10dB + LEVEL_SCALING: "3.162" check-regressions-short-enc--10db: - stage: test - needs: - - job: "check-be-to-target-short-enc--10db" - artifacts: true extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=0.3162 - - rm -rf tests/dut tests/ref - <<: *check-regressions-pytest-anchor + - .check-regressions-pytest-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE_ENCODER" + # -10dB + LEVEL_SCALING: "0.3162" check-regressions-short-dec-0db: - stage: test - needs: - - job: "check-be-to-target-short-dec-0db" - artifacts: true extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=1.0 - - rm -rf tests/dut tests/ref - <<: *check-regressions-pytest-anchor + - .check-regressions-pytest-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +/-0dB + LEVEL_SCALING: "1" check-regressions-short-dec-+10db: - stage: test - needs: - - job: "check-be-to-target-short-dec-+10db" - artifacts: true extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=3.162 - - rm -rf tests/dut tests/ref - <<: *check-regressions-pytest-anchor + - .check-regressions-pytest-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # +10dB + LEVEL_SCALING: "3.162" check-regressions-short-dec--10db: - stage: test - needs: - - job: "check-be-to-target-short-dec--10db" - artifacts: true extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_ENCODER_PATH=./IVAS_cod_ref - - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 - - rm -rf tests/dut tests/ref - <<: *check-regressions-pytest-anchor + - .check-regressions-pytest-job + variables: + TEST_SUITE: "$SHORT_TEST_SUITE" + # -10dB + LEVEL_SCALING: "0.3162" # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule @@ -1518,14 +1286,14 @@ ivas-smoke-test-saturation: - USE_LTV=1 - LEVEL_SCALING=32768 - - *print-common-info - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - if [ $USE_LTV -eq 1 ]; then - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh - fi - if [ $LEVEL_SCALING != "1.0" ];then - - *apply-testv-scaling + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh - fi - cp -r scripts/testv/* $TESTV_DIR/ @@ -1561,11 +1329,11 @@ coverage-test-on-main-scheduled: stage: test timeout: 3 hours script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *copy-ltv-files-to-testv-dir - - *build-float-ref-binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref # Build DuT binaries with GCOV - make clean >> /dev/null - make GCOV=1 -j @@ -1622,8 +1390,8 @@ be-2-evs-26444: stage: test timeout: "120 minutes" # To be revisited script: - - *print-common-info - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h - make -j >> /dev/null @@ -1687,7 +1455,7 @@ voip-be-on-merge-request: needs: ["build-codec-linux-make"] timeout: "10 minutes" script: - - *print-common-info + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh - make clean - make -j >> /dev/null - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py @@ -1778,10 +1546,10 @@ voip-be-on-merge-request: GET_WMOPS_ARGS: "mem_only" timeout: 3 hours 30 minutes before_script: - - *print-common-info - - *update-scripts-repo - - *update-ltv-repo - - *build-float-ref-and-dut-binaries + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh - *complexity-measurements-setup # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present) - rm -rf COMPLEXITY/logs @@ -2159,8 +1927,8 @@ pages: rules: - if: $UPDATE_PAGES script: - - *print-common-info - - *update-scripts-repo + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh + - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh - python3 ci/setup_pages.py - ls - ls -lh public diff --git a/.gitlab-ci/variables.yml b/.gitlab-ci/variables.yml new file mode 100644 index 0000000000000000000000000000000000000000..94bb71fb347b5349d9933564dbd35964e9daa0b7 --- /dev/null +++ b/.gitlab-ci/variables.yml @@ -0,0 +1,65 @@ +variables: + EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" + FLOAT_REF_BRANCH: "ivas-float-update" + SCRIPTS_DIR: "/usr/local/scripts" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" + LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE: "" + # These path variables are used by the pytest calls. + # They can be overwritten in the job templates to e.g. only test encoder or decoder in the chain + DUT_ENCODER_PATH: "./IVAS_cod" + DUT_DECODER_PATH: "./IVAS_dec" + REF_ENCODER_PATH: "./IVAS_cod_ref" + REF_DECODER_PATH: "./IVAS_dec_ref" + MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target" + # These path variables are used for building the binaries + # They should never be overwritten! + REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_ref" + REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_ref" + MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" + LEVEL_SCALING: "1.0" + BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" + PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" + TESTCASE_TIMEOUT_STV: 900 + TESTCASE_TIMEOUT_LTV: 2400 + TESTCASE_TIMEOUT_LTV_SANITIZERS: 10800 + CI_REGRESSION_THRESH_MLD: "0.1" + CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" + CI_REGRESSION_THRESH_SSNR: "-1" + CI_REGRESSION_THRESH_ODG: "-0.05" + INSTR_DIR: "scripts/c-code_instrument" + BUILD_WITH_DEBUG_MODE_INFO: "" + ENCODER_TEST: "" + DELTA_ODG: "" + COMPARE_DMX: "" + SPLIT_COMPARISON: "" + SKIP_REGRESSION_CHECK: "" + FAILED_TESTCASES_LIST: "failed-testcases.txt" + ERRORS_TESTCASES_LIST: "errors-testcases.txt" + PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" + MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" + FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" + CUT_COMMIT_FILE: "CuT-git-sha.txt" + MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" + MANUAL_PIPELINE_TYPE: + description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." + value: 'default' + options: + - 'default' + - 'pytest-compare' + - 'pytest-compare-enc-dmx' + - 'pytest-compare-long' + - 'pytest-compare-to-input' + - 'pytest-saturation-smoke-test' + - 'evs-26444' + - 'sanitizer' + - 'pytest-renderer' + - 'complexity' + - 'coverage' + - 'voip-be-test' + - 'peaq-enc-passthrough'