diff --git a/README.md b/README.md index 81c6e96fbc45eddda7df2632a480582fe320eab7..92a7b194405317965cde5afdf2bebfe83f971205 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,84 @@ Home of CI configs and scripts for: - [IVAS BASOP](https://forge.3gpp.org/rep/sa4/audio/ivas-basop) Refer to the [IVAS Wiki](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/home) for more info on the CI setup. -This repository is mainly meant to share scripts and configs between the two repositories and (for BASOP) between differently configured branches. +This repository is mainly meant to share scripts and configs between the two repositories and between differently configured branches. -## Notes about usage of the snippets +## Structure +``` bash +. +├── includes # folder for (possibly shared) yaml files which are included in the main ones +│   ├── default-variables-basop.yml +│   ├── default-variables.yml +│   ├── job-templates.yml +│   ├── rules-basop.yml +│   └── rules.yml +├── main-basop.yml # CI config for ivas-basop repo +├── main.yml # shared CI config +├── ... +└── snippets # Folder for scripts used in CI only. Shared ones go in the folder directly. Can be small ones or bigger scripts which implement a whole job + ├── basop + │   ├── ... + ├── float + │   └── ... + └── ... +``` + +## Usage + +For both repositories (ivas-codec and ivas-basop), there should be one main yml file in here which includes stuff from `includes`. +Ideally, there is no other CI code in the respective repositories except a minimal `.gitlab-ci.yml` file which includes from here (currently only achieved for ivas-basop), e.g. (from ivas-basop): + +``` yaml +variables: + # note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this + # see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main + +# all CI code and config is included from https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec-ci +include: + - project: ivas-codec-pc/ivas-codec-ci + ref: *IVAS_CODEC_CI_REF + file: main-basop.yml +``` + +This way, any CI change can be developed centrally in this repository and does not need to be merged/ported to different repositories/branches (e.g. keeping ivas-basop main and ivas-float-update in synch). + +### Adding new CI code + +1. Implement your new CI code on a branch in this repo, e.g. `add-awesome-test` and ideally add a merge request for easier review. +2. Test it by opening a branch in the respective ivas-* repository and changing IVAS_CODEC_CI_REF to your WIP branch, so the yaml there should read: + +``` yaml + IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF add-awesome-test +``` + +3. If your change affects multiple repos/branches, you might need to use multiple test branches +4. Once you are done and review is complete, merge `add-awesome-test` to `main` here. The test branches in the other repos are not needed and SHOULD NOT BE MERGED. + +### Overwriting CI variables in the ivas-* repos' CI configs + +Usually, everything should be done in the files here, but if there is the need to still overwrite some configuration/control CI variables - e.g. to switch off something temporarily or in a branch-specific manner - this should be done by in a variables: section at the end of the `.gitlab-ci.yml` file. +That way, everything else is included before and the final overwrite of a - possibly existing - variable will be as you want it to be. + +Example: The variable `DISABLE_HRTF` is used in ivas-basop to disable all testcase that use external HRTF files. +The relevant part in the `.gitlab-ci.yml` file in ivas-basop looks like given below. +In practice, one would uncomment the last three lines to switch off the HRTF testcase in the respective branch only. + +``` yaml +# If you need to set some config variable only in a local branch, then add an overwrite here +# One example is DISABLE_HRTF - this will be set on a branch which is about to be merged and will be removed in a subsequent second MR +# this is more easily done directly here in the child repo + +# these lines are suposed to stay commented out to serve as an example +#variables: +# # set this to true to skip the external HRTF testcases in pytest calls +# DISABLE_HRTF: "false" +``` + +### Some guidelines/hints + +- use only one include section at the top of the main yml file to keep things clear, remember that later stuff overwrites earlier stuff - in general prefer adding scripts here over using yaml anchors in a CI config file - if your script shall modify some environment variable, you need to run it with `source` instead of calling it. +- the merge CI config in the ivas-* repos can be viewed in the pipeline editor under "full configuration" - this is especially useful when making changes which should not functionally affect the CI, then it is helpful to copy the full merged config with and without the changes and compare the files directly or as parsed yaml dict locally. diff --git a/includes/basop-default-variables.yml b/includes/basop-default-variables.yml deleted file mode 100644 index 1da0ef17f3c8c50fd0a66fdc33716216d24fd9a4..0000000000000000000000000000000000000000 --- a/includes/basop-default-variables.yml +++ /dev/null @@ -1,5 +0,0 @@ -variables: - FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" - MERGE_SOURCE_FLOAT_REF_COMMIT_FILE: "merge-source-float-ref-git-sha.txt" - MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" - FLOAT_REF_COMMIT: "ivas-float-update" diff --git a/includes/default-variables-basop.yml b/includes/default-variables-basop.yml new file mode 100644 index 0000000000000000000000000000000000000000..b3b41435db6d8fb5c3efc4255c4f11449d24b341 --- /dev/null +++ b/includes/default-variables-basop.yml @@ -0,0 +1,100 @@ +variables: + EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" + # TODO: cleanup and use global var defined in ivas-codec-ci + FLOAT_REF_BRANCH: "ivas-float-update" + FLOAT_REF_BRANCH_MERGE_SOURCE: "" + SCRIPTS_DIR: "/usr/local/scripts" + LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv" + LONG_TEST_SUITE_NO_RENDERER: "tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_ltv.prm --use_ltv" + SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection" + SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" + LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" + TEST_SUITE: "" + # These path variables are used by the pytest calls. + # They can be overwritten in the job templates to e.g. only test encoder or decoder in the chain + DUT_ENCODER_PATH: "./IVAS_cod_dut" + DUT_DECODER_PATH: "./IVAS_dec_dut" + DUT_RENDERER_PATH: "./IVAS_rend_dut" + DUT_POST_RENDERER_PATH: "./ISAR_post_rend_dut" + REF_ENCODER_PATH: "./IVAS_cod_ref" + REF_DECODER_PATH: "./IVAS_dec_ref" + REF_RENDERER_PATH: "./IVAS_rend_ref" + REF_POST_RENDERER_PATH: "./ISAR_post_rend_ref" + MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target" + MERGE_TARGET_RENDERER_PATH: "./IVAS_rend_merge_target" + MERGE_TARGET_POST_RENDERER_PATH: "./ISAR_post_rend_merge_target" + MERGE_SOURCE_FLOAT_REF_ENCODER_PATH: "./IVAS_cod_merge_source_float_ref" + MERGE_SOURCE_FLOAT_REF_DECODER_PATH: "./IVAS_dec_merge_source_float_ref" + MERGE_SOURCE_FLOAT_REF_RENDERER_PATH: "./IVAS_rend_merge_source_float_ref" + MERGE_SOURCE_FLOAT_REF_POST_RENDERER_PATH: "./ISAR_post_rend_merge_source_float_ref" + # These path variables are used for building the binaries + # They should never be overwritten! + DUT_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_dut" + DUT_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_dut" + DUT_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_rend_dut" + DUT_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./ISAR_post_rend_dut" + REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_ref" + REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_ref" + REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_rend_ref" + REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./ISAR_post_rend_ref" + MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" + MERGE_TARGET_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_rend_merge_target" + MERGE_TARGET_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./ISAR_post_rend_merge_target" + MERGE_SOURCE_FLOAT_REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_merge_source_float_ref" + MERGE_SOURCE_FLOAT_REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_source_float_ref" + MERGE_SOURCE_FLOAT_REF_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_rend_merge_source_float_ref" + MERGE_SOURCE_FLOAT_REF_POST_RENDERER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./ISAR_post_rend_merge_source_float_ref" + LEVEL_SCALING: "1.0" + PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" + TESTCASE_TIMEOUT_STV: 900 + TESTCASE_TIMEOUT_LTV: 2400 + TESTCASE_TIMEOUT_LTV_SANITIZERS: 10800 + CI_REGRESSION_THRESH_MLD: "0.1" + CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" + CI_REGRESSION_THRESH_SSNR: "-1" + CI_REGRESSION_THRESH_ODG: "-0.05" + INSTR_DIR: "scripts/c-code_instrument" + BUILD_WITH_DEBUG_MODE_INFO: "" + ENCODER_TEST: "" + DELTA_ODG: "" + COMPARE_DMX: "" + SPLIT_COMPARISON: "" + SKIP_REGRESSION_CHECK: "" + FAILED_TESTCASES_LIST: "failed-testcases.txt" + ERRORS_TESTCASES_LIST: "errors-testcases.txt" + PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" + MEASURES_FOR_REPORT: "MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG" + FLOAT_REF_COMMIT: "ivas-float-update" + FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt" + CUT_COMMIT_FILE: "CuT-git-sha.txt" + MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt" + MERGE_SOURCE_FLOAT_REF_COMMIT_FILE: "merge-source-float-ref-git-sha.txt" + RUNNER_TAG: "ivas-basop-linux" + LOGS_BACKUP_SOURCE_DIR: "" + LOGS_BACKUP_TARGET_DIR: "" + # set this to true to skip the external HRTF testcases in pytest calls + DISABLE_HRTF: "false" + MANUAL_PIPELINE_TYPE: + description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." + value: 'default' + options: + - 'default' + - 'pytest-compare' + - 'pytest-compare-enc-dmx' + - 'pytest-compare-long' + - 'pytest-compare-long-fx-fx' + - 'pytest-compare-to-input' + - 'pytest-saturation-smoke-test' + - 'evs-26444' + - 'sanitizer' + - 'pytest-renderer' + - 'complexity' + - 'coverage' + - 'voip-be-test' + - 'renderer-framesize-be' + - 'peaq-enc-passthrough' + - 'long-term-logs' + - 'backup-long-term-logs' + - 'test-long-self-test' diff --git a/includes/default-variables.yml b/includes/default-variables.yml index 2774c578ba79b4182014d9a7071649b110dae633..f4b0f60d29049a1a4be8066189ccee93a4bcaf64 100644 --- a/includes/default-variables.yml +++ b/includes/default-variables.yml @@ -13,5 +13,3 @@ variables: SCALE_FACTOR: "3.162" BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" PYTEST_ADDOPTS: "" - # overwrite this via .gitlab/variables.yml in the respective other repo to disable "(model from file)" testcases - DISABLE_HRTF: "false" diff --git a/includes/job-templates.yml b/includes/job-templates.yml index 87c58f1f802d2a6208ce05dcc5889bcb313504dc..031f613ca77a44f8d98771bdab606448883e54fd 100644 --- a/includes/job-templates.yml +++ b/includes/job-templates.yml @@ -2,6 +2,9 @@ # if multiple templates and extended on, remember that on conflict, # latest overwrites the parameter. +# Those two first job templates implement getting scripts from this CI repository +# all jobs in the child repos need to somehow inherit from here. +# If a child job overwrites before_script, then use !reference to also add the part from this jobs .job-linux: tags: - ivas-linux diff --git a/includes/rules-basop.yml b/includes/rules-basop.yml new file mode 100644 index 0000000000000000000000000000000000000000..5eae33e977655e7722ffb2c8570380be69415dd8 --- /dev/null +++ b/includes/rules-basop.yml @@ -0,0 +1,53 @@ +# overwrites the default rules +# should be refactored and unified +.rules-basis: + rules: + # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + variables: + IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' + ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow + # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main + # variables: + # IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing + variables: + IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx' + variables: + IVAS_PIPELINE_NAME: 'Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input' + variables: + IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test' + variables: + IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444' + variables: + IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer' + variables: + IVAS_PIPELINE_NAME: 'Short testvectors sanitizers' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer' + variables: + IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity' + variables: + IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test' + variables: + IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough' + variables: + IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH' + - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch + variables: + IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH' diff --git a/main-basop.yml b/main-basop.yml index fda6a3fae70f41d66d8d642dec2868c1c0765c8c..ea671b24fe3346834c932dcb4121240e3b8f5415 100644 --- a/main-basop.yml +++ b/main-basop.yml @@ -1,6 +1,7 @@ include: - local: main.yml - - local: includes/basop-default-variables.yml + - local: includes/default-variables-basop.yml + - local: includes/rules-basop.yml stages: - .pre