Loading .gitlab-ci.yml +3 −351 Original line number Diff line number Diff line Loading @@ -281,11 +281,13 @@ build-codec-instrumented-linux: # make sure that the codec builds with msan, asan and usan build-codec-sanitizers-linux: extends: - .build-job-linux - .build-job-with-check-for-warnings - .rules-basis script: - *print-common-info - bash ci/build_codec_sanitizers_linux.sh # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...< - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-codec-windows-cmake: extends: Loading Loading @@ -696,286 +698,6 @@ evs-pytest-on-merge-request: script: - *print-common-info - *merge-request-comparison-setup-codec # some helper variables - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi ### prepare pytest # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest for EVS cases - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always paths: - report-junit-evs.xml - report.html expose_as: "pytest evs results" reports: junit: - report-junit-evs.xml voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request stage: compare # Or should it be test? Comparison is done within one git revision needs: ["build-codec-linux-make", codec-smoke-test] timeout: "10 minutes" script: - *print-common-info - bash ci/ivas_voip_be_test.sh clang-format-check: extends: - .test-job-linux - .rules-merge-request variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: validate needs: [] timeout: "5 minutes" script: # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch - > INSTRUCTIONS_GITLAB="To fix formatting issues:\n - download the diff patch available as artifact of this job\n - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n - run: git apply $PATCH_FILE_NAME\n - commit new changes" - > INSTRUCTIONS_README="To fix formatting issues:\n - place the patch file in the root directory of your local IVAS repo\n - run: git apply $PATCH_FILE_NAME\n - commit new changes" - scripts/check-format.sh -af -p 8 || format_problems=$? - if [ $format_problems == 0 ] ; then exit 0; fi - mkdir tmp-formatting-fix - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME" # Print instructions to job output - echo -e "$INSTRUCTIONS_GITLAB" # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface) - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt" - exit $format_problems artifacts: expire_in: 1 day paths: - tmp-formatting-fix/ when: on_failure name: "$ARTIFACT_BASE_NAME" expose_as: 'formatting patch' # check for crashes if first received frame on decoder side is an SID check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request tags: - ivas-linux stage: test # needs: ["build-codec-linux-cmake"] script: - *print-common-info - *update-ltv-repo - *check-for-testvectors - cmake . - make -j # TODO: for some MASA modes, we currently do not have testvectors that actually trigger DTX # SBA is run separately to use shorter part of file - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -v MASA | grep -v SBA) - scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 0:20 || exit_code_no_sba=$? - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep SBA) - scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 70:80 || exit_code_sba=$? - if [ $exit_code_no_sba != 0 || $exit_code_sba != 0]; then exit 1; fi artifacts: paths: - out/logs when: on_failure name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart" expose_as: "logs-sidstart" expire_in: "5 days" selection-test-processing: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request tags: - processing-scripts-linux stage: test needs: ["build-codec-linux-make"] script: # get processing script code - git clone https://forge.3gpp.org/rep/ivas-codec-pc/ivas-processing-scripts.git --single-branch -b main - mkdir -p ivas-processing-scripts/tests/data/testv - cp -r scripts/testv/* ivas-processing-scripts/tests/data/testv/ # copy binaries into local bin dir, those should take precendence over PATH - cp $PROCESSING_SCRIPTS_BIN_DIR/* ivas-processing-scripts/ivas_processing_scripts/bin/ # for testing with native binaries - rm ivas-processing-scripts/ivas_processing_scripts/bin/IVAS*.exe - rm ivas-processing-scripts/ivas_processing_scripts/bin/EVS*.exe # build codec and put into bin dir - make -j - cp ./IVAS_* ivas-processing-scripts/ivas_processing_scripts/bin/ # patch the use_windows_codec_binaries key (weird folding is needed so colons are accepted) - > sed -i "s/use_windows_codec_binaries: true/use_windows_codec_binaries: false/" ivas-processing-scripts/experiments/selection/*/config/*.yml # run experiments test - cd ivas-processing-scripts - python3 -m pytest tests/test_experiments.py::test_generate_test_items -n auto | tee log.txt artifacts: paths: - ivas-processing-scripts/experiments/selection/*/proc_output/*.log - log.txt when: on_failure expire_in: 1 week lc3plus-ensure-no-code-changes: extends: - .test-job-linux - .rules-merge-request stage: validate needs: [] timeout: "5 minutes" script: # Replace code commited to repo with code downloaded from ETSI - ./scripts/lc3plus_lib_setup/get_lc3plus.sh # Ensure git reports no changes - modified_files=$(git status -s) - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - .rules-main-push tags: - ivas-windows stage: test needs: ["build-codec-windows-msbuild"] timeout: "20 minutes" # To be revisited script: - *print-common-info-windows - $winoutdata = $null - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 # copy over to never change the testvector dir - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe - cd evs_be_win_test - python ../ci/run_evs_be_win_test.py # check bitexactness to EVS be-2-evs-linux: extends: - .test-job-linux - .rules-main-push tags: - be-2-evs-temp stage: test needs: ["build-codec-linux-cmake"] timeout: "20 minutes" # To be revisited script: - *print-common-info - mkdir build - cd build - cmake .. - make -j - cd .. # copy over to never change the testvector dir - cp -r $EVS_BE_TEST_DIR ./evs_be_test - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec - cd evs_be_test - python3 ../ci/run_evs_be_test.py codec-comparison-on-main-push: extends: - .test-job-linux - .rules-main-push stage: compare needs: ["build-codec-linux-cmake"] timeout: "30 minutes" # To be revisited script: - *print-common-info - latest_commit=$(git rev-parse HEAD) # Latest commit - *get-previous-merge-commit-sha # Stored in previous_merge_commit shell variable now - echo "Comparing changes from $previous_merge_commit to $latest_commit" - git --no-pager diff --stat $previous_merge_commit..$latest_commit # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - cd .. - rm -rf build/* ### compare to the previous merge commit in the main branch - git fetch origin main - git checkout $previous_merge_commit ### build reference binaries - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - cd .. # helper variable - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[non[ -]*be\]") || true - ref_using_main=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true ### re-checkout the latest commit in the main branch, if ref_using_main is not set - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py Loading Loading @@ -1116,76 +838,6 @@ sanitizer-test-masa: extends: .sanitizer-test-schedule-A rules: - if: $SANITIZER_SCHEDULE_A when: delayed start_in: 21 hours timeout: 3 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS ### --- sanitizer schedule B --- .sanitizer-test-schedule-B: extends: - .sanitizer-test-template timeout: 4 hours sanitizer-test-mc-5_1: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_2: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 4 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_4: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 8 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-7_1: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 12 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-7_1_4: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 16 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS ### --- sanitizer schedule C --- .sanitizer-test-schedule-C: extends: - .sanitizer-test-template timeout: 6 hours sanitizer-test-sba: extends: .sanitizer-test-schedule-C rules: - if: $SANITIZER_SCHEDULE_C Loading Makefile +1 −1 Original line number Diff line number Diff line Loading @@ -67,7 +67,7 @@ LDLIBS += -lm CCCLANG = clang ifeq "$(CLANG)" "1" CC = $(CCCLANG) CFLAGS += -fsanitize=memory -fsanitize-memory-track-origins CFLAGS += -fsanitize=memory LDFLAGS += -fsanitize=memory endif ifeq "$(CLANG)" "2" Loading tests/codec_be_on_mr_selection/__init__.py +23 −6 Original line number Diff line number Diff line Loading @@ -35,6 +35,7 @@ import filecmp from pathlib import Path import subprocess from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT, DTX_ON, FER_5PERC from ..testconfig import MD5_REF_DICT HERE = Path(__file__).parent # set environment variables in CI job Loading Loading @@ -98,8 +99,20 @@ def apply_error_pattern_on_bitstream( subprocess.run(cmd) def files_equal(dut_file, ref_file): return filecmp.cmp(dut_file, ref_file) def is_be_to_ref(dut_file): """ Check bitexactness either by comparing files directly or by comparing MD5 sums """ if MD5_REF_DICT == dict(): ref_file = REF_PATH.joinpath(dut_file.name) is_be = filecmp.cmp(dut_file, ref_file) else: md5_ref = MD5_REF_DICT[dut_file.name] cmd = f"powershell.exe (Get-FileHash {str(dut_file)} -Algorithm MD5).Hash" md5_dut = subprocess.check_output(cmd, shell=True).decode().splitlines()[-1] is_be = md5_ref == md5_dut return is_be def run_check( Loading @@ -113,6 +126,7 @@ def run_check( decoder_frontend, is_ref_creation, input_file_num=None, keep_files=True, ): sampling_rate = 48 output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] Loading Loading @@ -168,12 +182,15 @@ def run_check( # this should not be a problem as both the reference and the tdut output was generated by the codec, so # diverging headers should also indicate a problem - still, keep in mind if something bogus happens if not is_ref_creation: ref_output = REF_PATH.joinpath(dut_output.name) if not files_equal(dut_output, ref_output): if not is_be_to_ref(dut_output): pytest.fail("Decoder output differs from reference") elif not keep_files: os.remove(dut_output) os.remove(dut_bitstream) for md in metadata: md_suffix = "".join(md.suffixes) dut_md = DUT_PATH.joinpath(dut_output.with_suffix(md_suffix).name) ref_md = REF_PATH.joinpath(dut_output.with_suffix(md_suffix).name) if not files_equal(dut_md, ref_md): if not is_be_to_ref(dut_md): pytest.fail("Metadata file {md.name} differs from reference") elif not keep_files: os.remove(dut_md) tests/codec_be_on_mr_selection/test_experiments.py +6 −0 Original line number Diff line number Diff line Loading @@ -52,6 +52,7 @@ def test_p800( dut_encoder_frontend, dut_decoder_frontend, update_ref, keep_files, ): run_check( experiment, Loading @@ -63,6 +64,7 @@ def test_p800( dut_encoder_frontend, dut_decoder_frontend, update_ref == 1, keep_files=keep_files, ) Loading @@ -79,6 +81,7 @@ def test_bs1534_no_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref, keep_files, ): category = "" run_check( Loading @@ -92,6 +95,7 @@ def test_bs1534_no_masa( dut_decoder_frontend, update_ref == 1, input_file_num=input_file_num, keep_files=keep_files, ) Loading @@ -111,6 +115,7 @@ def test_bs1534_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref, keep_files, ): run_check( experiment, Loading @@ -123,4 +128,5 @@ def test_bs1534_masa( dut_decoder_frontend, update_ref == 1, input_file_num=input_file_num, keep_files=keep_files, ) tests/conftest.py +12 −0 Original line number Diff line number Diff line Loading @@ -133,6 +133,12 @@ def pytest_addoption(parser): " Use --keep_files to prevent these deletions.", ) parser.addoption( "--selection_be_md5_file", type=Path, help="Path to file with md5 sums for the reference signals of the selection-BE test" ) @pytest.fixture(scope="session", autouse=True) def update_ref(request): Loading Loading @@ -513,3 +519,9 @@ def pytest_configure(config): ) if config.option.param_file: testconfig.PARAM_FILE = config.option.param_file if config.option.selection_be_md5_file: md5_file_path = config.option.selection_be_md5_file if not platform.system() == "Windows": raise NotImplementedError("MD5 comparison is currently hardcoded for windows") with open(md5_file_path) as f: testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()} Loading
.gitlab-ci.yml +3 −351 Original line number Diff line number Diff line Loading @@ -281,11 +281,13 @@ build-codec-instrumented-linux: # make sure that the codec builds with msan, asan and usan build-codec-sanitizers-linux: extends: - .build-job-linux - .build-job-with-check-for-warnings - .rules-basis script: - *print-common-info - bash ci/build_codec_sanitizers_linux.sh # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...< - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-codec-windows-cmake: extends: Loading Loading @@ -696,286 +698,6 @@ evs-pytest-on-merge-request: script: - *print-common-info - *merge-request-comparison-setup-codec # some helper variables - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi ### prepare pytest # create references - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### Run test using branch scripts and input - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi ### run pytest for EVS cases - exit_code=0 - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --html=report.html --self-contained-html --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - *merge-request-comparison-check allow_failure: exit_codes: - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" expire_in: 1 week when: always paths: - report-junit-evs.xml - report.html expose_as: "pytest evs results" reports: junit: - report-junit-evs.xml voip-be-on-merge-request: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request stage: compare # Or should it be test? Comparison is done within one git revision needs: ["build-codec-linux-make", codec-smoke-test] timeout: "10 minutes" script: - *print-common-info - bash ci/ivas_voip_be_test.sh clang-format-check: extends: - .test-job-linux - .rules-merge-request variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: validate needs: [] timeout: "5 minutes" script: # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch - > INSTRUCTIONS_GITLAB="To fix formatting issues:\n - download the diff patch available as artifact of this job\n - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n - run: git apply $PATCH_FILE_NAME\n - commit new changes" - > INSTRUCTIONS_README="To fix formatting issues:\n - place the patch file in the root directory of your local IVAS repo\n - run: git apply $PATCH_FILE_NAME\n - commit new changes" - scripts/check-format.sh -af -p 8 || format_problems=$? - if [ $format_problems == 0 ] ; then exit 0; fi - mkdir tmp-formatting-fix - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME" # Print instructions to job output - echo -e "$INSTRUCTIONS_GITLAB" # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface) - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt" - exit $format_problems artifacts: expire_in: 1 day paths: - tmp-formatting-fix/ when: on_failure name: "$ARTIFACT_BASE_NAME" expose_as: 'formatting patch' # check for crashes if first received frame on decoder side is an SID check-first-frame-is-sid: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request tags: - ivas-linux stage: test # needs: ["build-codec-linux-cmake"] script: - *print-common-info - *update-ltv-repo - *check-for-testvectors - cmake . - make -j # TODO: for some MASA modes, we currently do not have testvectors that actually trigger DTX # SBA is run separately to use shorter part of file - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep -v MASA | grep -v SBA) - scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 0:20 || exit_code_no_sba=$? - modes=$(scripts/runIvasCodec.py -l | grep dtx | grep SBA) - scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 70:80 || exit_code_sba=$? - if [ $exit_code_no_sba != 0 || $exit_code_sba != 0]; then exit 1; fi artifacts: paths: - out/logs when: on_failure name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart" expose_as: "logs-sidstart" expire_in: "5 days" selection-test-processing: extends: - .test-job-linux-needs-testv-dir - .rules-merge-request tags: - processing-scripts-linux stage: test needs: ["build-codec-linux-make"] script: # get processing script code - git clone https://forge.3gpp.org/rep/ivas-codec-pc/ivas-processing-scripts.git --single-branch -b main - mkdir -p ivas-processing-scripts/tests/data/testv - cp -r scripts/testv/* ivas-processing-scripts/tests/data/testv/ # copy binaries into local bin dir, those should take precendence over PATH - cp $PROCESSING_SCRIPTS_BIN_DIR/* ivas-processing-scripts/ivas_processing_scripts/bin/ # for testing with native binaries - rm ivas-processing-scripts/ivas_processing_scripts/bin/IVAS*.exe - rm ivas-processing-scripts/ivas_processing_scripts/bin/EVS*.exe # build codec and put into bin dir - make -j - cp ./IVAS_* ivas-processing-scripts/ivas_processing_scripts/bin/ # patch the use_windows_codec_binaries key (weird folding is needed so colons are accepted) - > sed -i "s/use_windows_codec_binaries: true/use_windows_codec_binaries: false/" ivas-processing-scripts/experiments/selection/*/config/*.yml # run experiments test - cd ivas-processing-scripts - python3 -m pytest tests/test_experiments.py::test_generate_test_items -n auto | tee log.txt artifacts: paths: - ivas-processing-scripts/experiments/selection/*/proc_output/*.log - log.txt when: on_failure expire_in: 1 week lc3plus-ensure-no-code-changes: extends: - .test-job-linux - .rules-merge-request stage: validate needs: [] timeout: "5 minutes" script: # Replace code commited to repo with code downloaded from ETSI - ./scripts/lc3plus_lib_setup/get_lc3plus.sh # Ensure git reports no changes - modified_files=$(git status -s) - if [[ $modified_files ]]; then printf 'LC3plus codebase was modified!\n\n'"$modified_files"'\n\n'; exit $EXIT_CODE_FAIL; fi # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- # check bitexactness to EVS windows binaries be-2-evs-windows: extends: - .rules-main-push tags: - ivas-windows stage: test needs: ["build-codec-windows-msbuild"] timeout: "20 minutes" # To be revisited script: - *print-common-info-windows - $winoutdata = $null - MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Release | tee -variable winoutdata - $winoutdata | Out-File $BUILD_OUTPUT -Encoding Utf8 # copy over to never change the testvector dir - cp -r $EVS_BE_WIN_TEST_DIR ./evs_be_win_test - cp IVAS_cod.exe ./evs_be_win_test/bin/IVAS_cod.exe - cp IVAS_dec.exe ./evs_be_win_test/bin/IVAS_dec.exe - cd evs_be_win_test - python ../ci/run_evs_be_win_test.py # check bitexactness to EVS be-2-evs-linux: extends: - .test-job-linux - .rules-main-push tags: - be-2-evs-temp stage: test needs: ["build-codec-linux-cmake"] timeout: "20 minutes" # To be revisited script: - *print-common-info - mkdir build - cd build - cmake .. - make -j - cd .. # copy over to never change the testvector dir - cp -r $EVS_BE_TEST_DIR ./evs_be_test - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec - cd evs_be_test - python3 ../ci/run_evs_be_test.py codec-comparison-on-main-push: extends: - .test-job-linux - .rules-main-push stage: compare needs: ["build-codec-linux-cmake"] timeout: "30 minutes" # To be revisited script: - *print-common-info - latest_commit=$(git rev-parse HEAD) # Latest commit - *get-previous-merge-commit-sha # Stored in previous_merge_commit shell variable now - echo "Comparing changes from $previous_merge_commit to $latest_commit" - git --no-pager diff --stat $previous_merge_commit..$latest_commit # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - cd .. - rm -rf build/* ### compare to the previous merge commit in the main branch - git fetch origin main - git checkout $previous_merge_commit ### build reference binaries - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - cd .. # helper variable - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[non[ -]*be\]") || true - ref_using_main=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true ### re-checkout the latest commit in the main branch, if ref_using_main is not set - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py Loading Loading @@ -1116,76 +838,6 @@ sanitizer-test-masa: extends: .sanitizer-test-schedule-A rules: - if: $SANITIZER_SCHEDULE_A when: delayed start_in: 21 hours timeout: 3 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS ### --- sanitizer schedule B --- .sanitizer-test-schedule-B: extends: - .sanitizer-test-template timeout: 4 hours sanitizer-test-mc-5_1: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_2: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 4 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-5_1_4: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 8 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-7_1: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 12 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS sanitizer-test-mc-7_1_4: extends: .sanitizer-test-schedule-B rules: - if: $SANITIZER_SCHEDULE_B when: delayed start_in: 16 hours script: - *update-ltv-repo - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS ### --- sanitizer schedule C --- .sanitizer-test-schedule-C: extends: - .sanitizer-test-template timeout: 6 hours sanitizer-test-sba: extends: .sanitizer-test-schedule-C rules: - if: $SANITIZER_SCHEDULE_C Loading
Makefile +1 −1 Original line number Diff line number Diff line Loading @@ -67,7 +67,7 @@ LDLIBS += -lm CCCLANG = clang ifeq "$(CLANG)" "1" CC = $(CCCLANG) CFLAGS += -fsanitize=memory -fsanitize-memory-track-origins CFLAGS += -fsanitize=memory LDFLAGS += -fsanitize=memory endif ifeq "$(CLANG)" "2" Loading
tests/codec_be_on_mr_selection/__init__.py +23 −6 Original line number Diff line number Diff line Loading @@ -35,6 +35,7 @@ import filecmp from pathlib import Path import subprocess from .constants import OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT, DTX_ON, FER_5PERC from ..testconfig import MD5_REF_DICT HERE = Path(__file__).parent # set environment variables in CI job Loading Loading @@ -98,8 +99,20 @@ def apply_error_pattern_on_bitstream( subprocess.run(cmd) def files_equal(dut_file, ref_file): return filecmp.cmp(dut_file, ref_file) def is_be_to_ref(dut_file): """ Check bitexactness either by comparing files directly or by comparing MD5 sums """ if MD5_REF_DICT == dict(): ref_file = REF_PATH.joinpath(dut_file.name) is_be = filecmp.cmp(dut_file, ref_file) else: md5_ref = MD5_REF_DICT[dut_file.name] cmd = f"powershell.exe (Get-FileHash {str(dut_file)} -Algorithm MD5).Hash" md5_dut = subprocess.check_output(cmd, shell=True).decode().splitlines()[-1] is_be = md5_ref == md5_dut return is_be def run_check( Loading @@ -113,6 +126,7 @@ def run_check( decoder_frontend, is_ref_creation, input_file_num=None, keep_files=True, ): sampling_rate = 48 output_mode, options = OUTPUT_MODES_AND_OPTIONS_FOR_EXPERIMENT[experiment] Loading Loading @@ -168,12 +182,15 @@ def run_check( # this should not be a problem as both the reference and the tdut output was generated by the codec, so # diverging headers should also indicate a problem - still, keep in mind if something bogus happens if not is_ref_creation: ref_output = REF_PATH.joinpath(dut_output.name) if not files_equal(dut_output, ref_output): if not is_be_to_ref(dut_output): pytest.fail("Decoder output differs from reference") elif not keep_files: os.remove(dut_output) os.remove(dut_bitstream) for md in metadata: md_suffix = "".join(md.suffixes) dut_md = DUT_PATH.joinpath(dut_output.with_suffix(md_suffix).name) ref_md = REF_PATH.joinpath(dut_output.with_suffix(md_suffix).name) if not files_equal(dut_md, ref_md): if not is_be_to_ref(dut_md): pytest.fail("Metadata file {md.name} differs from reference") elif not keep_files: os.remove(dut_md)
tests/codec_be_on_mr_selection/test_experiments.py +6 −0 Original line number Diff line number Diff line Loading @@ -52,6 +52,7 @@ def test_p800( dut_encoder_frontend, dut_decoder_frontend, update_ref, keep_files, ): run_check( experiment, Loading @@ -63,6 +64,7 @@ def test_p800( dut_encoder_frontend, dut_decoder_frontend, update_ref == 1, keep_files=keep_files, ) Loading @@ -79,6 +81,7 @@ def test_bs1534_no_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref, keep_files, ): category = "" run_check( Loading @@ -92,6 +95,7 @@ def test_bs1534_no_masa( dut_decoder_frontend, update_ref == 1, input_file_num=input_file_num, keep_files=keep_files, ) Loading @@ -111,6 +115,7 @@ def test_bs1534_masa( dut_encoder_frontend, dut_decoder_frontend, update_ref, keep_files, ): run_check( experiment, Loading @@ -123,4 +128,5 @@ def test_bs1534_masa( dut_decoder_frontend, update_ref == 1, input_file_num=input_file_num, keep_files=keep_files, )
tests/conftest.py +12 −0 Original line number Diff line number Diff line Loading @@ -133,6 +133,12 @@ def pytest_addoption(parser): " Use --keep_files to prevent these deletions.", ) parser.addoption( "--selection_be_md5_file", type=Path, help="Path to file with md5 sums for the reference signals of the selection-BE test" ) @pytest.fixture(scope="session", autouse=True) def update_ref(request): Loading Loading @@ -513,3 +519,9 @@ def pytest_configure(config): ) if config.option.param_file: testconfig.PARAM_FILE = config.option.param_file if config.option.selection_be_md5_file: md5_file_path = config.option.selection_be_md5_file if not platform.system() == "Windows": raise NotImplementedError("MD5 comparison is currently hardcoded for windows") with open(md5_file_path) as f: testconfig.MD5_REF_DICT = {line.split()[0]: line.split()[1] for line in f.readlines()}
