Commit 1d47bf7b authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into fix_binary_loading_tests

parents cec64c8e b7be58e4

.gitlab-ci.yml

+116 −70
Original line number Diff line number Diff line
@@ -22,6 +22,8 @@ variables: 
      - 'test-long-self-test'

      - 'ivas-conformance'

  GIT_CLEAN_FLAGS: -ffdxq

  TESTCASE_TIMEOUT_STV_SANITIZERS: 180

  TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200





default:
@@ -186,6 +188,18 @@ stages: 
  - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt"

  - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace '<TreatWarningAsError>false', '<TreatWarningAsError>true' | Set-Content -Path $_.FullName }



# to be reused in MR and LTV-scheduled sanitizer test jobs

# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section

.sanitizer-selftest-anchor: &sanitizer-selftest-anchor

  script:

    - *print-common-info

    - *copy-ltv-files-to-testv-dir

    - make clean

    - make -j CLANG=$CLANG_NUM

    - testcase_timeout=$SELFTEST_SANITY_TIMEOUT

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec





# ---------------------------------------------------------------

# Job templates

# ---------------------------------------------------------------
@@ -255,6 +269,39 @@ stages: 
    - python3 scripts/prepare_combined_format_inputs.py

    - cp -r scripts/testv/* $TESTV_DIR/



.sanitizer-selftest-on-mr:

  stage: test

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



.sanitizer-selftest-ltv:

  stage: test

  extends:

    - .test-job-linux-needs-testv-dir

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 2 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit.xml



# ---------------------------------------------------------------

# .pre jobs for setting up things

# ---------------------------------------------------------------
@@ -449,83 +496,32 @@ codec-smoke-test: 
# code selftest testvectors with memory-sanitizer binaries

codec-msan:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  stage: test

  needs: ["build-codec-sanitizers-linux"]

  script:

    - *print-common-info

    - make clean

    - make -j CLANG=1

    - testcase_timeout=180

    - exit_code=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?

    - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang memory-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "msan selftest results"

    reports:

      junit:

        - report-junit.xml

    - .sanitizer-selftest-on-mr

  before_script:

    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with address-sanitizer binaries

codec-asan:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  stage: test

  needs: ["build-codec-sanitizers-linux"]

  script:

    - *print-common-info

    - make clean

    - make -j CLANG=2

    - testcase_timeout=180

    - exit_code=0

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?

    - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang address-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "asan selftest results"

    reports:

      junit:

        - report-junit.xml

    - .sanitizer-selftest-on-mr

  before_script:

    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with address-sanitizer binaries

# code selftest testvectors with undefined-behaviour-sanitizer binaries

codec-usan:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request

  stage: test

  needs: ["build-codec-sanitizers-linux"]

  script:

    - *print-common-info

    - make clean

    - make -j CLANG=3

    - testcase_timeout=180

    - exit_code=0

    - UBSAN_OPTIONS=suppressions=scripts/ubsan.supp,report_error_type=1 python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py -v --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code=$?

    - if [ $exit_code != 0 ] ; then echo "Run errors found by Clang undefined-behavior-sanitizer"; exit 1; fi

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    paths:

      - report-junit.xml

      - report.html

    expose_as: "usan selftest results"

    reports:

      junit:

        - report-junit.xml

    - .sanitizer-selftest-on-mr

  before_script:

    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"

  <<: *sanitizer-selftest-anchor



# test renderer executable

renderer-smoke-test:
@@ -1397,6 +1393,56 @@ test-long-self-test: 
# ---------------------------------------------------------------

# Scheduled jobs on main

# ---------------------------------------------------------------



# code selftest long testvectors with memory-sanitizer binaries

ltv-msan:

  extends:

    - .sanitizer-selftest-ltv

  rules:

    - if: $SANITIZER_SCHEDULE_E

  timeout: 1 hour

  tags:

    - ivas-linux-fast

  before_script:

    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"

  <<: *sanitizer-selftest-anchor



# code selftest long testvectors with address-sanitizer binaries

ltv-asan:

  extends:

    - .sanitizer-selftest-ltv

  rules:

    - if: $SANITIZER_SCHEDULE_E

      when: delayed

      start_in: 1 hours

  tags:

    - ivas-linux-fast

  timeout: 1 hour

  before_script:

    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"

  <<: *sanitizer-selftest-anchor



# code selftest long testvectors with undefined-behaviour-sanitizer binaries

ltv-usan:

  extends:

    - .sanitizer-selftest-ltv

  rules:

    - if: $SANITIZER_SCHEDULE_E

      when: delayed

      start_in: 2 hours

  tags:

    - ivas-linux-fast

  timeout: 1 hour

  before_script:

    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="tests/test_param_file_ltv.py"

  <<: *sanitizer-selftest-anchor



.sanitizer-test-template:

  extends:

    - .test-job-linux

.gitlab/issue_templates/asan_ltv_error.md

0 → 100644
+39 −0
Original line number Diff line number Diff line 
### Basic info



- Commit SHA: 



### Bug description



Clang ASAN sanitizer test in pipeline found an error:



<!--- Copy sanitizer traceback from command line here -->

```



```



<!--- Copy test job URL here -->

Link to test pipeline: XXX



### Ways to reproduce



Using the [scripts](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/Software-development/pyivastest-howto#how-to-reproduce-tests):

<!--- check correct sanitizer type -->

<!--- add error pattern if needed -->

```

python3 scripts/IvasBuildAndRunChecks.py --checks CLANG2 -m MODE -p /path/to/my/local/ci_linux_ltv_local.json

```

or directly:



```

make clean

make -j CLANG=2

./IVAS_cod ...

networkSimulator_g192 dly_profile.dat bit bit_err trace_dump 1

./IVAS_dec ...

```



<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.

      Add further information to the first row and remove and add labels as necessary.  -->



/label ~"Priority::Critical" ~Company: ~Subpart:

/label ~Type:Bug ~Status::ToDo

.gitlab/issue_templates/msan_ltv_error.md

0 → 100644
+46 −0
Original line number Diff line number Diff line 
### Basic info



- Commit SHA: 



### Bug description



Clang MSAN sanitizer test in pipeline found an error:



<!--- Copy sanitizer traceback from command line here -->

```



```



<!--- Copy test job URL here -->

Link to test pipeline: XXX



### Ways to reproduce



Activate [origin-tracking](https://clang.llvm.org/docs/MemorySanitizer.html#msan-origins) (more detailed traceback about where the undefined value came from) by appending

```

 -fsanitize-memory-track-origins

```

in the `Makefile` at lines 71 and 72. Note that this may increase runtime heavily.



Using the [scripts](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/Software-development/pyivastest-howto#how-to-reproduce-tests):

<!--- check correct sanitizer type -->

<!--- add error pattern if needed -->



```

python3 scripts/IvasBuildAndRunChecks.py --checks CLANG1 -m MODE -p /path/to/my/local/ci_linux_ltv_local.json

```

or directly:



```

make clean

make -j CLANG=1

./IVAS_cod ...

networkSimulator_g192 dly_profile.dat bit bit_err trace_dump 1

./IVAS_dec ...

```



<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.

      Add further information to the first row and remove and add labels as necessary.  -->



/label ~"Priority::Critical" ~Company: ~Subpart:

/label ~Type:Bug ~Status::ToDo

.gitlab/issue_templates/sanitizer_error.md→.gitlab/issue_templates/usan_ltv_error.md

+4 −4
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ 


### Bug description



Clang (m,a,u)san sanitizer test in pipeline found an error:

Clang USAN sanitizer test in pipeline found an error:



<!--- Copy sanitizer traceback from command line here -->

```
@@ -20,15 +20,15 @@ Using the [scripts](https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec/-/wikis/ 
<!--- check correct sanitizer type -->

<!--- add error pattern if needed -->

```

python3 scripts/IvasBuildAndRunChecks.py --checks CLANGX -m MODE -p /path/to/my/local/ci_linux_ltv_local.json --usan_supp_file scripts/ubsan.supp

python3 scripts/IvasBuildAndRunChecks.py --checks CLANG3 -m MODE -p /path/to/my/local/ci_linux_ltv_local.json --usan_supp_file scripts/ubsan.supp

```

or directly:



```

make clean

make -j CLANG=X

make -j CLANG=3

./IVAS_cod ...

eid-xor -vbr -fer bit ep_015.g192 bit_fer

networkSimulator_g192 dly_profile.dat bit bit_err trace_dump 1

./IVAS_dec ...

```

ci/run_scheduled_sanitizer_test.py

+1 −1
Original line number Diff line number Diff line
@@ -55,7 +55,7 @@ GENPATT_CMD = f"gen-patt -tailstat -fer -g192 -gamma 0 -rate 0.15 -tol 0.001 -re 
MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]

AMBISONICS_MODES = ["HOA3", "HOA2", "FOA", "PlanarHOA3", "PlanarHOA2", "PlanarFOA"]

TIMEOUT = (

    60 * 15

    60 * 20

)  # timeout of 15 minutes per en/decoding to safeguard against endless loops



CONSOLE_OUT_FILE = "output_san.txt"