Commit 87550f51 authored by vaclav's avatar vaclav
Browse files

- Merge remote-tracking branch 'remotes/origin/main' into DISCLAIMER_UPDATE_BASOP

parents a16345d1 a3d8a210

.gitlab-ci.yml

+72 −37
Original line number Diff line number Diff line
@@ -23,7 +23,7 @@ variables: 
  PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm"

  TESTCASE_TIMEOUT_STV: 900

  TESTCASE_TIMEOUT_LTV: 2400

  TESTCASE_TIMEOUT_STV_SANITIZERS: 1800

  TESTCASE_TIMEOUT_LTV_SANITIZERS: 6000

  CI_REGRESSION_THRESH_MLD: "0.1"

  CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50"

  CI_REGRESSION_THRESH_SSNR: "-1"
@@ -505,12 +505,13 @@ stages: 
  script:

    - *print-common-info

    - *update-scripts-repo

    - *copy-ltv-files-to-testv-dir

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - *build-reference-and-dut-binaries

    - make clean

    - make -j CLANG=$CLANG_NUM

    - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi

    - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
@@ -931,7 +932,7 @@ ivas-pytest-dec-msan: 
  before_script:

    - CLANG_NUM=1

    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-dec-asan:
@@ -940,7 +941,7 @@ ivas-pytest-dec-asan: 
  before_script:

    - CLANG_NUM=2

    - DUT_ENCODER_PATH=./$REF_ENCODER_PATH

    - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST

    - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-dec-usan:
@@ -1204,6 +1205,9 @@ voip-be-on-merge-request: 


  - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id

  # this is a testing/maintenance mechanism to force getting the log history from a specific job id

  # see below in the concrete complexity jobs

  - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi

  - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - ls
@@ -1214,21 +1218,13 @@ voip-be-on-merge-request: 
  - rm artifacts.zip

  - rm -rf $public_dir



  ### 1.5.part: get the corresponding measurement from ivas-float-update-cmplx

  - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update-cmplx $CI_JOB_NAME $CI_PROJECT_ID)

  ### 1.5.part: get the corresponding measurement from ivas-float-update

  - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id

  - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip

  - unzip -j artifacts_ref.zip "*latest_WMOPS.csv"



  ### 2. part: setup specific for BASOP repo

  # hack for using the reference encoder -> need to build manually to make script use ref enc and BASOP dec

  - mkdir COMPLEXITY

  - cp IVAS_cod_ref COMPLEXITY/IVAS_cod

  # build branch code aain with instrumentation

  - make clean

  - bash scripts/prepare_instrumentation.sh -p BASOP -m MEM_ONLY

  - make -j -C $INSTR_DIR

  - cp $INSTR_DIR/IVAS_dec COMPLEXITY/IVAS_dec

  - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true

  # add file to arguments only if the artifact could be retrieved to prevent error later.

  - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi



.complexity-measurements-prepare-artifacts:

  &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory
@@ -1256,6 +1252,7 @@ voip-be-on-merge-request: 
  stage: test

  variables:

    ret_val: 0

    GET_WMOPS_ARGS: "mem_only basop"

  timeout: 3 hours 30 minutes

  before_script:

    - *print-common-info
@@ -1276,10 +1273,12 @@ complexity-stereo-in-stereo-out: 
    - .complexity-template

  rules:

    - if: $MEASURE_COMPLEXITY_LINUX

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=stereo

    - out_format=stereo

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1290,11 +1289,13 @@ complexity-ism-in-binaural-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 1 hour

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=ISM

    - out_format=BINAURAL

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1305,11 +1306,13 @@ complexity-ism-in-binaural_room_ir-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 2 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=ISM

    - out_format=BINAURAL_ROOM_IR

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1320,11 +1323,13 @@ complexity-ism-in-ext-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 3 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=ISM

    - out_format=EXT

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1335,11 +1340,13 @@ complexity-sba-hoa3-in-hoa3-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 4 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=HOA3

    - out_format=HOA3

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1350,11 +1357,13 @@ complexity-sba-hoa3-in-binaural-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 5 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=HOA3

    - out_format=BINAURAL

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1365,11 +1374,13 @@ complexity-sba-hoa3-in-binaural_room_ir-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 6 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=HOA3

    - out_format=BINAURAL_ROOM_IR

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1380,11 +1391,13 @@ complexity-mc-in-7_1_4-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 7 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=MC

    - out_format=7_1_4

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1395,11 +1408,13 @@ complexity-mc-in-binaural-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 10 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=MC

    - out_format=BINAURAL

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1410,11 +1425,13 @@ complexity-mc-in-binaural_room_ir-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 12 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=MC

    - out_format=BINAURAL_ROOM_IR

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1425,11 +1442,13 @@ complexity-masa-in-ext-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 15 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=MASA

    - out_format=EXT

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1440,11 +1459,13 @@ complexity-masa-in-binaural-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 16 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=MASA

    - out_format=BINAURAL

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1455,11 +1476,13 @@ complexity-masa-in-hoa3-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 17 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=MASA

    - out_format=HOA3

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1470,11 +1493,13 @@ complexity-masa-in-hoa3-out: 
#     - if: $MEASURE_COMPLEXITY_LINUX

#       when: delayed

#       start_in: 13 hours

  # variables:

  #   JOB_ID_INJECT: ""

#   script:

#     - in_format=OMASA

#     - out_format=EXT

#     - ret_val=0

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val
@@ -1485,11 +1510,13 @@ complexity-omasa-in-binaural-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 18 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=OMASA

    - out_format=BINAURAL

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1500,11 +1527,13 @@ complexity-omasa-in-hoa3-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 20 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=OMASA

    - out_format=HOA3

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1515,11 +1544,13 @@ complexity-StereoDmxEVS-stereo-in-mono-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 22 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=StereoDmxEVS

    - out_format=mono

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1534,7 +1565,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: 
#     - in_format=OSBA

#     - out_format=EXT

#     - ret_val=0

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val
@@ -1545,11 +1576,13 @@ complexity-osba-in-binaural-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 22 hours 30 minutes

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=OSBA

    - out_format=BINAURAL

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val
@@ -1560,11 +1593,13 @@ complexity-osba-in-binaural_room_ir-out: 
    - if: $MEASURE_COMPLEXITY_LINUX

      when: delayed

      start_in: 25 hours

  variables:

    JOB_ID_INJECT: ""

  script:

    - in_format=OSBA

    - out_format=BINAURAL_ROOM_IR

    - ret_val=0

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$?

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val

.gitlab/issue_templates/default.md

+13 −3
Original line number Diff line number Diff line
@@ -2,14 +2,24 @@ 


<!--- Add commit SHA used to reproduce -->



- Float reference:

  - Encoder (float):

- Decoder:

  - Decoder (float):

- Fixed point:

  - Encoder (fixed):

  - Decoder (fixed):



# Bug description



# Ways to reproduce



(Clear steps or refer to a failing automated test, e.g. with a pipeline link)

Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx



<!-- Commandline or script -->



```bash



```



<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.

      Add further information to the first row and remove and add labels as necessary.  -->

.gitlab/issue_templates/default_complexity.md

0 → 100644
+34 −0
Original line number Diff line number Diff line 
# Basic info



<!--- Add commit SHA used to reproduce -->



- Float reference:

  - Encoder (float):

  - Decoder (float):

- Fixed point:

  - Encoder (fixed):

  - Decoder (fixed):



# Bug description



|   Metric   |  BASOP  |  Float  |

|------------|---------|---------|

| WMOPS Max. | | |

| WMOPS Min. | | |

| WMOPS Avg. | | |



# Ways to reproduce



Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx



<!-- Commandline or script -->



```bash



```



<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.

      Add further information to the first row and remove and add labels as necessary.  -->



/label ~Priority:: ~Company: ~Lib: ~Module:

/label ~Type:Bug ~Status::ToDo

.gitlab/issue_templates/fixed_point_decoder_crash.md

0 → 100644
+24 −0
Original line number Diff line number Diff line 
# Basic info



<!--- Add commit SHA used to reproduce -->



- Encoder and Decoder (float):

- Decoder (fixed):



# Bug description



# Ways to reproduce



Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx



<!-- Commandline or script -->



```bash



```



<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.

      Add further information to the first row and remove and add labels as necessary.  -->



/label ~Priority:: ~Company: ~Lib: ~Module:

/label ~Type:Bug ~Status::ToDo

.gitlab/issue_templates/fixed_point_encoder_crash.md

0 → 100644
+24 −0
Original line number Diff line number Diff line 
# Basic info



<!--- Add commit SHA used to reproduce -->



- Encoder and Decoder (float):

- Encoder (fixed):



# Bug description



# Ways to reproduce



Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx



<!-- Commandline or script -->



```bash



```



<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.

      Add further information to the first row and remove and add labels as necessary.  -->



/label ~Priority:: ~Company: ~Lib: ~Module:

/label ~Type:Bug ~Status::ToDo