Loading .gitlab-ci.yml +72 −37 Original line number Diff line number Diff line Loading @@ -23,7 +23,7 @@ variables: PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 TESTCASE_TIMEOUT_LTV_SANITIZERS: 6000 CI_REGRESSION_THRESH_MLD: "0.1" CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" CI_REGRESSION_THRESH_SSNR: "-1" Loading Loading @@ -505,12 +505,13 @@ stages: script: - *print-common-info - *update-scripts-repo - *copy-ltv-files-to-testv-dir - python3 ci/remove_unsupported_testcases.py $PRM_FILES - *build-reference-and-dut-binaries - make clean - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" Loading Loading @@ -931,7 +932,7 @@ ivas-pytest-dec-msan: before_script: - CLANG_NUM=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: Loading @@ -940,7 +941,7 @@ ivas-pytest-dec-asan: before_script: - CLANG_NUM=2 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: Loading Loading @@ -1204,6 +1205,9 @@ voip-be-on-merge-request: - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id # this is a testing/maintenance mechanism to force getting the log history from a specific job id # see below in the concrete complexity jobs - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - ls Loading @@ -1214,21 +1218,13 @@ voip-be-on-merge-request: - rm artifacts.zip - rm -rf $public_dir ### 1.5.part: get the corresponding measurement from ivas-float-update-cmplx - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update-cmplx $CI_JOB_NAME $CI_PROJECT_ID) ### 1.5.part: get the corresponding measurement from ivas-float-update - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" ### 2. part: setup specific for BASOP repo # hack for using the reference encoder -> need to build manually to make script use ref enc and BASOP dec - mkdir COMPLEXITY - cp IVAS_cod_ref COMPLEXITY/IVAS_cod # build branch code aain with instrumentation - make clean - bash scripts/prepare_instrumentation.sh -p BASOP -m MEM_ONLY - make -j -C $INSTR_DIR - cp $INSTR_DIR/IVAS_dec COMPLEXITY/IVAS_dec - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi .complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory Loading Loading @@ -1256,6 +1252,7 @@ voip-be-on-merge-request: stage: test variables: ret_val: 0 GET_WMOPS_ARGS: "mem_only basop" timeout: 3 hours 30 minutes before_script: - *print-common-info Loading @@ -1276,10 +1273,12 @@ complexity-stereo-in-stereo-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX variables: JOB_ID_INJECT: "" script: - in_format=stereo - out_format=stereo - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1290,11 +1289,13 @@ complexity-ism-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 1 hour variables: JOB_ID_INJECT: "" script: - in_format=ISM - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1305,11 +1306,13 @@ complexity-ism-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 2 hours variables: JOB_ID_INJECT: "" script: - in_format=ISM - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1320,11 +1323,13 @@ complexity-ism-in-ext-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 3 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=ISM - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1335,11 +1340,13 @@ complexity-sba-hoa3-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 4 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=HOA3 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1350,11 +1357,13 @@ complexity-sba-hoa3-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 5 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1365,11 +1374,13 @@ complexity-sba-hoa3-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 6 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1380,11 +1391,13 @@ complexity-mc-in-7_1_4-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 7 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=MC - out_format=7_1_4 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1395,11 +1408,13 @@ complexity-mc-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 10 hours variables: JOB_ID_INJECT: "" script: - in_format=MC - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1410,11 +1425,13 @@ complexity-mc-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 12 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=MC - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1425,11 +1442,13 @@ complexity-masa-in-ext-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 15 hours variables: JOB_ID_INJECT: "" script: - in_format=MASA - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1440,11 +1459,13 @@ complexity-masa-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 16 hours variables: JOB_ID_INJECT: "" script: - in_format=MASA - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1455,11 +1476,13 @@ complexity-masa-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 17 hours variables: JOB_ID_INJECT: "" script: - in_format=MASA - out_format=HOA3 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1470,11 +1493,13 @@ complexity-masa-in-hoa3-out: # - if: $MEASURE_COMPLEXITY_LINUX # when: delayed # start_in: 13 hours # variables: # JOB_ID_INJECT: "" # script: # - in_format=OMASA # - out_format=EXT # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - exit $ret_val Loading @@ -1485,11 +1510,13 @@ complexity-omasa-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 18 hours variables: JOB_ID_INJECT: "" script: - in_format=OMASA - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1500,11 +1527,13 @@ complexity-omasa-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 20 hours variables: JOB_ID_INJECT: "" script: - in_format=OMASA - out_format=HOA3 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1515,11 +1544,13 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 22 hours variables: JOB_ID_INJECT: "" script: - in_format=StereoDmxEVS - out_format=mono - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1534,7 +1565,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: # - in_format=OSBA # - out_format=EXT # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - exit $ret_val Loading @@ -1545,11 +1576,13 @@ complexity-osba-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 22 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=OSBA - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1560,11 +1593,13 @@ complexity-osba-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 25 hours variables: JOB_ID_INJECT: "" script: - in_format=OSBA - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading .gitlab/issue_templates/default.md +13 −3 Original line number Diff line number Diff line Loading @@ -2,14 +2,24 @@ <!--- Add commit SHA used to reproduce --> - Float reference: - Encoder (float): - Decoder: - Decoder (float): - Fixed point: - Encoder (fixed): - Decoder (fixed): # Bug description # Ways to reproduce (Clear steps or refer to a failing automated test, e.g. with a pipeline link) Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> Loading .gitlab/issue_templates/default_complexity.md 0 → 100644 +34 −0 Original line number Diff line number Diff line # Basic info <!--- Add commit SHA used to reproduce --> - Float reference: - Encoder (float): - Decoder (float): - Fixed point: - Encoder (fixed): - Decoder (fixed): # Bug description | Metric | BASOP | Float | |------------|---------|---------| | WMOPS Max. | | | | WMOPS Min. | | | | WMOPS Avg. | | | # Ways to reproduce Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> /label ~Priority:: ~Company: ~Lib: ~Module: /label ~Type:Bug ~Status::ToDo .gitlab/issue_templates/fixed_point_decoder_crash.md 0 → 100644 +24 −0 Original line number Diff line number Diff line # Basic info <!--- Add commit SHA used to reproduce --> - Encoder and Decoder (float): - Decoder (fixed): # Bug description # Ways to reproduce Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> /label ~Priority:: ~Company: ~Lib: ~Module: /label ~Type:Bug ~Status::ToDo .gitlab/issue_templates/fixed_point_encoder_crash.md 0 → 100644 +24 −0 Original line number Diff line number Diff line # Basic info <!--- Add commit SHA used to reproduce --> - Encoder and Decoder (float): - Encoder (fixed): # Bug description # Ways to reproduce Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> /label ~Priority:: ~Company: ~Lib: ~Module: /label ~Type:Bug ~Status::ToDo Loading
.gitlab-ci.yml +72 −37 Original line number Diff line number Diff line Loading @@ -23,7 +23,7 @@ variables: PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 TESTCASE_TIMEOUT_LTV_SANITIZERS: 6000 CI_REGRESSION_THRESH_MLD: "0.1" CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" CI_REGRESSION_THRESH_SSNR: "-1" Loading Loading @@ -505,12 +505,13 @@ stages: script: - *print-common-info - *update-scripts-repo - *copy-ltv-files-to-testv-dir - python3 ci/remove_unsupported_testcases.py $PRM_FILES - *build-reference-and-dut-binaries - make clean - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" Loading Loading @@ -931,7 +932,7 @@ ivas-pytest-dec-msan: before_script: - CLANG_NUM=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: Loading @@ -940,7 +941,7 @@ ivas-pytest-dec-asan: before_script: - CLANG_NUM=2 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: Loading Loading @@ -1204,6 +1205,9 @@ voip-be-on-merge-request: - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id # this is a testing/maintenance mechanism to force getting the log history from a specific job id # see below in the concrete complexity jobs - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - ls Loading @@ -1214,21 +1218,13 @@ voip-be-on-merge-request: - rm artifacts.zip - rm -rf $public_dir ### 1.5.part: get the corresponding measurement from ivas-float-update-cmplx - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update-cmplx $CI_JOB_NAME $CI_PROJECT_ID) ### 1.5.part: get the corresponding measurement from ivas-float-update - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" ### 2. part: setup specific for BASOP repo # hack for using the reference encoder -> need to build manually to make script use ref enc and BASOP dec - mkdir COMPLEXITY - cp IVAS_cod_ref COMPLEXITY/IVAS_cod # build branch code aain with instrumentation - make clean - bash scripts/prepare_instrumentation.sh -p BASOP -m MEM_ONLY - make -j -C $INSTR_DIR - cp $INSTR_DIR/IVAS_dec COMPLEXITY/IVAS_dec - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true # add file to arguments only if the artifact could be retrieved to prevent error later. - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi .complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory Loading Loading @@ -1256,6 +1252,7 @@ voip-be-on-merge-request: stage: test variables: ret_val: 0 GET_WMOPS_ARGS: "mem_only basop" timeout: 3 hours 30 minutes before_script: - *print-common-info Loading @@ -1276,10 +1273,12 @@ complexity-stereo-in-stereo-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX variables: JOB_ID_INJECT: "" script: - in_format=stereo - out_format=stereo - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1290,11 +1289,13 @@ complexity-ism-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 1 hour variables: JOB_ID_INJECT: "" script: - in_format=ISM - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1305,11 +1306,13 @@ complexity-ism-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 2 hours variables: JOB_ID_INJECT: "" script: - in_format=ISM - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1320,11 +1323,13 @@ complexity-ism-in-ext-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 3 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=ISM - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1335,11 +1340,13 @@ complexity-sba-hoa3-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 4 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=HOA3 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1350,11 +1357,13 @@ complexity-sba-hoa3-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 5 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1365,11 +1374,13 @@ complexity-sba-hoa3-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 6 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1380,11 +1391,13 @@ complexity-mc-in-7_1_4-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 7 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=MC - out_format=7_1_4 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1395,11 +1408,13 @@ complexity-mc-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 10 hours variables: JOB_ID_INJECT: "" script: - in_format=MC - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1410,11 +1425,13 @@ complexity-mc-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 12 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=MC - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1425,11 +1442,13 @@ complexity-masa-in-ext-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 15 hours variables: JOB_ID_INJECT: "" script: - in_format=MASA - out_format=EXT - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1440,11 +1459,13 @@ complexity-masa-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 16 hours variables: JOB_ID_INJECT: "" script: - in_format=MASA - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1455,11 +1476,13 @@ complexity-masa-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 17 hours variables: JOB_ID_INJECT: "" script: - in_format=MASA - out_format=HOA3 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1470,11 +1493,13 @@ complexity-masa-in-hoa3-out: # - if: $MEASURE_COMPLEXITY_LINUX # when: delayed # start_in: 13 hours # variables: # JOB_ID_INJECT: "" # script: # - in_format=OMASA # - out_format=EXT # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - exit $ret_val Loading @@ -1485,11 +1510,13 @@ complexity-omasa-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 18 hours variables: JOB_ID_INJECT: "" script: - in_format=OMASA - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1500,11 +1527,13 @@ complexity-omasa-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 20 hours variables: JOB_ID_INJECT: "" script: - in_format=OMASA - out_format=HOA3 - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1515,11 +1544,13 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 22 hours variables: JOB_ID_INJECT: "" script: - in_format=StereoDmxEVS - out_format=mono - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1534,7 +1565,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: # - in_format=OSBA # - out_format=EXT # - ret_val=0 # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? # - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? # - *complexity-measurements-prepare-artifacts # - exit $ret_val Loading @@ -1545,11 +1576,13 @@ complexity-osba-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 22 hours 30 minutes variables: JOB_ID_INJECT: "" script: - in_format=OSBA - out_format=BINAURAL - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading @@ -1560,11 +1593,13 @@ complexity-osba-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 25 hours variables: JOB_ID_INJECT: "" script: - in_format=OSBA - out_format=BINAURAL_ROOM_IR - ret_val=0 - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val Loading
.gitlab/issue_templates/default.md +13 −3 Original line number Diff line number Diff line Loading @@ -2,14 +2,24 @@ <!--- Add commit SHA used to reproduce --> - Float reference: - Encoder (float): - Decoder: - Decoder (float): - Fixed point: - Encoder (fixed): - Decoder (fixed): # Bug description # Ways to reproduce (Clear steps or refer to a failing automated test, e.g. with a pipeline link) Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> Loading
.gitlab/issue_templates/default_complexity.md 0 → 100644 +34 −0 Original line number Diff line number Diff line # Basic info <!--- Add commit SHA used to reproduce --> - Float reference: - Encoder (float): - Decoder (float): - Fixed point: - Encoder (fixed): - Decoder (fixed): # Bug description | Metric | BASOP | Float | |------------|---------|---------| | WMOPS Max. | | | | WMOPS Min. | | | | WMOPS Avg. | | | # Ways to reproduce Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> /label ~Priority:: ~Company: ~Lib: ~Module: /label ~Type:Bug ~Status::ToDo
.gitlab/issue_templates/fixed_point_decoder_crash.md 0 → 100644 +24 −0 Original line number Diff line number Diff line # Basic info <!--- Add commit SHA used to reproduce --> - Encoder and Decoder (float): - Decoder (fixed): # Bug description # Ways to reproduce Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> /label ~Priority:: ~Company: ~Lib: ~Module: /label ~Type:Bug ~Status::ToDo
.gitlab/issue_templates/fixed_point_encoder_crash.md 0 → 100644 +24 −0 Original line number Diff line number Diff line # Basic info <!--- Add commit SHA used to reproduce --> - Encoder and Decoder (float): - Encoder (fixed): # Bug description # Ways to reproduce Box folder: ...\Box_EXTERNAL_IVAS_BASOP_VERIFICATION\issues\issue-xxxx <!-- Commandline or script --> ```bash ``` <!--- Below are labels that will be added but are not shown in description. This is a template to help fill them. Add further information to the first row and remove and add labels as necessary. --> /label ~Priority:: ~Company: ~Lib: ~Module: /label ~Type:Bug ~Status::ToDo
