Commit 620be4c7 authored by Vladimir Malenovsky's avatar Vladimir Malenovsky
Browse files

Merge branch 'main' into ci/add-logging-for-encoder

parents 5636a113 04440bc5

.gitlab-ci.yml

+293 −8
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ variables: 
  SCRIPTS_DIR: "/usr/local/scripts"

  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm"

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv"

  SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection"

  TEST_SUITE: ""

  DUT_ENCODER_PATH: "./IVAS_cod"
@@ -26,6 +26,7 @@ variables: 
  CI_REGRESSION_THRESH_SSNR: "-1"

  CI_REGRESSION_THRESH_ODG: "-0.05"

  GIT_CLEAN_FLAGS: -ffdxq

  INSTR_DIR: "scripts/c-code_instrument"

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."

    value: 'default'
@@ -37,6 +38,7 @@ variables: 
      - 'evs-26444'

      - 'sanitizer-stv'

      - 'pytest-renderer'

      - 'complexity'





default:
@@ -51,7 +53,7 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      variables:

        IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'

    ### disabled for now because pipeline setup is redundant with MR pipeline with current workflow

    ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow

    # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    #   variables:

    #     IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH'
@@ -76,6 +78,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity'

      variables:

        IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
@@ -113,6 +118,7 @@ stages: 
  - mv ./IVAS_rend ./IVAS_rend_ref



  ### build test binaries

  - git restore .

  - git checkout $current_commit_sha

  - make clean

  - make -j
@@ -125,9 +131,7 @@ stages: 
  - python3 tests/create_short_testvectors.py

  # create references

  - exit_code=0

  - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

  - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? # Catch exit code to prevent halt in case this step produces zero tests



  - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?



.update-scripts-repo: &update-scripts-repo

  - cd $SCRIPTS_DIR
@@ -431,7 +435,7 @@ stages: 
    - make -j CLANG=$CLANG_NUM

    - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi

    - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

    - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always
@@ -530,10 +534,12 @@ build-codec-linux-instrumented-make: 
      when: never

  extends:

    - .build-job-linux

  timeout: "7 minutes"

  script:

    - *print-common-info

    - sed -i".bak" "s/\/\*#define WMOPS 1\*\//#define WMOPS 1/g" lib_com/count.h

    - make -j

    - *update-scripts-repo

    - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP

    - make -j -C $INSTR_DIR



# ---------------------------------------------------------------

# Short test jobs that run in merge request pipelines
@@ -900,6 +906,285 @@ ivas-pytest-renderer: 
  <<: *ivas-pytest-anchor





# ---------------------------------------------------------------

# Complexity measurement jobs

# ---------------------------------------------------------------



.complexity-measurements-setup:

  &complexity-measurements-setup # create necessary environment

  ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work

  - mkdir -p wmops/logs



  - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id

  - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - ls

  - public_dir="$CI_JOB_NAME-public"

  # if is needed to catch case when no artifact is there (first run), similarly as above

  - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/;  fi

  - ls wmops

  - rm artifacts.zip

  - rm -rf $public_dir



  ### 2. part: setup specific for BASOP repo

  # hack for using the reference encoder -> need to build manually to make script use ref enc and BASOP dec

  - mkdir COMPLEXITY

  - cp IVAS_cod_ref COMPLEXITY/IVAS_cod

  # build branch code aain with instrumentation

  - make clean

  - bash scripts/prepare_instrumentation.sh -p BASOP -m MEM_ONLY

  - make -j -C $INSTR_DIR

  - cp $INSTR_DIR/IVAS_dec COMPLEXITY/IVAS_dec



.complexity-measurements-prepare-artifacts:

  &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory

  - public_dir="$CI_JOB_NAME-public"

  - mkdir $public_dir

  - mv -f wmops/log_*_all.txt wmops/*.js ${public_dir}/

  # move logfiles for links

  - mkdir $public_dir/logs

  # first move logs

  - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g")

  - echo $log_files

  - ls wmops/logs

  - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done

  # copy index page blueprint

  - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html

  # patch the format in the title

  - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html

  # do separately here to avoid overwrite complaints by mv

  - mv -f ci/complexity_measurements/style.css ${public_dir}/

  - ls $public_dir



.complexity-template:

  extends:

    - .test-job-linux

  timeout: 3 hours 30 minutes

  stage: test

  variables:

    ret_val: 0

  before_script:

    - *print-common-info

    - *update-scripts-repo

    - *update-ltv-repo

    - *build-reference-and-dut-binaries

    - *complexity-measurements-setup

    - which coan

  artifacts:

    name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"

    when: always

    expire_in: 1 week

    paths:

      - $CI_JOB_NAME-public



complexity-stereo-in-stereo-out:

  extends:

    - .complexity-template

  rules:

    - if: $MEASURE_COMPLEXITY_LINUX

  script:

    - in_format=stereo

    - out_format=stereo

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val



# complexity-stereo-in-7_1_4-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#   script:

#     - in_format=stereo

#     - out_format=7_1_4

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-ism-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 1 hours

#   script:

#     - in_format=ISM

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-ism-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 1 hours

#   script:

#     - in_format=ISM

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-hoa3-in-hoa3-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=HOA3

#     - out_format=HOA3

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-hoa3-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=HOA3

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-osba-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OSBA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-osba-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OSBA

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-mc-in-7_1_4-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MC

#     - out_format=7_1_4

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-mc-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MC

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-masa-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MASA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-masa-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MASA

#     - out_format="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-omasa-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OMASA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-omasa-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OMASA

#     - out_format="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-StereoDmxEVS-stereo-in-mono-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=StereoDmxEVS

#     - out_format=mono

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val





# job that sets up gitlab pages website

pages:

  stage: deploy

Makefile

+4 −0
Original line number Diff line number Diff line
@@ -103,6 +103,10 @@ CFLAGS += -fprofile-arcs -ftest-coverage -fprofile-update=atomic 
LDFLAGS += -fprofile-arcs -ftest-coverage -fprofile-update=atomic

endif



ifeq "$(WMOPS)" "1"

CFLAGS += -DWMOPS=1

endif



ifeq "$(STRIP)" "1"

CFLAGS   += -fdata-sections -ffunction-sections

ifneq ($(UNAME_S),Darwin)

lib_com/basop_util.c

+3 −3
Original line number Diff line number Diff line
@@ -872,7 +872,7 @@ Word16 divide3232( Word32 L_num, Word32 L_denom ) 
#endif





    sign = L_and( L_xor( L_num, L_denom ), 0x80000000 );

    sign = L_and( L_xor( L_num, L_denom ), (Word32) 0x80000000 );



    L_num = L_abs( L_num );

    L_denom = L_abs( L_denom );
@@ -1520,7 +1520,7 @@ Word32 imult3216( Word32 x, Word16 y ) 
    Mpy_32_16_ss( x, y, &mh, &ml );



    mh = L_shl( mh, 15 );

    ml = lshr( ml, 1 );

    ml = (UWord16) lshr( (Word16) ml, 1 );



    return L_or( mh, L_deposit_l( ml ) );

}
@@ -2459,7 +2459,7 @@ Word16 BASOP_util_atan2( /* o: atan2(y,x) [-pi,pi] Q13 */ 
    }



    /* --- division */

    L_sign = L_and( L_xor( x, y ), 0x80000000 );

    L_sign = L_and( L_xor( x, y ), (Word32) 0x80000000 );



    q = 32767 /*1.0f Q15*/; /* y/x = neg/zero = -Inf */

    sf = 0;

lib_com/bitstream_fx.c

+2 −2
Original line number Diff line number Diff line
@@ -422,8 +422,8 @@ UWord16 get_next_indice_fx( /* o : value of the indice */ 


    FOR( i = 0; i < nb_bits; i++ )

    {

        value = lshl( value, 1 );

        value = add( value, st_fx->bit_stream[add( st_fx->next_bit_pos, i )] );

        value = (UWord16) L_shl( value, 1 );

        value = (UWord16) L_add( value, st_fx->bit_stream[add( st_fx->next_bit_pos, i )] );

    }



    /* update the position in the bitstream */

lib_com/cnst.h

+1 −0
Original line number Diff line number Diff line
@@ -2749,6 +2749,7 @@ enum 
#define LG10                                  24660       /*  10*log10(2)  in Q13                 */

#define LG10_s3_0                             16440       /* 10*log10(2)/1.55 = 1.00343331 in Q14              */

#define LOG2_10                               27213      /* log base 2 of 10 in Q12 */

#define LOG2_10_Q29                           1783446566      /* log base 2 of 10 in Q12 */

#define LOG10_2_Q31                           646456993      /* inverse log base 10 of 2 in Q31 */

#define MU_MA_FX                              10923     /* original prediction factor for the AMR WB tables (Q15) */