Commit a622625e authored by vaclav's avatar vaclav
Browse files

Merge branch 'main' into my_development

parents 76d5747c 4ce4c288

.gitlab-ci.yml

+104 −22
Original line number Diff line number Diff line 
variables:

  TESTV_DIR: "/usr/local/testv"

  LTV_DIR: "/usr/local/ltv"

  BUILD_OUTPUT: "build_output.txt"

  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"

  SANITIZER_TESTS: "CLANG1 CLANG2"
@@ -82,15 +83,17 @@ stages: 
  - mv IVAS_cod_test IVAS_cod

  - mv IVAS_dec_test IVAS_dec



  ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything)

  - git checkout $source_branch_commit_sha



.merge-request-comparison-check: &merge-request-comparison-check

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact  cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi

    - exit 0



.update-ltv-repo: &update-ltv-repo

  - cd $LTV_DIR

  - git pull

  - cd -



# ---------------------------------------------------------------

# Job templates

# ---------------------------------------------------------------
@@ -334,6 +337,10 @@ ivas-pytest-on-merge-request: 


    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true



    ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there

    - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi



    ### prepare pytest

    # create short test vectors
@@ -342,6 +349,9 @@ ivas-pytest-on-merge-request: 
    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?
@@ -375,11 +385,18 @@ evs-pytest-on-merge-request: 
    

    # some helper variables - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true

    - ref_using_main=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true



    ### If ref_using_main is not set, checkoug the source branch to use scripts and input from there

    - if [ $ref_using_main == 0 ]; then git checkout $source_branch_commit_sha; fi



    ### prepare pytest

    # create references

    - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm



    ### Run test using branch scripts and input

    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    ### run pytest for EVS cases

    - exit_code=0

    - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$?
@@ -400,6 +417,49 @@ evs-pytest-on-merge-request: 
      junit:

        - report-junit-evs.xml



clang-format-check:

  extends:

    - .test-job-linux

    - .rules-merge-request

  variables:

    ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix"

  stage: validate

  needs: []

  timeout: "5 minutes"

  script:

    # Set up variables. This can't be done in the "variables" section because variables are not expanded properly there

    - PATCH_FILE_NAME="$ARTIFACT_BASE_NAME".patch

    - >

      INSTRUCTIONS_GITLAB="To fix formatting issues:\n

      - download the diff patch available as artifact of this job\n

      - unzip the artifact and place the patch file in the root directory of your local IVAS repo\n

      - run: git apply $PATCH_FILE_NAME\n

      - commit new changes"

    - >

      INSTRUCTIONS_README="To fix formatting issues:\n

      - place the patch file in the root directory of your local IVAS repo\n

      - run: git apply $PATCH_FILE_NAME\n

      - commit new changes"

    

    - scripts/check-format.sh -af -p 8 || format_problems=$?

    - if [ $format_problems == 0 ] ; then exit 0; fi



    - mkdir tmp-formatting-fix

    - git diff > "tmp-formatting-fix/$PATCH_FILE_NAME"



    # Print instructions to job output

    - echo -e "$INSTRUCTIONS_GITLAB"



    # Include readme in the artifact, in case someone misses the job printout (e.g. getting the artifact via MR interface)

    - echo -e "$INSTRUCTIONS_README" > "tmp-formatting-fix/readme.txt"



    - exit $format_problems

  artifacts:

    paths:

      - tmp-formatting-fix/

    when: on_failure

    name: "$ARTIFACT_BASE_NAME"

    expose_as: 'formatting patch'



# ---------------------------------------------------------------

# Test jobs for main branch
@@ -472,11 +532,12 @@ codec-comparison-on-main-push: 
    - mv IVAS_dec ../IVAS_dec_ref

    - cd ..



    ### re-checkout the latest commit in the main branch

    - git checkout $latest_commit



    # helper variable - "|| true" to prevent failures from grep not finding anything

    - non_be_flag=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[non[ -]*be\]") || true

    - ref_using_main=$(echo $CI_COMMIT_MESSAGE | grep -c --ignore-case "\[ref[ -]*using[ -]*main\]") || true



    ### re-checkout the latest commit in the main branch, if ref_using_main is not set

    - if [ $ref_using_main == 0 ]; then git checkout $latest_commit;fi



    ### prepare pytest

    # create short test vectors
@@ -488,6 +549,9 @@ codec-comparison-on-main-push: 
    - python3 -m pytest tests -v --update_ref 1 -m create_ref

    - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2



    ### re-checkout the latest commit here, if ref_using_main is set

    - if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi



    ### run pytest

    - exit_code=0

    - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$?
@@ -513,10 +577,13 @@ codec-comparison-on-main-push: 
# ---------------------------------------------------------------

.sanitizer-test-template:

  extends:

    # TODO: still needed since MASA ltv vectors are not there yet

    #       when they were added, we can add a needs-ltv-dir template

    - .test-job-linux-needs-testv-dir

  stage: test

  tags:

    - sanitizer_test_main

  timeout: "2 hours"

  artifacts:

    name: "$CI_JOB_NAME--main--sha-$CI_COMMIT_SHORT_SHA"

    when: always
@@ -530,6 +597,7 @@ sanitizer-test-mono: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS



sanitizer-test-stereo:
@@ -537,8 +605,9 @@ sanitizer-test-stereo: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 20 minutes

      start_in: 1 hour

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS



sanitizer-test-stereodmxevs:
@@ -546,17 +615,19 @@ sanitizer-test-stereodmxevs: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 40 minutes

      start_in: 2 hours

  script:

    - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py StereoDmxEVS mono --tests $SANITIZER_TESTS



sanitizer-test-ism1:

  extends: .sanitizer-test-template

  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 1 hour

      start_in: 3 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism2:
@@ -564,8 +635,9 @@ sanitizer-test-ism2: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 1 hour 30 minutes

      start_in: 4 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism3:
@@ -573,8 +645,9 @@ sanitizer-test-ism3: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 2 hours

      start_in: 6 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-ism4:
@@ -582,8 +655,9 @@ sanitizer-test-ism4: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 2 hours 30 minutes

      start_in: 8 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1:
@@ -591,8 +665,9 @@ sanitizer-test-mc-5_1: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 3 hours

      start_in: 10 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1_2:
@@ -600,8 +675,9 @@ sanitizer-test-mc-5_1_2: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 4 hours

      start_in: 12 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-5_1_4:
@@ -609,8 +685,9 @@ sanitizer-test-mc-5_1_4: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 5 hours

      start_in: 14 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-7_1:
@@ -618,8 +695,9 @@ sanitizer-test-mc-7_1: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 6 hours

      start_in: 16 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-mc-7_1_4:
@@ -627,8 +705,9 @@ sanitizer-test-mc-7_1_4: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 7 hours

      start_in: 18 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-masa:
@@ -636,8 +715,9 @@ sanitizer-test-masa: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 8 hours

      start_in: 20 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS



sanitizer-test-sba:
@@ -645,8 +725,9 @@ sanitizer-test-sba: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 9 hours

      start_in: 22 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



sanitizer-test-planarsba:
@@ -654,8 +735,9 @@ sanitizer-test-planarsba: 
  rules:

    - if: $IS_SANITIZER_TEST_RUN

      when: delayed

      start_in: 10 hours

      start_in: 24 hours

  script:

    - *update-ltv-repo

    - python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS



# GCOV/LCOV coverage analysis of self_test suite

apps/encoder.c

+22 −16
Original line number Diff line number Diff line
@@ -120,10 +120,8 @@ typedef struct 
    const char *ca_config_file;

    bool mimeOutput;



#ifdef AGC_ENABLE_FOR_LBR

#ifdef DEBUG_AGC_ENCODER_CMD_OPTION

    IVAS_ENC_AGC agc;

#else

    bool agc;

#endif

    bool pca;

#ifdef DEBUG_FOA_AGC
@@ -444,11 +442,25 @@ int main( 
            }

            break;

        case IVAS_ENC_INPUT_SBA:

            if ( ( error =

                       IVAS_ENC_ConfigureForAmbisonics(

                           hIvasEnc,

                           arg.inputFs,

                           totalBitrate,

                           arg.max_bwidth_user,

                           bandwidth,

                           arg.dtxConfig,

                           arg.inputFormatConfig.sba.order,

                           arg.inputFormatConfig.sba.isPlanar,

#ifdef DEBUG_AGC_ENCODER_CMD_OPTION

                           arg.agc,

#endif

                           arg.pca

#ifdef DEBUG_SBA_AUDIO_DUMP

            if ( ( error = IVAS_ENC_ConfigureForAmbisonics( hIvasEnc, arg.inputFs, totalBitrate, arg.max_bwidth_user, bandwidth, arg.dtxConfig, arg.inputFormatConfig.sba.order, arg.inputFormatConfig.sba.isPlanar, arg.agc, arg.pca, &numTransportChannels ) ) != IVAS_ERR_OK )

#else

            if ( ( error = IVAS_ENC_ConfigureForAmbisonics( hIvasEnc, arg.inputFs, totalBitrate, arg.max_bwidth_user, bandwidth, arg.dtxConfig, arg.inputFormatConfig.sba.order, arg.inputFormatConfig.sba.isPlanar, arg.agc, arg.pca ) ) != IVAS_ERR_OK )

                           ,

                           &numTransportChannels

#endif

                           ) ) != IVAS_ERR_OK )

            {

                fprintf( stderr, "\nIVAS_ENC_ConfigureForAmbisonics failed: %s\n\n", IVAS_ENC_GetErrorMessage( error ) );

                goto cleanup;
@@ -874,10 +886,8 @@ static void initArgStruct( EncArguments *arg ) 
    arg->ca_config_file = NULL;

    arg->mimeOutput = false;



#ifdef AGC_ENABLE_FOR_LBR

#ifdef DEBUG_AGC_ENCODER_CMD_OPTION

    arg->agc = IVAS_ENC_AGC_UNDEFINED;

#else

    arg->agc = IVAS_DEFAULT_AGC;

#endif

    arg->pca = false;

#ifdef DEBUG_FOA_AGC
@@ -1384,16 +1394,13 @@ static bool parseCmdlIVAS_enc( 
            arg->inputFormatConfig.stereoToMonoDownmix = true;

            i++;

        }

#ifdef DEBUG_AGC_ENCODER_CMD_OPTION

        else if ( strcmp( argv_to_upper, "-AGC" ) == 0 )

        {

            i++;

            if ( i < argc - 4 )

            {

#ifdef AGC_ENABLE_FOR_LBR

                arg->agc = ( atoi( argv[i] ) ) ? IVAS_ENC_AGC_ENABLED : IVAS_ENC_AGC_DISABLED;

#else

                arg->agc = (int16_t) atoi( argv[i] );

#endif

                if ( argv[i] == NULL || atoi( argv[i] ) < 0 || atoi( argv[i] ) > 1 )

                {

                    fprintf( stderr, "Error: wrong adaptive gain control option specified (%d), expected 0 or 1\n\n", (int32_t) atoi( argv[i] ) );
@@ -1409,6 +1416,7 @@ static bool parseCmdlIVAS_enc( 
                return false;

            }

        }

#endif

        else if ( strcmp( argv_to_upper, "-BYPASS" ) == 0 ) // VE: should be renamed to "-pca"

        {

            i++;
@@ -1648,12 +1656,10 @@ static void usage_enc( void ) 
#ifdef DEBUG_SBA

    fprintf( stdout, "-tag                : Tag name for intermediate debug files\n" );

#endif

#ifdef AGC_ENABLE_FOR_LBR

#ifdef DEBUG_AGC_ENCODER_CMD_OPTION

    fprintf( stdout, "-agc op             : SBA Adaptive gain control, op = (0, 1).  \n" );

    fprintf( stdout, "                      By default op is 1 (activated) for bitrates between 24400 and 32000,\n" );

    fprintf( stdout, "                      otherwise it is 0 (deactivated) for all other bitrates\n" );

#else

    fprintf( stdout, "-agc op             : SBA Adaptive gain control, op = (0, 1), by default op is 0 or deactivated\n" );

#endif



    fprintf( stdout, "-bypass mode        : SBA PCA by-pass, mode = (1, 2), 1 = PCA off, 2 = signal adaptive, default is 1\n" );

ci/run_scheduled_sanitizer_test.py

+2 −5
Original line number Diff line number Diff line
@@ -6,8 +6,8 @@ import subprocess 
import pathlib





DURATION = "120"

CFG = "ci_linux.json"

DURATION = "30"

CFG = "ci_linux_ltv.json"

SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3", "VALGRIND"]

EP_FILE = "ep_015.g192"

GENPATT_CMD = f"gen-patt -tailstat -fer -g192 -gamma 0 -rate 0.15 -tol 0.001 -reset -n {int(DURATION) * 50} {EP_FILE}"
@@ -48,9 +48,6 @@ def get_modes(in_format: str) -> list: 
        in_format = "MC_" + in_format + "_b"



    mode_list = [m for m in output.splitlines() if in_format in m]

    if "SBA" in in_format:

        # rate switching not implemented yet

        mode_list = [m for m in mode_list if not "_rs" in m]



    return mode_list

ci/smoke_test.sh

+2 −5
Original line number Diff line number Diff line
@@ -8,8 +8,5 @@ fi 
make clean

make all -j 8



# get all modes except SBA rate switching (which is broken currently)

list=$(./scripts/runIvasCodec.py -l | grep -v "SBA.*rs")

./scripts/runIvasCodec.py -p ./scripts/config/ci_linux.json -m $list -U 1 | tee smoke_test_output.txt



./scripts/runIvasCodec.py -p ./scripts/config/ci_linux.json -m $list -U 1 -D="-fec 15" --decoder_only | tee smoke_test_output_plc.txt
 
./scripts/runIvasCodec.py -p ./scripts/config/ci_linux.json -U 1 | tee smoke_test_output.txt

./scripts/runIvasCodec.py -p ./scripts/config/ci_linux.json -U 1 -D="-fec 15" --decoder_only | tee smoke_test_output_plc.txt

lib_com/cnst.h

+3 −0
Original line number Diff line number Diff line
@@ -132,6 +132,9 @@ enum{ 


#define MAX_V_MULT_MAT                  100             /* maximum array length for the function v_mult_mat() */



#define SBA_AGC_FORCE_ENABLE            1

#define SBA_AGC_FORCE_DISABLE           0

#define SBA_AGC_DEFAULT                 -1



/*----------------------------------------------------------------------------------*

 * Layers