Commit 26c97862 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into add_evs_be_test_to_ci

parents 3b5e831f d7e0ff79

.gitlab-ci.yml

+90 −17
Original line number Diff line number Diff line 
variables:

  TESTV_DIR: "/usr/local/testv"

  BUILD_OUTPUT: "build_output.txt"

  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"





# prevent running two pipelines on pushes to merge request branches

workflow:

  rules:
@@ -32,6 +32,14 @@ stages: 
  tags:

    - ivas-linux



.build-job-linux:

  extends: .test-job

  stage: build

  timeout: "2 minutes"

  tags:

    - ivas-linux





# template for test jobs on linux that need the TESTV_DIR

.test-job-linux-needs-testv-dir:

  extends: .test-job-linux
@@ -39,41 +47,76 @@ stages: 
    - if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi

    - cp -r scripts/testv/* $TESTV_DIR/



# template for build jobs to include the check for warnings

.build-job-with-check-for-warnings:

  extends: .build-job-linux

  stage: build

  allow_failure:

    exit_codes:

      - 123



# build all components of the project, i.e. codec itself, the unittests, the prerenderer and the standalone version of the TD object renderer

build-all-linux-make:

  extends: .test-job-linux



build-codec-linux-make:

  extends: .build-job-with-check-for-warnings

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  script:

    - make -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-unittests-linux:

  extends: .build-job-with-check-for-warnings

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  script:

    - make unittests -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-prerenderer-linux:

  extends: .build-job-with-check-for-warnings

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  script:

    - make -C scripts/prerenderer -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-td-object-renderer-standalone-linux:

  extends: .build-job-with-check-for-warnings

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  stage: build

  script:

    - bash ci/build_all_linux.sh

    - make -C scripts/td_object_renderer/object_renderer_standalone -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-codec-linux-cmake:

  extends: .test-job-linux

  extends: .build-job-with-check-for-warnings

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  stage: build

  script:

    - mkdir build

    - cd build

    - cmake ..

    - make -j

    - cd ..

    - make -C build -j 2>&1 | tee $BUILD_OUTPUT

    # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<

    - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?



build-codec-instrumented-linux:

  extends: .test-job-linux

  extends: .build-job-linux

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  stage: build

  script:

    - bash ci/build_codec_instrumented_linux.sh



# make sure that the codec builds with msan, asan and usan

build-codec-sanitizers-linux:

  extends: .test-job-linux

  extends: .build-job-linux

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  stage: build

  script:

    - bash ci/build_codec_sanitizers_linux.sh
@@ -82,8 +125,7 @@ build-codec-sanitizers-linux: 
codec-smoke-test:

  extends: .test-job-linux-needs-testv-dir

  # temporarily restrict this job to the only runner which (so far) seems to runit without problems

  tags:

    - test-fhg-linux-runner1

  timeout: "5 minutes"

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  stage: test
@@ -120,6 +162,34 @@ be-2-evs-linux: 


    - cd evs_be_test

    - python3 ../ci/run_evs_be_test.py

# code selftest testvectors with memory-sanitizer binaries

msan-on-merge-request-linux:

  extends: .test-job-linux

  stage: test

  needs: [ "build-codec-sanitizers-linux" ]

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  script:

    - make clean

    - make -j CLANG=1

    - python3 scripts/self_test.py --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi





# code selftest testvectors with address-sanitizer binaries

asan-on-merge-request-linux:

  extends: .test-job-linux

  stage: test

  needs: [ "build-codec-sanitizers-linux" ]

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  script:

    - make clean

    - make -j CLANG=2

    - python3 scripts/self_test.py --create | tee test_output.txt

    - run_errors=$(cat test_output.txt | grep -ic "run errors") || true

    - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi





# compare bit exactness between target and source branch
@@ -131,6 +201,7 @@ self-test-on-merge-request: 
  needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

  timeout: "10 minutes"

  script:

    ### build test binaries, initial clean for paranoia reasons

    - make clean
@@ -194,7 +265,7 @@ self-test-on-merge-request: 
    - exit_code=0

    - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$?

    - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi

    - zero_errors=$(cat report-junit.xml | grep -c 'testsuite errors="0"') || true

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

    # return exit code from selftest if everything went well with the pytest run
@@ -203,10 +274,12 @@ self-test-on-merge-request: 
    exit_codes:

      - 123

  artifacts:

    when: always

    paths:

      - test_output.txt

      - scripts/test/logs

      - scripts/ref/logs

      - report-junit.xml

    reports:

      junit: report-junit.xml

apps/decoder.c

+23 −0
Original line number Diff line number Diff line
@@ -57,6 +57,13 @@ 
#include "debug.h"

#endif

#include "render_config_reader.h"

#ifdef DEBUG_SBA

#include "ivas_cnst.h"

#include "spar_debug.h"

int16_t numTransportChannels = 1;

int16_t numOutChannels = 1;

int16_t pca_ingest_channels = 1;

#endif



#define WMC_TOOL_MAN
@@ -389,6 +396,15 @@ int main( 
        }

    }



#ifdef DEBUG_SBA

    spar_debug_t dbg_params;

    dbg_params.fs = arg.output_Fs;

    dbg_params.n_ch = numOutChannels;

    dbg_params.n_transport = numTransportChannels;

    dbg_params.pca_ingest_channels = pca_ingest_channels;

    ivas_open_decoder_debug_files( &dbg_params );

#endif



    /*-----------------------------------------------------------------*

     * Print information about FEC

     *-----------------------------------------------------------------*/
@@ -570,6 +586,13 @@ cleanup: 
        printf( "\n" );

    }



#ifdef DEBUG_SBA

    dbg_params.fs = arg.output_Fs;

    dbg_params.n_ch = numOutChannels;

    dbg_params.n_transport = numTransportChannels;

    dbg_params.pca_ingest_channels = pca_ingest_channels;

    ivas_close_decoder_debug_files( &dbg_params );

#endif

#ifdef DEBUGGING

    dbgclose();

#endif

apps/encoder.c

+72 −41
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ 
#endif

#ifdef DEBUGGING

#include "debug.h"

#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

#include "ivas_cnst.h"

#include "spar_debug.h"

#endif
@@ -129,7 +129,7 @@ typedef struct 
#ifdef DEBUG_FOA_AGC

    FILE *agcBitstream; /* temporary */

#endif

#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

    const char *dbg_file_tag;

#endif
@@ -218,7 +218,7 @@ int main( 
    FILE *f_forcedModeProfile = NULL;

#endif



#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

    spar_debug_t dbg_params;

#endif

#ifdef WMOPS
@@ -252,18 +252,6 @@ int main( 
        goto cleanup;

    }



#ifdef DEBUG_SPAR_FOA

    dbg_params.agc = (int16_t) arg.agc;

    dbg_params.dtx_on = (int16_t) arg.dtxConfig.enabled;

    dbg_params.file_tag = arg.dbg_file_tag;

    dbg_params.ivas_total_brate = arg.initBitrate;

#ifdef DEBUG_AGC

    ivas_open_agc_debug_files( &dbg_params );

#endif

    ivas_open_debug_files( &dbg_params );

#endif





    /*------------------------------------------------------------------------------------------*

     * Open and initialize IVAS encoder

     *------------------------------------------------------------------------------------------*/
@@ -366,9 +354,10 @@ int main( 
    {

        if ( bandwidth == IVAS_ENC_BANDWIDTH_NB && arg.inputFormat != IVAS_ENC_INPUT_MONO )

        {

            fprintf( stdout, "\nNB coding not supported in IVAS. Switching to WB.\n" );

            fprintf( stdout, "\nNB coding not supported in IVAS. Switching to WB.\n\n" );

        }



        else

        {

            switch ( bandwidth )

            {

                case IVAS_ENC_BANDWIDTH_UNDEFINED:
@@ -391,6 +380,11 @@ int main( 
                    goto cleanup;

            }

        }

    }



    /*------------------------------------------------------------------------------------------*

     * Handle Channel-aware mode

     *------------------------------------------------------------------------------------------*/



    IVAS_ENC_CHANNEL_AWARE_CONFIG caConfig = arg.caConfig;
@@ -410,6 +404,13 @@ int main( 
        fprintf( stdout, "Channel-aware mode:     ON, FEC indicator : %s  FEC offset: %d \n\n", ( caConfig.fec_indicator == IVAS_ENC_FEC_LO ) ? "LO" : "HI", caConfig.fec_offset );

    }



    if ( arg.inputFormat != IVAS_ENC_INPUT_MONO && ( caConfig.channelAwareModeEnabled || arg.ca_config_file ) )

    {

        fprintf( stderr, "Channel-aware mode is not supported in IVAS.\n\n" );

        usage_enc();

        goto cleanup;

    }



    /*------------------------------------------------------------------------------------------*

     * Configure and initialize (allocate memory for static variables) the encoder

     *------------------------------------------------------------------------------------------*/
@@ -468,7 +469,11 @@ int main( 
            goto cleanup;

    }



    IVAS_ENC_PrintConfig( hIvasEnc, caConfig.channelAwareModeEnabled );

    if( ( error = IVAS_ENC_PrintConfig( hIvasEnc, caConfig.channelAwareModeEnabled ) ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "\n IVAS_ENC_PrintConfig failed %s\n\n", IVAS_ENC_GetErrorMessage( error ) );

        goto cleanup;

    }



    /*------------------------------------------------------------------------------------------*

     * Open input metadata files
@@ -553,6 +558,32 @@ int main( 
    uint16_t bitStream[IVAS_MAX_BITS_PER_FRAME];

    uint16_t numBits = 0;



#ifdef DEBUG_SBA

    int16_t numInputChannels;

    if ( ( error = IVAS_ENC_GetNumInputChannels( hIvasEnc, &numInputChannels ) ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "\nGetNumInputChannels failed: %s\n\n", IVAS_ENC_GetErrorMessage( error ) );

        goto cleanup;

    }

    dbg_params.agc = (int16_t) arg.agc;

    dbg_params.dtx_on = (int16_t) arg.dtxConfig.enabled;

    dbg_params.file_tag = arg.dbg_file_tag;

    dbg_params.ivas_total_brate = arg.initBitrate;

    dbg_params.fs = arg.inputFs;

    dbg_params.n_ch = numInputChannels;

    int16_t numTransportChannels;

    if ( ( error = getTransportChannel( hIvasEnc, &numTransportChannels ) ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "\nGetNumInputChannels failed: %s\n\n", IVAS_ENC_GetErrorMessage( error ) );

        goto cleanup;

    }

    dbg_params.n_transport = numTransportChannels;

#ifdef DEBUG_AGC

    ivas_open_agc_debug_files( &dbg_params );

#endif

    ivas_open_debug_files( &dbg_params );

#endif



    if ( !arg.quietModeEnabled )

    {

        fprintf( stdout, "\n------ Running the encoder ------\n\n" );
@@ -815,7 +846,7 @@ cleanup: 
    dbgclose();

#endif



#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

    ivas_close_remove_debug_files();

#ifdef DEBUG_AGC

    ivas_close_agc_debug_files( &io_params );
@@ -862,7 +893,7 @@ static void initArgStruct( EncArguments *arg ) 
#ifdef DEBUG_FOA_AGC

    arg->agcBitstream = NULL;

#endif

#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

    arg->dbg_file_tag = NULL;

#endif
@@ -1417,7 +1448,7 @@ static bool parseCmdlIVAS_enc( 
                return false;

            }

        }

#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

        /*-----------------------------------------------------------------*

         * IVAS SPAR debug files tag

         *-----------------------------------------------------------------*/
@@ -1620,7 +1651,7 @@ static void usage_enc( void ) 
    fprintf( stdout, "                      The encoder produces TS26.445 Annex.2.6 Mime Storage Format, (not RFC4867 Mime Format).\n" );

    fprintf( stdout, "                      default output bitstream file format is G.192\n" );



#ifdef DEBUG_SPAR_FOA

#ifdef DEBUG_SBA

    fprintf( stdout, "-tag                : Tag name for intermediate debug files\n" );

#endif

    fprintf( stdout, "-agc op             : SBA Adaptive gain control, op = (0, 1), by default op is 0 or deactivated\n" );

ci/check_for_warnings.py

0 → 100755
+33 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3

import argparse

import sys





SEARCH_FOR = "warning:"

RETURN_FOUND = 123





def main(log_file):

    with open(log_file) as f:

        lines_with_warnings = [l for l in f.readlines() if SEARCH_FOR in l]



    n_warnings = len(lines_with_warnings)

    if n_warnings > 0:

        print(f"========== Found {n_warnings} warnings: =========")

        for l in lines_with_warnings:

            print(l)

        return RETURN_FOUND

    else:

        return 0





if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument(

        "compilation-output",

        type=str,

        help="text output of compilation process to analyze",

    )

    args = vars(parser.parse_args())

    ret_code = main(args[ "compilation-output" ])

    sys.exit(ret_code)

lib_com/options.h

+8 −4
Original line number Diff line number Diff line
@@ -114,8 +114,10 @@ 
/*#define TDREND_HRTF_TABLE_METHODS*/           /* Enable HRTF lookup from tables, for testing & evaluation. Supply file in table format to use. Note that a suitable HR filter lookup method should be written if the filters sample point grids are not in the formats. */

/*#define TDREND_STANDALONE*/                   /* Used when renderer is built in standalone form, without IVAS encoding/decoding (see scripts/object_renderer_standalone). This is just here to ensure this is cleaned out by prepare_instrumentation.sh */



/*#define DEBUG_SPAR_FOA*/                      /* debug SPAR in-out */

#ifdef DEBUG_SPAR_FOA

/*#define DEBUG_SBA*/                           /* debug DIRAC/SPAR in-out */

#ifdef DEBUG_SBA

/*#define DEBUG_SBA_AUDIO_DUMP*/                /* SBA intermediate audio wav file dumping */

/*#define DEBUG_SBA_MD_DUMP*/                   /* SBA metadata and variable file dumping */

/*#define DEBUG_SPAR_MD_TARGET_TUNING*/         /* SPAR MD target bitrate tuning debug code */

/*#define DEBUG_SPAR_BYPASS_EVS_CODEC*/         /* bypass EVS coding in float precision, emulating EVS encoder/decoder delay */

/*#define DEBUG_SPAR_WRITE_OUT_COV*/            /* write covariance per frame into a text file for verification */
@@ -146,14 +148,16 @@ 
/*#define FIX_IVAS_185_MDCT_ST_PLC_FADEOUT*/            /* IVAS-185 fix bug in TCX-PLC fadeout for MDCT-Stereo and improve fadeout by fading to background noise instead of white noise */

/*#define FIX_I1_113*/                                  /* under review : MCT bit distribution optimization for SBA high bitrates*/





#define SIMPLIFY_SBA_RENDERING_LOGIC                    /* SBA rendering maintenance related to ticket #45 */

#define SBA_CLEANING                                    /* SBA maintenance related to ticket #45 */ 



#define REFACTOR_REVERB_INIT_ADD_CFG_CHECKS             /* Simplify and improve reverb initialization + add range checks on config parameter values */



#define DIRAC_DRCT_GAIN_TUNING



#define FIX_34                                          /* Nokia: Fix bug in MASA format EXT output spherical indexing */



#define REFACTOR_REVERB_INIT_ADD_CFG_CHECKS             /* Simplify and improve reverb initialization + add range checks on config parameter values */ 

#define FIX_I54_LS_CONVERSION                           /* FhG: fix incorrect downmix matrix for 5_1_4 to 5_1_2 and upmix matrix for 7_1 to 7_1_4 */



/* ################## End DEVELOPMENT switches ######################### */

/* clang-format on */