Commit 90a1403a authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into ci/interop-test 

+ some changes mixed within
parents ec6812e7 e6df9804

.gitlab-ci.yml

+20 −33
Original line number Diff line number Diff line
@@ -6,13 +6,15 @@ variables: 
  SCRIPTS_DIR: "/usr/local/scripts"

  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/test_param_file_ltv.py tests/renderer"

  LONG_TEST_SUITE: "tests/test_param_file_ltv.py tests/renderer"

  SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection"

  USE_REF_ENC: "1"

  TEST_SUITE: ""

  DUT_ENCODER_PATH: "./IVAS_cod"

  DUT_DECODER_PATH: "./IVAS_dec"

  LEVEL_SCALING: "1.0"

  IVAS_PIPELINE_NAME: ''

  BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch"

  PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm"

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger

    value: 'default'
@@ -85,6 +87,7 @@ stages: 
  - make -j

  - mv ./IVAS_cod ./IVAS_cod_ref

  - mv ./IVAS_dec ./IVAS_dec_ref

  - mv ./IVAS_rend ./IVAS_rend_ref



  ### build test binaries

  - git checkout $current_commit_sha
@@ -99,13 +102,16 @@ stages: 
  # create short test vectors

  - python3 tests/create_short_testvectors.py

  # create references

  - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref -n auto

  - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto

  - exit_code=0

  - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref --create_ref -n auto || exit_code=$? 

  - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto || exit_code=$? # Catch exit code to prevent halt in case this step produces zero tests





.update-scripts-repo: &update-scripts-repo

  - cd $SCRIPTS_DIR

  - git remote set-branches --add origin $BASOP_CI_BRANCH_PC_REPO

  - git remote prune origin

  - git branch

  - git fetch

  - git checkout $BASOP_CI_BRANCH_PC_REPO

  - git pull
@@ -115,19 +121,10 @@ stages: 
  - cp -r $SCRIPTS_DIR/tests .

  - cp $SCRIPTS_DIR/pytest.ini .





.pytest-check-results: &pytest-check-results

  - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

  - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi





# TODO: this needs to be updated in case the reference is updated

.remove-unsupported-testcases: &remove-unsupported-testcases

  - sed -i '1701,1707d' scripts/config/self_test.prm

  - sed -i '1659,1661d' scripts/config/self_test.prm

  - sed -i '1635,1637d' scripts/config/self_test.prm

  - sed -i '1603,1605d' scripts/config/self_test.prm



.apply-testv-scaling: &apply-testv-scaling

  - echo "Applying level scaling in scripts/testv using scale=$LEVEL_SCALING"

  - tests/scale_pcm.py ./scripts/testv/ $LEVEL_SCALING
@@ -188,7 +185,7 @@ stages: 
.ivas-pytest-interop-anchor: &ivas-pytest-interop-anchor

  - *print-common-info

  - *update-scripts-repo

  - *remove-unsupported-testcases

  - python3 ci/remove_unsupported_testcases.py $PRM_FILES



  ### run pytest

  # create short test vectors
@@ -219,7 +216,7 @@ stages: 
    -    *update-ltv-repo

    -    *copy-ltv-files-to-testv-dir

    - fi

    - *remove-unsupported-testcases

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - if [ $LEVEL_SCALING != "1.0" ];then

    -   *apply-testv-scaling

    - fi
@@ -229,11 +226,7 @@ stages: 
    - exit_code=0

    # timeout of 15 min per individual testcase - hopefully too much, but better be safe for now

    - testcase_timeout=900

    - if [ $USE_REF_ENC -eq 1 ]; then

    -    python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path ./IVAS_cod_ref -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - else

    -    python3 -m pytest $TEST_SUITE -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - fi

    - python3 -m pytest $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - python3 scripts/parse_mld_xml.py report-junit.xml mld.csv
@@ -272,7 +265,7 @@ stages: 
  script:

    - *print-common-info

    - *update-scripts-repo

    - *remove-unsupported-testcases

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - *setup-codec

    - make clean

    - make -j CLANG=$CLANG_NUM
@@ -335,7 +328,6 @@ ivas-pytest-mld-enc-dec: 
    - .test-job-linux

  before_script:

    - USE_LTV=0

    - USE_REF_ENC=0

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-mld-anchor
@@ -346,7 +338,7 @@ ivas-pytest-mld-dec: 
    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - USE_REF_ENC=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-mld-anchor
@@ -357,7 +349,6 @@ ivas-pytest-mld-enc-dec-lev-10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - USE_REF_ENC=0

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-mld-anchor
@@ -368,7 +359,7 @@ ivas-pytest-mld-dec-lev-10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - USE_REF_ENC=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-mld-anchor
@@ -379,7 +370,6 @@ ivas-pytest-mld-enc-dec-lev+10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - USE_REF_ENC=0

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-mld-anchor
@@ -390,7 +380,7 @@ ivas-pytest-mld-dec-lev+10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - USE_REF_ENC=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-mld-anchor
@@ -440,7 +430,6 @@ ivas-pytest-mld-long-enc-dec: 
    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - USE_REF_ENC=0

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-mld-anchor
@@ -451,7 +440,7 @@ ivas-pytest-mld-long-dec: 
    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - USE_REF_ENC=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-mld-anchor
@@ -462,7 +451,6 @@ ivas-pytest-mld-long-enc-dec-lev-10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - USE_REF_ENC=0

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-mld-anchor
@@ -473,7 +461,7 @@ ivas-pytest-mld-long-dec-lev-10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - USE_REF_ENC=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-mld-anchor
@@ -484,7 +472,6 @@ ivas-pytest-mld-long-enc-dec-lev+10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - USE_REF_ENC=0

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-mld-anchor
@@ -495,7 +482,7 @@ ivas-pytest-mld-long-dec-lev+10: 
    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - USE_REF_ENC=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-mld-anchor

apps/encoder.c

+2 −3
Original line number Diff line number Diff line
@@ -414,13 +414,12 @@ int main( 
    /*------------------------------------------------------------------------------------------*

     * Open input audio file

     *------------------------------------------------------------------------------------------*/

    short isInputRawFile;

    if ( AudioFileReader_open( &audioReader, arg.inputWavFilename, &isInputRawFile ) != IVAS_ERR_OK )



    if ( AudioFileReader_open( &audioReader, arg.inputWavFilename ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "\nCan't open %s\n\n", arg.inputWavFilename );

        goto cleanup;

    }

    set_raw_file_flag( hIvasEnc, isInputRawFile );



    /* Validate input sampling rate */

    int32_t inFileSampleRate = 0;

apps/renderer.c

+2 −2
Original line number Diff line number Diff line
@@ -780,8 +780,8 @@ int main( 
            exit( -1 );

        }

    }

    short ignore;

    if ( AudioFileReader_open( &audioReader, audioFilePath, &ignore ) != IVAS_ERR_OK )



    if ( AudioFileReader_open( &audioReader, audioFilePath ) != IVAS_ERR_OK )

    {

        fprintf( stderr, "Error opening file: %s\n", audioFilePath );

        exit( -1 );

lib_com/bitstream_fx.c

+2 −2
Original line number Diff line number Diff line
@@ -665,7 +665,7 @@ void write_indices_fx( 
 *-------------------------------------------------------------------*/



void write_indices_buf_fx(

    Encoder_State_fx *st_fx,  /* i/o: encoder state structure */

    Encoder_State *st_fx,  /* i/o: encoder state structure */

    BSTR_ENC_HANDLE hBstr, /* i/o: encoder state structure */

    UWord16 *out_buf,      /* i  : output bitstream buf   */

    UWord8 *pFrame,        /* i: byte array with bit packet and byte aligned coded speech data */
@@ -726,7 +726,7 @@ void write_indices_buf_fx( 
    {

        /* Create and write ToC header */

        /*  qbit always  set to  1 on encoder side  for AMRWBIO ,  no qbit in use for EVS, but set to 0(bad)  */

        header = (UWord8) ( st_fx->Opt_AMR_WB_fx << 5 | st_fx->Opt_AMR_WB_fx << 4 | rate2EVSmode( hBstr->nb_bits_tot_fx * 50, &isAmrWb ) );

        header = (UWord8) ( st_fx->Opt_AMR_WB << 5 | st_fx->Opt_AMR_WB << 4 | rate2EVSmode( hBstr->nb_bits_tot_fx * 50, &isAmrWb ) );

        // fwrite(&header, sizeof(UWord8), 1, file);

        memcpy( out_buf, &header, sizeof( UWord8 ) );

        *num_bits += sizeof( UWord8 );

lib_com/cldfb.c

+79 −0
Original line number Diff line number Diff line
@@ -2295,6 +2295,45 @@ static void cldfb_init_proto_and_twiddles( 
 *

 * Save the memory of filter; to be restored with cldfb_restore_memory_ivas()

 *--------------------------------------------------------------------*/

#ifdef IVAS_FLOAT_FIXED

ivas_error cldfb_save_memory_ivas_fx(

    HANDLE_CLDFB_FILTER_BANK hs /* i/o: filter bank handle */

)

{

    UWord16 offset = sub(hs->p_filter_length , hs->no_channels);



    IF ( hs->memory_flt != NULL || NE_16(hs->memory_length , 0) )

    {

        /* memory already stored; Free memory first */

        return IVAS_ERR_OK;

    }



    IF ( EQ_16(hs->type , CLDFB_ANALYSIS) )

    {

        hs->memory_length = offset;

        move16();

    }

    ELSE

    {

        hs->memory_length = hs->p_filter_length;

        move16();

    }

#if 1/*Floating point memory allocation: To be removed later*/

    IF ( ( hs->memory_flt = (float *) malloc( hs->memory_length * sizeof( float ) ) ) == NULL )

    {

        return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for CLDFB\n" );

    }

#endif

    IF ( ( hs->memory32 = (Word32 *) malloc( hs->memory_length * sizeof(Word32) ) ) == NULL )

    {

        return IVAS_ERROR( IVAS_ERR_FAILED_ALLOC, "Can not allocate memory for CLDFB\n" );

    }

    /* save the memory */

    Copy32( hs->cldfb_state_fx, hs->memory32, hs->memory_length );



    return IVAS_ERR_OK;

}

#endif // IVAS_FLOAT_FIXED



ivas_error cldfb_save_memory_ivas(

    HANDLE_CLDFB_FILTER_BANK hs /* i/o: filter bank handle */
@@ -2334,6 +2373,46 @@ ivas_error cldfb_save_memory_ivas( 
 *

 * Restores the memory of filter; memory to be save by cldfb_save_memory_ivas()

 *--------------------------------------------------------------------*/

#ifdef IVAS_FLOAT_FIXED

void cldfb_restore_memory_ivas_fx(

    HANDLE_CLDFB_FILTER_BANK hs /* i/o: filter bank handle */

)

{

    UWord16 offset = sub(hs->p_filter_length , hs->no_channels);

    UWord16 size;

    IF ( hs->memory_flt == NULL || EQ_16(hs->memory_length , 0) )

    {

        /* memory not allocated */

        return;

    }



    IF ( EQ_16(hs->type , CLDFB_ANALYSIS) )

    {

        size = offset;

        move16();

    }

    ELSE

    {

        size = hs->p_filter_length;

        move16();

    }



    /* read the memory */

    Copy32( hs->memory32, hs->cldfb_state_fx, hs->memory_length );



    /* adjust sample rate if it was changed in the meanwhile */

    IF ( NE_16(hs->memory_length , size) )

    {

        L_lerp_fx_q11( hs->cldfb_state_fx, hs->cldfb_state_fx, size, hs->memory_length);

    }

    hs->cldfb_state_length = size;

    hs->memory_length = 0;

    free( hs->memory32 );

    hs->memory32 = NULL;



    return;

}

#endif // IVAS_FLOAT_FIXED



void cldfb_restore_memory_ivas(

    HANDLE_CLDFB_FILTER_BANK hs /* i/o: filter bank handle */