Commit c491e12b authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into ci/add-logging-for-encoder

parents fab8bac7 96c285ba

.gitlab-ci.yml

+95 −80
Original line number Diff line number Diff line
@@ -18,17 +18,19 @@ variables: 
  PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm"

  TESTCASE_TIMEOUT_STV: 900

  TESTCASE_TIMEOUT_LTV: 2400

  TESTCASE_TIMEOUT_STV_SANITIZERS: 1800

  GIT_CLEAN_FLAGS: -ffdxq

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger

    value: 'default'

    options:

      - 'default'

      - 'pytest-mld'

      - 'pytest-mld-interop'

      - 'pytest-mld-long'

      - 'pytest-saturation-smoke-test'

      - 'evs-26444'

      - 'sanitizer-stv'

      - 'pytest-renderer'





default:
@@ -53,9 +55,6 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld'

      variables:

        IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-interop'

      variables:

        IVAS_PIPELINE_NAME: 'Run MLD tool agains float ref - interop: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long'

      variables:

        IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH'
@@ -68,6 +67,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer-stv'

      variables:

        IVAS_PIPELINE_NAME: 'Short testvectors sanitizers'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
@@ -172,15 +174,6 @@ stages: 
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      when: never





.rules-pytest-mld-interop:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-interop"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      when: never



.rules-pytest-saturation-smoke-test:

  rules:

    - if: $PYTEST_SMOKE_TEST # Set by scheduled pipeline
@@ -222,6 +215,8 @@ stages: 
    MLD_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

    MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv"

    PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html"

    SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"

    IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"

  script:

    - *print-common-info

    - *update-scripts-repo
@@ -240,10 +235,13 @@ stages: 


    ### run pytest

    - exit_code=0

    - python3 -m pytest $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - python3 scripts/parse_mld_xml.py report-junit.xml $MLD_ARTIFACT_NAME

    - python3 scripts/parse_xml_report.py report-junit.xml $MLD_ARTIFACT_NAME

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $MLD_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME



    - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

    -    id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)
@@ -255,7 +253,7 @@ stages: 
    -    python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $MLD_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME

    - else

    # create empty file for artifacts to avoid errors

    -    touch $CI_JOB_NAME-index.html

    -    touch $PAGES_HTML_ARTIFACT_NAME

    -    touch $MERGED_CSV_ARTIFACT_NAME

    - fi
@@ -276,6 +274,9 @@ stages: 
      - $PAGES_HTML_ARTIFACT_NAME

      - $MLD_ARTIFACT_NAME

      - $MERGED_CSV_ARTIFACT_NAME

      - images

      - $SUMMARY_HTML_ARTIFACT_NAME

      - $IMAGES_ARTIFACT_NAME

    expose_as: "pytest mld results"

    reports:

      junit:
@@ -301,8 +302,8 @@ stages: 
    - make clean

    - make -j CLANG=$CLANG_NUM

    - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi

    - testcase_timeout=$TESTCASE_TIMEOUT_STV

    - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec

    - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always
@@ -410,98 +411,94 @@ build-codec-linux-instrumented-make: 
# Short test jobs 

# ---------------------------------------------------------------



#ivas-pytest-mld-enc-dec:

#  extends:

#    - .rules-pytest-mld

#    - .test-job-linux

#  before_script:

#    - USE_LTV=0

#    - TEST_SUITE="$SHORT_TEST_SUITE"

#    - LEVEL_SCALING=1.0

#  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-dec:

### jobs that test fx encoder -> flt decoder

ivas-pytest-mld-enc:

  extends:

    - .rules-pytest-mld

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-anchor



#ivas-pytest-mld-enc-dec-lev-10:

#  extends:

#    - .rules-pytest-mld

#    - .test-job-linux    

#  before_script:

#    - USE_LTV=0  

#    - TEST_SUITE="$SHORT_TEST_SUITE"

#    - LEVEL_SCALING=0.3162

#  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-dec-lev-10:

ivas-pytest-mld-enc-lev-10:

  extends:

    - .rules-pytest-mld

    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-anchor



#ivas-pytest-mld-enc-dec-lev+10:

#  extends:

#    - .rules-pytest-mld

#    - .test-job-linux    

#  before_script:

#    - USE_LTV=0  

#    - TEST_SUITE="$SHORT_TEST_SUITE"

#    - LEVEL_SCALING=3.162

#  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-dec-lev+10:

ivas-pytest-mld-enc-lev+10:

  extends:

    - .rules-pytest-mld

    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-anchor



ivas-pytest-mld-long-enc-fx-dec-flt-interop:

ivas-pytest-enc-msan:

  extends:

    - .rules-pytest-mld-interop

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - CLANG_NUM=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-enc-asan:

  extends:

    - .test-job-linux

  before_script:

    - CLANG_NUM=2

    - DUT_DECODER_PATH=./IVAS_dec_ref

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-enc-usan:

  extends:

    - .test-job-linux

  before_script:

    - CLANG_NUM=3

    - DUT_DECODER_PATH=./IVAS_dec_ref

  <<: *ivas-pytest-sanitizers-anchor

  

### jobs that test flt encoder -> fx decoder

ivas-pytest-mld-dec:

  extends:

    - .rules-pytest-mld

    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-enc-fx-dec-flt-lev-10-interop:

ivas-pytest-mld-dec-lev-10:

  extends:

    - .rules-pytest-mld-interop

    - .rules-pytest-mld

    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-enc-fx-dec-flt-lev+10-interop:

ivas-pytest-mld-dec-lev+10:

  extends:

    - .rules-pytest-mld-interop

    - .rules-pytest-mld

    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-anchor
@@ -511,6 +508,7 @@ ivas-pytest-dec-msan: 
    - .test-job-linux

  before_script:

    - CLANG_NUM=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-dec-asan:
@@ -518,6 +516,7 @@ ivas-pytest-dec-asan: 
    - .test-job-linux

  before_script:

    - CLANG_NUM=2

    - DUT_ENCODER_PATH=./IVAS_cod_ref

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-dec-usan:
@@ -525,18 +524,20 @@ ivas-pytest-dec-usan: 
    - .test-job-linux

  before_script:

    - CLANG_NUM=3

    - DUT_ENCODER_PATH=./IVAS_cod_ref

  <<: *ivas-pytest-sanitizers-anchor



# ---------------------------------------------------------------

# Long test jobs 

# ---------------------------------------------------------------



ivas-pytest-mld-long-enc-dec:

ivas-pytest-mld-long-enc:

  extends:

    - .rules-pytest-mld-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-anchor
@@ -552,12 +553,13 @@ ivas-pytest-mld-long-dec: 
    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-enc-dec-lev-10:

ivas-pytest-mld-long-enc-lev-10:

  extends:

    - .rules-pytest-mld-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-anchor
@@ -573,12 +575,13 @@ ivas-pytest-mld-long-dec-lev-10: 
    - LEVEL_SCALING=0.3162

  <<: *ivas-pytest-anchor



ivas-pytest-mld-long-enc-dec-lev+10:

ivas-pytest-mld-long-enc-lev+10:

  extends:

    - .rules-pytest-mld-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=3.162

  <<: *ivas-pytest-anchor
@@ -678,6 +681,18 @@ be-2-evs-26444: 
      junit:

        - report-junit.xml    



ivas-pytest-renderer:

  extends:

    - .test-job-linux

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-renderer"

  before_script:

    - USE_LTV=0

    - TEST_SUITE="tests/renderer"

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-anchor





# job that sets up gitlab pages website

pages:

  stage: deploy

Makefile

+2 −0
Original line number Diff line number Diff line
@@ -60,6 +60,8 @@ CFLAGS += -std=c99 -pedantic -Wcast-qual -Wall -W -Wextra -Wno-long-long \ 
            -Wno-implicit-fallthrough -ffp-contract=off

# to be uncommented in CI

# CFLAGS += -Werror

CFLAGS += -Winit-self

CFLAGS += -Wunused-but-set-variable



# libs to link

LDLIBS   += -lm

Workspace_msvc/lib_com.vcxproj

+1 −2
Original line number Diff line number Diff line
@@ -399,8 +399,7 @@ 
    <ClInclude Include="..\lib_com\move.h" />

    <ClInclude Include="..\lib_com\options.h" />

    <ClInclude Include="..\lib_com\prot.h" />

    <ClInclude Include="..\lib_com\prot_fx2.h" />

    <ClInclude Include="..\lib_com\prot_fx1.h" />

    <ClInclude Include="..\lib_com\prot_fx.h" />

    <ClInclude Include="..\lib_com\rom_basop_util.h" />

    <ClInclude Include="..\lib_com\rom_com.h" />

    <ClInclude Include="..\lib_com\rom_com_fx.h" />

Workspace_msvc/lib_com.vcxproj.filters

+1 −2
Original line number Diff line number Diff line
@@ -593,11 +593,10 @@ 
    <ClInclude Include="..\lib_com\rom_basop_util.h">

      <Filter>common_h</Filter>

    </ClInclude>

    <ClInclude Include="..\lib_com\prot_fx2.h" />

    <ClInclude Include="..\lib_com\prot_fx1.h" />

    <ClInclude Include="..\lib_com\rom_com_fx.h">

      <Filter>common_h</Filter>

    </ClInclude>

    <ClInclude Include="..\lib_com\prot_fx.h" />

  </ItemGroup>

  <ItemGroup>

    <Filter Include="common_evs_c">

Workspace_msvc/renderer.vcxproj

+1 −1
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ 
    </Midl>

    <ClCompile>

      <Optimization>Disabled</Optimization>

      <AdditionalIncludeDirectories>..\lib_com;..\lib_debug;..\lib_util;..\lib_rend;.%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

      <AdditionalIncludeDirectories>..\lib_com;..\lib_debug;..\lib_util;..\lib_rend;..\lib_dec;..\lib_enc;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;$(Macros);%(PreprocessorDefinitions)</PreprocessorDefinitions>

      <ExceptionHandling />

      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>