Commit d9342024 authored by Jan Kiene's avatar Jan Kiene
Browse files

remove use of "mld" from ci file and enable comparison to main

parent af8cb9bd

.gitlab-ci.yml

+63 −51
Original line number Diff line number Diff line
@@ -21,12 +21,12 @@ variables: 
  TESTCASE_TIMEOUT_STV_SANITIZERS: 1800

  GIT_CLEAN_FLAGS: -ffdxq

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-mld' to run MLD test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger

    description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." # Not implemented yet, but may be good to have a manual pipeline trigger

    value: 'default'

    options:

      - 'default'

      - 'pytest-mld'

      - 'pytest-mld-long'

      - 'pytest-compare'

      - 'pytest-compare-long'

      - 'pytest-saturation-smoke-test'

      - 'evs-26444'

      - 'sanitizer-stv'
@@ -52,12 +52,12 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing

      variables:

        IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'

      variables:

        IVAS_PIPELINE_NAME: 'Run MLD tool against float ref: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-mld-long'

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-comapre-long'

      variables:

        IVAS_PIPELINE_NAME: 'Run MLD tool against float ref (long test vectors): $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test'

      variables:

        IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH'
@@ -95,10 +95,10 @@ stages: 
    echo "Commit time was $CI_COMMIT_TIMESTAMP"

    date | xargs echo "System time is"



.setup-codec: &setup-codec

.build-reference-and-dut-binaries: &build-reference-and-dut-binaries

  - current_commit_sha=$(git rev-parse HEAD)

  ### build reference binaries

  - git checkout $REFERENCE_BRANCH

  - git checkout $REFERENCE_BRANCH_LOCAL

  - git pull

  - make clean

  - make -j
@@ -111,9 +111,8 @@ stages: 
  - make clean

  - make -j





.mld-test-setup-codec: &mld-test-setup-codec

  - *setup-codec

.build-and-create-reference-outputs: &build-and-create-reference-outputs

  - *build-reference-and-dut-binaries



  ### prepare pytest

  # create short test vectors
@@ -155,9 +154,9 @@ stages: 
.activate-Werror-linux: &activate-Werror-linux

  - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile



.rules-pytest-mld:

.rules-pytest-short:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare"

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main

    - if: $CI_PIPELINE_SOURCE == 'push'
@@ -165,10 +164,10 @@ stages: 
    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never



.rules-pytest-mld-long:

.rules-pytest-long:

  rules:

    - if: $PYTEST_MLD_LONG # Set by scheduled pipeline

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-mld-long"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare-long"

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -212,7 +211,8 @@ stages: 
  needs: ["build-codec-linux-make"]

  timeout: "240 minutes"

  variables:

    MLD_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

    # keep "mld" in artifact name for backwards compatibility reasons

    CSV_ARTIFACT_NAME: "mld--$CI_JOB_NAME-$CI_JOB_ID--sha-$CI_COMMIT_SHORT_SHA.csv"

    MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv"

    PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html"

    SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"
@@ -231,16 +231,16 @@ stages: 
    - if [ $LEVEL_SCALING != "1.0" ];then

    -   *apply-testv-scaling

    - fi

    - *mld-test-setup-codec

    - *build-and-create-reference-outputs



    ### run pytest

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py report-junit.xml $MLD_ARTIFACT_NAME

    - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $MLD_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME images_"$CI_JOB_NAME"/summary_"$MEASURE".csv images_"$CI_JOB_NAME"/summary_"$MEASURE".png --measure $MEASURE; done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME



    - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
@@ -250,7 +250,7 @@ stages: 
    -    unzip artifacts.zip -d previous_artifacts

    # This wildcard thingy relies on only one csv file being present per job

    -    file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv"

    -    python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $MLD_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME

    -    python3 ci/basop-pages/create_report_pages.py $PAGES_HTML_ARTIFACT_NAME $MERGED_CSV_ARTIFACT_NAME $CSV_ARTIFACT_NAME $file_previous $CI_JOB_ID $id_previous $CI_JOB_NAME

    - else

    # create empty file for artifacts to avoid errors

    -    touch $PAGES_HTML_ARTIFACT_NAME
@@ -272,12 +272,12 @@ stages: 
      - report-junit.xml

      - report.html

      - $PAGES_HTML_ARTIFACT_NAME

      - $MLD_ARTIFACT_NAME

      - $CSV_ARTIFACT_NAME

      - $MERGED_CSV_ARTIFACT_NAME

      - images

      - $SUMMARY_HTML_ARTIFACT_NAME

      - $IMAGES_ARTIFACT_NAME

    expose_as: "pytest mld results"

    expose_as: "pytest compare results"

    reports:

      junit:

        - report-junit.xml
@@ -298,7 +298,7 @@ stages: 
    - *print-common-info

    - *update-scripts-repo

    - python3 ci/remove_unsupported_testcases.py $PRM_FILES

    - *setup-codec

    - *build-reference-and-dut-binaries

    - make clean

    - make -j CLANG=$CLANG_NUM

    - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi
@@ -412,37 +412,40 @@ build-codec-linux-instrumented-make: 
# ---------------------------------------------------------------



### jobs that test fx encoder -> flt decoder

ivas-pytest-mld-enc:

ivas-pytest-compare-enc:

  extends:

    - .rules-pytest-mld

    - .rules-pytest-short

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor



ivas-pytest-mld-enc-lev-10:

ivas-pytest-compare-enc-lev-10:

  extends:

    - .rules-pytest-mld

    - .rules-pytest-short

    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=0.3162

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor



ivas-pytest-mld-enc-lev+10:

ivas-pytest-compare-enc-lev+10:

  extends:

    - .rules-pytest-mld

    - .rules-pytest-short

    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=3.162

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor



ivas-pytest-enc-msan:
@@ -470,40 +473,43 @@ ivas-pytest-enc-usan: 
  <<: *ivas-pytest-sanitizers-anchor

  

### jobs that test flt encoder -> fx decoder

ivas-pytest-mld-dec:

ivas-pytest-compare_main-short-dec:

  extends:

    - .rules-pytest-mld

    - .rules-pytest-short

    - .test-job-linux    

  before_script:

    - USE_LTV=0

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=1.0

    - REFERENCE_BRANCH_LOCAL="main"

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-dec-lev-10:

ivas-pytest-compare_main-short-lev-10:

  extends:

    - .rules-pytest-mld

    - .rules-pytest-short

    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=0.3162

    - REFERENCE_BRANCH_LOCAL="main"

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-dec-lev+10:

ivas-pytest-compare_main-short-dec-lev+10:

  extends:

    - .rules-pytest-mld

    - .rules-pytest-short

    - .test-job-linux    

  before_script:

    - USE_LTV=0  

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$SHORT_TEST_SUITE"

    - LEVEL_SCALING=3.162

    - REFERENCE_BRANCH_LOCAL="main"

  <<: *ivas-pytest-anchor



ivas-pytest-dec-msan:

ivas-pytest-dec-msan-short:

  extends:

    - .test-job-linux

  before_script:
@@ -511,7 +517,7 @@ ivas-pytest-dec-msan: 
    - DUT_ENCODER_PATH=./IVAS_cod_ref

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-dec-asan:

ivas-pytest-dec-asan-short:

  extends:

    - .test-job-linux

  before_script:
@@ -519,7 +525,7 @@ ivas-pytest-dec-asan: 
    - DUT_ENCODER_PATH=./IVAS_cod_ref

  <<: *ivas-pytest-sanitizers-anchor



ivas-pytest-dec-usan:

ivas-pytest-dec-usan-short:

  extends:

    - .test-job-linux

  before_script:
@@ -531,70 +537,76 @@ ivas-pytest-dec-usan: 
# Long test jobs 

# ---------------------------------------------------------------



ivas-pytest-mld-long-enc:

ivas-pytest-compare_ref-long-enc:

  extends:

    - .rules-pytest-mld-long

    - .rules-pytest-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=1.0

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-dec:

ivas-pytest-compare_ref-long-dec:

  extends:

    - .rules-pytest-mld-long

    - .rules-pytest-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=1.0

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-enc-lev-10:

ivas-pytest-compare_ref-long-enc-lev-10:

  extends:

    - .rules-pytest-mld-long

    - .rules-pytest-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=0.3162

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-dec-lev-10:

ivas-pytest-compare_ref-long-dec-lev-10:

  extends:

    - .rules-pytest-mld-long

    - .rules-pytest-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=0.3162

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor



ivas-pytest-mld-long-enc-lev+10:

ivas-pytest-compare_ref-long-enc-lev+10:

  extends:

    - .rules-pytest-mld-long

    - .rules-pytest-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=3.162

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor

  

ivas-pytest-mld-long-dec-lev+10:

ivas-pytest-compare_ref-long-dec-lev+10:

  extends:

    - .rules-pytest-mld-long

    - .rules-pytest-long

    - .test-job-linux    

  before_script:

    - USE_LTV=1  

    - DUT_ENCODER_PATH=./IVAS_cod_ref

    - TEST_SUITE="$LONG_TEST_SUITE"

    - LEVEL_SCALING=3.162

    - REFERENCE_BRANCH_LOCAL="$REFERENCE_BRANCH"

  <<: *ivas-pytest-anchor



ivas-smoke-test-saturation: