Commit 1b33b073 authored by norvell's avatar norvell
Browse files

Update .gitlab-ci.yml

parent e005a5d0

.gitlab-ci.yml

+294 −23
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ variables: 
  SCRIPTS_DIR: "/usr/local/scripts"

  EXIT_CODE_NON_BE: 123

  EXIT_CODE_FAIL: 1

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm"

  LONG_TEST_SUITE: "tests/codec_be_on_mr_nonselection tests/renderer --param_file scripts/config/self_test_ltv.prm --use_ltv"

  SHORT_TEST_SUITE: "tests/codec_be_on_mr_nonselection"

  TEST_SUITE: ""

  DUT_ENCODER_PATH: "./IVAS_cod"
@@ -26,6 +26,7 @@ variables: 
  CI_REGRESSION_THRESH_SSNR: "-1"

  CI_REGRESSION_THRESH_ODG: "-0.05"

  GIT_CLEAN_FLAGS: -ffdxq

  INSTR_DIR: "scripts/c-code_instrument"

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."

    value: 'default'
@@ -37,6 +38,7 @@ variables: 
      - 'evs-26444'

      - 'sanitizer-stv'

      - 'pytest-renderer'

      - 'complexity'





default:
@@ -51,7 +53,7 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      variables:

        IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'

    ### disabled for now because pipeline setup is redundant with MR pipeline with current workflow

    ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow

    # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    #   variables:

    #     IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH'
@@ -76,6 +78,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity'

      variables:

        IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
@@ -113,6 +118,7 @@ stages: 
  - mv ./IVAS_rend ./IVAS_rend_ref



  ### build test binaries

  - git restore .

  - git checkout $current_commit_sha

  - make clean

  - make -j
@@ -125,16 +131,7 @@ stages: 
  - python3 tests/create_short_testvectors.py

  # create references

  - exit_code=0

  # - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

  - python3 -m pytest $TEST_SUITE -k "not test_sba_plc_system" -v --update_ref 1 -m create_ref --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

  - python3 -m pytest $TEST_SUITE -k "test_sba_plc_system" -v --update_ref 1 -m create_ref --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

  

  # - python3 -m pytest $TEST_SUITE -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

  - python3 -m pytest $TEST_SUITE -k "not test_sba_plc_system" -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

  - python3 -m pytest $TEST_SUITE -k "test_sba_plc_system" -v --update_ref 1 -m create_ref_part2 -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?



  # Catch exit code to prevent halt in case this step produces zero tests



  - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?



.update-scripts-repo: &update-scripts-repo

  - cd $SCRIPTS_DIR
@@ -265,12 +262,8 @@ stages: 


    ### run pytest

    - exit_code=0

    # - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut -k "not test_sba_plc_system" --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut -k "test_sba_plc_system" --html=report.html --self-contained-html --junit-xml=report-junit.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true





    - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME

    - mkdir $IMAGES_ARTIFACT_NAME
@@ -442,10 +435,7 @@ stages: 
    - make -j CLANG=$CLANG_NUM

    - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi

    - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS

    # - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

    - python3 -m pytest $SHORT_TEST_SUITE -k "not test_sba_plc_system" -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

    - python3 -m pytest $SHORT_TEST_SUITE -k "test_sba_plc_system" -v --tb=no --update_ref 1 -m create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

    

    - python3 -m pytest $SHORT_TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH

  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always
@@ -544,10 +534,12 @@ build-codec-linux-instrumented-make: 
      when: never

  extends:

    - .build-job-linux

  timeout: "7 minutes"

  script:

    - *print-common-info

    - sed -i".bak" "s/\/\*#define WMOPS 1\*\//#define WMOPS 1/g" lib_com/count.h

    - make -j

    - *update-scripts-repo

    - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP

    - make -j -C $INSTR_DIR



# ---------------------------------------------------------------

# Short test jobs that run in merge request pipelines
@@ -914,6 +906,285 @@ ivas-pytest-renderer: 
  <<: *ivas-pytest-anchor





# ---------------------------------------------------------------

# Complexity measurement jobs

# ---------------------------------------------------------------



.complexity-measurements-setup:

  &complexity-measurements-setup # create necessary environment

  ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work

  - mkdir -p wmops/logs



  - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID)

  - echo $job_id

  - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip

  - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - ls

  - public_dir="$CI_JOB_NAME-public"

  # if is needed to catch case when no artifact is there (first run), similarly as above

  - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/;  fi

  - ls wmops

  - rm artifacts.zip

  - rm -rf $public_dir



  ### 2. part: setup specific for BASOP repo

  # hack for using the reference encoder -> need to build manually to make script use ref enc and BASOP dec

  - mkdir COMPLEXITY

  - cp IVAS_cod_ref COMPLEXITY/IVAS_cod

  # build branch code aain with instrumentation

  - make clean

  - bash scripts/prepare_instrumentation.sh -p BASOP -m MEM_ONLY

  - make -j -C $INSTR_DIR

  - cp $INSTR_DIR/IVAS_dec COMPLEXITY/IVAS_dec



.complexity-measurements-prepare-artifacts:

  &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory

  - public_dir="$CI_JOB_NAME-public"

  - mkdir $public_dir

  - mv -f wmops/log_*_all.txt wmops/*.js ${public_dir}/

  # move logfiles for links

  - mkdir $public_dir/logs

  # first move logs

  - log_files=$(cat $public_dir/graphs*.js | grep logFile | sed "s/.*\(wmops_newsletter_.*\.csv\).*/\1/g")

  - echo $log_files

  - ls wmops/logs

  - for f in $log_files; do [ -f wmops/logs/$f ] && mv wmops/logs/$f $public_dir/logs/$f; done

  # copy index page blueprint

  - cp ci/complexity_measurements/index_complexity.html ${public_dir}/index.html

  # patch the format in the title

  - sed -i "s/IVAS FORMAT/IVAS $in_format to $out_format/g" ${public_dir}/index.html

  # do separately here to avoid overwrite complaints by mv

  - mv -f ci/complexity_measurements/style.css ${public_dir}/

  - ls $public_dir



.complexity-template:

  extends:

    - .test-job-linux

  timeout: 3 hours 30 minutes

  stage: test

  variables:

    ret_val: 0

  before_script:

    - *print-common-info

    - *update-scripts-repo

    - *update-ltv-repo

    - *build-reference-and-dut-binaries

    - *complexity-measurements-setup

    - which coan

  artifacts:

    name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"

    when: always

    expire_in: 1 week

    paths:

      - $CI_JOB_NAME-public



complexity-stereo-in-stereo-out:

  extends:

    - .complexity-template

  rules:

    - if: $MEASURE_COMPLEXITY_LINUX

  script:

    - in_format=stereo

    - out_format=stereo

    - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only || ret_val=$?

    - *complexity-measurements-prepare-artifacts

    - exit $ret_val



# complexity-stereo-in-7_1_4-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#   script:

#     - in_format=stereo

#     - out_format=7_1_4

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-ism-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 1 hours

#   script:

#     - in_format=ISM

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-ism-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 1 hours

#   script:

#     - in_format=ISM

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-hoa3-in-hoa3-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=HOA3

#     - out_format=HOA3

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-hoa3-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=HOA3

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-osba-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OSBA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-osba-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OSBA

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-mc-in-7_1_4-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MC

#     - out_format=7_1_4

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-mc-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MC

#     - out_format=BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-masa-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MASA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-masa-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=MASA

#     - out_format="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-omasa-in-ext-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OMASA

#     - out_format=EXT

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-omasa-in-binaural-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=OMASA

#     - out_format="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val



# complexity-StereoDmxEVS-stereo-in-mono-out:

#   extends:

#     - .complexity-template

#   rules:

#     - if: $MEASURE_COMPLEXITY_LINUX

#       # when: delayed:

#       # start_in: 2 hours

#   script:

#     - in_format=StereoDmxEVS

#     - out_format=mono

#     - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" || ret_val=$?

#     - *complexity-measurements-prepare-artifacts

#     - exit $ret_val





# job that sets up gitlab pages website

pages:

  stage: deploy