Commit f5b202cb authored by Archit Tamarapu's avatar Archit Tamarapu
Browse files

[revertme] profile renderer pytest testcase durations

parent a76ddac0

main-basop.yml

+301 −309
Original line number Diff line number Diff line
@@ -15,77 +15,76 @@ stages: 
  - postvalidate

  - deploy





workflow:

  name: '$IVAS_PIPELINE_NAME'

  name: "$IVAS_PIPELINE_NAME"

  rules:

    # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

      variables:

        IVAS_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'

        IVAS_PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"

    ### disabled for now because pipeline cd is redundant with MR pipeline with current workflow

    # - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main

    #   variables:

    #     IVAS_PIPELINE_NAME: 'Push pipeline: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing

      variables:

        IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Web run pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs'

      variables:

        IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Aggregate long term logs: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs'

      variables:

        IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Backup long term logs: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Run comparison tools against float ref: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-enc-dmx'

      variables:

        IVAS_PIPELINE_NAME: 'Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Run encoder dmx comparison against float ref: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Run comparison tools against float ref (long test vectors): $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-long-fx-fx'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Run comparison tools against float ref FX-FX (long test vectors): $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare-to-input'

      variables:

        IVAS_PIPELINE_NAME: 'Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Run comparison tools against input (pass-through only): $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-saturation-smoke-test'

      variables:

        IVAS_PIPELINE_NAME: 'Run saturation smoke-test: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Run saturation smoke-test: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'evs-26444'

      variables:

        IVAS_PIPELINE_NAME: 'EVS 26.444 test: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "EVS 26.444 test: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'sanitizer'

      variables:

        IVAS_PIPELINE_NAME: 'Long testvectors sanitizers'

        IVAS_PIPELINE_NAME: "Long testvectors sanitizers"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-renderer'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer test: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Renderer test: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'complexity'

      variables:

        IVAS_PIPELINE_NAME: 'Complexity Measurement on $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Complexity Measurement on $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test'

      variables:

        IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Voip BE test on $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'renderer-framesize-be'

      variables:

        IVAS_PIPELINE_NAME: 'Renderer framesize BE test on $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Renderer framesize BE test on $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough'

      variables:

        IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "PEAQ encoder pass-through test: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'

      variables:

        IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'

        IVAS_PIPELINE_NAME: "Test long self-test against main pipeline: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

      variables:

        IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: "IVAS Conformance Linux: $CI_COMMIT_BRANCH"

    - if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch

      variables:

        IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'        

        IVAS_PIPELINE_NAME: "Scheduled pipeline: $CI_COMMIT_BRANCH"



# ---------------------------------------------------------------

# Generic script anchors
@@ -681,7 +680,6 @@ workflow: 
        - $XML_REPORT_BRANCH

        - $XML_REPORT_MAIN





.ivas-pytest-sanitizers-anchor: &ivas-pytest-sanitizers-anchor

  extends:

    - .job-linux
@@ -937,7 +935,6 @@ clang-format-check: 
    name: "$ARTIFACT_BASE_NAME"

    expose_as: "formatting patch"





# This job runs in manual jobs and scheduled jobs that compares to the float reference.

# It fetches the latest commit of ivas-float-update and stores in FLOAT_COMMIT_REF

pytest-to-ref-pre:
@@ -1296,13 +1293,13 @@ renderer-pytest-on-merge-request: 


    - exit_code=0

    - testcase_timeout=60

    - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -q --log-level ERROR -n auto -rA tests/renderer_short/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout --durations=0 || exit_code=$?



    ### Run test using branch scripts and input

    - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --odg --html=$REPORT_HTML --self-contained-html --junit-xml=$REPORT_XML tests/renderer_short/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout --durations=0 || exit_code=$?

    - zero_errors=$(cat $REPORT_XML | grep -c 'errors="0"') || true



    - python3 scripts/parse_xml_report.py $REPORT_XML $REPORT_CSV
@@ -1564,7 +1561,6 @@ check-first-frame-is-sid: 
    expose_as: "logs-hrtf-loading"

    expire_in: "5 days"





### From Float

# Check interop IVAS_cod_test -> IVAS_dec_ref

ivas-interop-on-merge-request:
@@ -2290,7 +2286,6 @@ test-long-self-test: 
    - make clean

    - make -j





    ### prepare pytest



    # create references
@@ -2306,7 +2301,6 @@ test-long-self-test: 
    - if [ $exit_code -ne 0 ]; then echo "Non-bitexact cases encountered!"; exit $EXIT_CODE_NON_BE; fi

    - exit 0





  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always
@@ -2361,7 +2355,6 @@ ivas-smoke-test-saturation: 
      - smoke_test_output_hrtf.txt

    expose_as: "saturation smoke test results"





# GCOV/LCOV coverage analysis of self_test suite

coverage-test-on-main-scheduled:

  extends:
@@ -2669,7 +2662,6 @@ ivas-long-term-regressions: 
    - mv regression*bash regressions_"$MEASURE"

    - done





  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    expire_in: 4 weeks
@@ -2722,14 +2714,13 @@ backup-long-term-job-logs: 
    - exit 1

    - fi





# ---------------------------------------------------------------

# Complexity measurement jobs

# ---------------------------------------------------------------



# create necessary environment

.complexity-measurements-setup: &complexity-measurements-setup

  ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work

.complexity-measurements-setup:

  &complexity-measurements-setup ### 1. part: mainly same as in float repo - this is boilerplate code to make the gitlab pages presentation work

  - mkdir -p wmops/logs



  - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID)
@@ -2776,7 +2767,8 @@ backup-long-term-job-logs: 
  - fi



# prepare artifacts -> move to public directory

.complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts

.complexity-measurements-prepare-artifacts:

  &complexity-measurements-prepare-artifacts

  - public_dir="$CI_JOB_NAME-public"

  - mkdir $public_dir

  - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/