Commit 8520a7af authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'main' into renderer-smoke-test-basop

parents fc0fceb6 6932825e

.gitlab-ci.yml

0 → 100644
+139 −0
Original line number Diff line number Diff line 
default:

  interruptible: true



stages:

  - test



workflow:

  rules:

    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"

      when: never

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'



# Get the merged CI config for ivas-codec and ivas-basop repos based on the current branch and check for it to be valid.

check-merged-ci-config-is-valid:

  stage: test

  tags:

    - ivas-linux



  variables:

    REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec"

    REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop"

  parallel:

    matrix:

      - REPO:

        - ivas-codec

        - ivas-basop



  script:

    - set -euo pipefail



    - |

      if [ "$REPO" == "ivas-codec" ]; then

        repo_url=$REPO_URL_FLOAT

        patch_yml="main-float.yml"

      else

        repo_url=$REPO_URL_BASOP

        patch_yml="main-basop.yml"

      fi



    # artifact names

    - MERGED_RESPONSE="${REPO}-merged.json"

    - BASELINE_RESPONSE="${REPO}-baseline.json"

    - ARTIFACTS_MERGED_YAML="${REPO}-merged.yml"

    - ARTIFACTS_BASELINE_YAML="${REPO}-baseline.yml"

    - ARTIFACTS_DIFF_YAML="${REPO}-diff.yml"

    - TEST_YML="${REPO}-ci.yml"

    - BASE_YML="${REPO}-base-ci.yml"



    # Approach for validating the CI configs:

    # Due to intricacies of gitlab API token perissions/setup, the setup is a bit convoluted. We can't

    # just call the the lint API endpoint of the repective repos, but instead need to

    #   - get the CI file from the respective repo (which includes from this repo's main)

    #   - patch the include section so that it includes the correct local file in this repo (main-float.yml or main-basop-yml)

    #   - call CI lint in THIS repo with passing the patched file -> this should have the same result as the other repo included remotely

    #   - to get the merged yaml back, we need to pass dry_run: true, BUT this causes CI lint to actually run a pipeline which fails

    #     because there are no push pipeline defined...

    #   - so now we need to use the merged yaml file that we got back to again run the static lint and then finally we get a result for the whole thing being valid

 

    # Get CI file from the respective repo

    - curl -sS --url "${repo_url}/-/raw/main/.gitlab-ci.yml?inline=false" --output $TEST_YML



    - cp $TEST_YML $BASE_YML

    - sed -i '/^include:/,$d' $BASE_YML

    - |

      echo -e "include:\n  - local: ${patch_yml}" >> $BASE_YML



    # Get merged baseline with target branch ref

    - |

      jq -Rs '{ dry_run: true, content: . }' < $BASE_YML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $BASELINE_RESPONSE \

         --data-binary @-



    # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...)

    - jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML

    - |

      jq -Rs '{ content: . }' < $ARTIFACTS_BASELINE_YAML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $BASELINE_RESPONSE \

         --data-binary @-



    - sed -i '/^include:/,$d' $TEST_YML

    - |

      echo -e "include:\n  - local: ${patch_yml}" >> $TEST_YML



    # Get current with feature branch ref

    - |

      jq -Rs '{ dry_run: true, content: . }' < $TEST_YML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_REF_NAME}" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $MERGED_RESPONSE \

         --data-binary @-



    # lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...)

    - jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML

    - |

      jq -Rs '{ content: . }' <$ARTIFACTS_MERGED_YAML \

      | curl -sS -X POST \

         --url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \

         -H "Content-Type: application/json" \

         -H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \

         --output $MERGED_RESPONSE \

         --data-binary @-



    # find semantic diff between merged yaml with main and with this branch

    - dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML | tee $ARTIFACTS_DIFF_YAML || true



    # Check validity

    - baseline_valid=$(jq -r ".valid" $BASELINE_RESPONSE)

    - merged_valid=$(jq -r ".valid" $MERGED_RESPONSE)

    - |

      if [ "$baseline_valid" != "true" ]; then

        echo "=== WARNING: BASELINE CONFIG IS INVALID ==="

        jq -r ".errors" $BASELINE_RESPONSE

      fi

    - |

      if [ "$merged_valid" != "true" ]; then

        echo "=== MERGED CONFIG IS INVALID ==="

        jq -r ".errors" $MERGED_RESPONSE

        exit 1

      fi

  artifacts:

    when: always

    access: all

    expire_in: "5 days"

    paths:

      - "*-baseline.yml"

      - "*-merged.yml"

      - "*-diff.yml"

      - "*-baseline.json"

      - "*-merged.json"

includes/check-first-frame-is-sid.yml

0 → 100644
+26 −0
Original line number Diff line number Diff line 
# check for crashes if first received frame on decoder side is an SID

check-first-frame-is-sid:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake"]

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - |

      if [ "$CI_PROJECT_ID" == "$PROJECT_ID_BASOP" ]; then

          bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

      fi

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    # this rm makes check-for-testvectors only check for the signals we actually need in this test

    - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/check-for-testvectors.sh

    - bash ci/run-first-frame-is-sid-test.sh

  timeout: "10 minutes"

  artifacts:

    paths:

      - logs

    when: always

    name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart"

    expose_as: "logs-sidstart"

    expire_in: "5 days"

main-basop.yml

+225 −119

File changed.

Preview size limit exceeded, changes collapsed.

main-float.yml

+42 −38
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ include: 
  - local: includes/smoke-test.yml

  - local: includes/build-jobs.yml

  - local: includes/rules.yml

  - local: includes/check-first-frame-is-sid.yml



default:

  # by default, we want all jobs to be interruptible, and we need to configure this explicitly
@@ -172,7 +173,7 @@ workflow: 


# to be reused in MR and LTV-scheduled sanitizer test jobs

# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section

.sanitizer-selftest-anchor:

.sanitizer-selftest-template:

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh
@@ -196,6 +197,32 @@ workflow: 
    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-20ms.xml errors-20ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-10ms.xml errors-10ms.csv

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit-5ms.xml errors-5ms.csv

  artifacts:

    name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"

    expire_in: 1 week

    when: always

    # NOTE: artifacts paths can't contain '*' when used with 'expose_as'

    # --> reminder for next time you wanna make this more concise...

    paths:

      - report-junit-20ms.xml

      - report-junit-10ms.xml

      - report-junit-5ms.xml

      - report-20ms.html

      - report-10ms.html

      - report-5ms.html

      - errors-20ms.csv

      - errors-10ms.csv

      - errors-5ms.csv

    expose_as: "Sanitizer selftest results"

    reports:

      junit:

        - report-junit-20ms.xml

        - report-junit-10ms.xml

        - report-junit-5ms.xml



# ---------------------------------------------------------------

# Validation jobs
@@ -401,7 +428,7 @@ basop-compat-smoke-test: 
codec-msan:

  extends:

    - .sanitizer-selftest-on-mr

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  tags:

    - ivas-linux-fast

  before_script:
@@ -415,7 +442,7 @@ codec-msan: 
codec-asan:

  extends:

    - .sanitizer-selftest-on-mr

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  before_script:

    - !reference [.job-linux, before_script]

    - CLANG_NUM=2
@@ -427,7 +454,7 @@ codec-asan: 
codec-usan:

  extends:

    - .sanitizer-selftest-on-mr

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  tags:

    - ivas-linux-fast

  before_script:
@@ -526,6 +553,7 @@ renderer-smoke-test: 
    paths:

      - report-junit.xml

      - report.html

      - errors.csv

    reports:

      junit:

        - report-junit.xml
@@ -533,6 +561,8 @@ renderer-smoke-test: 
    - cmake -B cmake-build -G "Unix Makefiles" -DCLANG=$SANITIZER_BUILD_STRING -DCOPY_EXECUTABLES_FROM_BUILD_DIR=true

    - cmake --build cmake-build -- -j

    - testcase_timeout=180

  after_script:

    - python3 scripts/parse_sanitizer_errors_from_xml_report.py report-junit.xml errors.csv



# test renderer executable with cmake + asan

renderer-asan:
@@ -597,7 +627,7 @@ renderer-pytest-on-merge-request: 
    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -748,7 +778,7 @@ split-rendering-pytest-on-merge-request: 
    - if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi



    # run test

    - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -807,7 +837,7 @@ ivas-pytest-on-merge-request: 
    ### run pytest

    - exit_code=0

    - testcase_timeout=60

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    - *merge-request-comparison-check
@@ -855,7 +885,7 @@ ivas-interop-on-merge-request: 
    - exit_code2=0

    # set timeout for individual testcase runs to 60 seconds

    - testcase_timeout=60

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_failures=$(cat report-junit.xml | grep -c 'failures="0"') || true



    - if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -988,32 +1018,6 @@ clang-format-check: 
    name: "$ARTIFACT_BASE_NAME"

    expose_as: "formatting patch"



# check for crashes if first received frame on decoder side is an SID

check-first-frame-is-sid:

  extends:

    - .test-job-linux-needs-testv-dir

    - .rules-merge-request-to-main

  stage: test

  needs: ["build-codec-linux-cmake"]

  script:

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh

    # this rm makes check-for-testvectors only check for the signals we actually need in this test

    - rm scripts/config/ci_linux_ltv.json scripts/config/ci_linux.json

    - bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/check-for-testvectors.sh

    - bash ci/run-first-frame-is-sid-test.sh



  artifacts:

    paths:

      - logs_enc

      - logs_dec_msan

      - logs_dec_asan

      - logs_dec_usan

    when: always

    name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart"

    expose_as: "logs-sidstart"

    expire_in: "5 days"



lc3plus-ensure-no-code-changes:

  extends:

    - .test-job-linux
@@ -1491,7 +1495,7 @@ test-long-self-test: 


    ### run pytest self-test using long test vectors

    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -1583,7 +1587,7 @@ test-branch-vs-input-passthrough: 
ltv-msan:

  extends:

    - .sanitizer-selftest-ltv

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  rules:

    - if: $SANITIZER_SCHEDULE_E

  timeout: 5 hours
@@ -1600,7 +1604,7 @@ ltv-msan: 
ltv-asan:

  extends:

    - .sanitizer-selftest-ltv

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  rules:

    - if: $SANITIZER_SCHEDULE_E

      when: delayed
@@ -1619,7 +1623,7 @@ ltv-asan: 
ltv-usan:

  extends:

    - .sanitizer-selftest-ltv

    - .sanitizer-selftest-anchor

    - .sanitizer-selftest-template

  rules:

    - if: $SANITIZER_SCHEDULE_E

      when: delayed