Commit 2c639832 authored by norvell's avatar norvell
Browse files

Skip non-passthrough tests with compare_to_input, add manual test job

parent c6e7dd9d

.gitlab-ci.yml

+40 −1
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ variables: 
      - 'ivas-conformance'

      - 'ivas-conformance-linux'

      - 'check-float-reference'

      - 'test-branch-vs-input-passthrough'

  GIT_CLEAN_FLAGS: -ffdxq

  TESTCASE_TIMEOUT_STV_SANITIZERS: 180

  TESTCASE_TIMEOUT_LTV_SANITIZERS: 1200
@@ -69,7 +70,9 @@ workflow: 
    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-float-reference'

      variables:

        IVAS_PIPELINE_NAME: 'check-float-reference: $CI_COMMIT_BRANCH'

    

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

      variables:

        IVAS_PIPELINE_NAME: 'Pass-through comparison vs input: $CI_COMMIT_BRANCH'



stages:

  - .pre
@@ -265,6 +268,8 @@ stages: 
      when: never      

    - if: $MANUAL_PIPELINE_TYPE == 'check-float-reference'

      when: never

    - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

      when: never

    - when: on_success



.rules-merge-request:
@@ -1639,6 +1644,40 @@ test-long-self-test: 
      junit:

        - report-junit-ltv.xml



test-branch-vs-input-passthrough:

  tags:

    - ivas-linux

  stage: compare

  timeout: "30 minutes" # TBD

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

  allow_failure:

    exit_codes:

      - 123

  script:

    - *print-common-info

    - make -j



    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --compare_to_input --mld --ssnr --odg || exit_code=$?

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit.xml report.csv

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi

    - if [ $exit_code -eq 1 ]; then echo "Differences encountered."; exit $EXIT_CODE_NON_BE; fi

    - exit 0



  artifacts:

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: 4 mos

    paths:

      - report-junit.xml

      - report.html

      - report.csv

    expose_as: "$CI_JOB_NAME results"

    reports:

      junit:

        - report-junit-ltv.xml



# ---------------------------------------------------------------

# Scheduled jobs on main

tests/codec_be_on_mr_nonselection/test_param_file.py

+18 −0
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ import platform 
from pathlib import Path

from subprocess import run

import pytest

import re

import numpy as np



from tests.cmp_pcm import cmp_pcm
@@ -51,6 +52,17 @@ from tests.constants import ( 
    SCRIPTS_DIR,

)



PASSTHROUGH_CONF = [

    (r'[\s\S]*',"EXT"), # Any input with EXT output is pass-through

    (r'-stereo',"STEREO"),

    (r'-sba\s?[-+]?1',"FOA"),

    (r'-sba\s?[-+]?2',"HOA2"),

    (r'-sba\s?[-+]?3',"HOA3"),

    (r'-mc\s5_1',"5_1"),

    (r'-mc\s7_1',"7_1"),

    (r'-mc\s5_1_4',"5_1_4"),

    (r'-mc\s7_1_4',"7_1_4"),

]



VALID_DEC_OUTPUT_CONF = [

    "MONO",
@@ -165,6 +177,12 @@ def test_param_file_tests( 
):

    enc_opts, dec_opts, sim_opts, eid_opts = param_file_test_dict[test_tag]



    # If compare_to_input is set, only run pass-through test cases

    if compare_to_input:

        passthrough = [(a, b) for (a, b) in PASSTHROUGH_CONF if re.search(a, enc_opts) and b in dec_opts]

        if len(passthrough) != 1:

            pytest.skip()



    tag_str = convert_test_string_to_tag(test_tag)



    # evaluate encoder options

tests/codec_be_on_mr_nonselection/test_sba.py

+1 −1
Original line number Diff line number Diff line
@@ -1138,7 +1138,7 @@ def sba_dec( 
        allow_differing_lengths = False

        if compare_to_input:

            # If comparing to input, set input as reference and allow different length

            ref_out_file = test_vector_path

            ref_out_file = f"{test_vector_path}/{tag}.wav"

            allow_differing_lengths = True



        sampling_rate_Hz = int(sampling_rate) * 1000