Commit d8ff2d01 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/encoder-prm-files-refinement' into 'main' 

[CI] Refine .prm files

See merge request !1907
parents d705aad4 6afdb757

.gitlab-ci.yml

+52 −31
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ variables: 
      - 'test-long-self-test'

      - 'ivas-conformance'

      - 'ivas-conformance-linux'

      - 'check-float-reference'

      - 'check-clipping'

      - 'test-branch-vs-input-passthrough'
@@ -70,9 +69,6 @@ workflow: 
      variables:

        IVAS_PIPELINE_NAME: 'Draft IVAS Conformance test -- Linux: $CI_COMMIT_BRANCH'        

    - if: $CI_PIPELINE_SOURCE == 'trigger'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-float-reference'

      variables:

        IVAS_PIPELINE_NAME: 'check-float-reference: $CI_COMMIT_BRANCH'

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'check-clipping'

      variables:

        IVAS_PIPELINE_NAME: 'Check core input clipping: $CI_COMMIT_BRANCH'
@@ -277,8 +273,6 @@ stages: 
      when: never      

    - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'

      when: never      

    - if: $MANUAL_PIPELINE_TYPE == 'check-float-reference'

      when: never

    - if: $MANUAL_PIPELINE_TYPE == 'check-clipping'

    - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough'

      when: never
@@ -421,6 +415,16 @@ check-self-test-names-pre: 
  script:

    - python3 ci/check_self_test_names.py scripts/config/self_test.prm 135



check-no-duplicates-in-self-tests:

  extends:

    - .rules-merge-request

  stage: prevalidate

  needs: []

  tags:

    - ivas-linux

  script:

    - python3 ci/find_duplicates_in_prm_files.py



branch-is-up-to-date-with-main-post:

  extends:

    - .rules-merge-request
@@ -432,8 +436,7 @@ branch-is-up-to-date-with-main-post: 
    - echo $commits_behind_count

    - if [ $commits_behind_count -eq 0 ]; then exit 0; else echo "Your branch is behind main, possibly main changed during your pipeline run, run 'git merge origin/main' to update." exit 1; fi;



# for merges to basop-ci-branch, run the long test suite in reference generation mode to catch problems already here and not only in the BASOP repo

check-compatibility-with-basop-reference-branch:

.basop-ci-branch-compat-template:

  extends:

    - .test-job-linux

  rules:
@@ -441,46 +444,64 @@ check-compatibility-with-basop-reference-branch: 
      changes:

        - tests/**/*

        - scripts/**/*

        - ci/**/*

    - if: ($CI_PIPELINE_SOURCE == 'web' || $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'check-float-reference'

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

  tags:

    - ivas-linux-fast

  script:

  before_script:

    - git clone -b $BASOP_REFERENCE_BRANCH https://forge.3gpp.org/rep/sa4/audio/ivas-basop.git --single-branch

    - make -j -C ivas-basop

    - cp ivas-basop/IVAS_cod ./IVAS_cod_ref

    - cp ivas-basop/IVAS_dec ./IVAS_dec_ref

    - cp ivas-basop/IVAS_cod ./IVAS_cod # Not used but needed to launch the pytest

    - cp ivas-basop/IVAS_dec ./IVAS_dec # Not used but needed to launch the pytest

    - python3 ci/remove_unsupported_testcases.py scripts/config/self_test.prm

    - python3 ci/remove_unsupported_testcases.py $PARAM_FILE

    - python3 tests/create_short_testvectors.py



    - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test.prm -v --update_ref 1 --html=report-selftest-full.html --self-contained-html --junit-xml=report-junit-selftest-full.xml || true

    - zero_errors=$(cat report-junit-selftest-full.xml | grep -c 'errors="0"') || true

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered with scripts/config/self_test.prm !"; exit $EXIT_CODE_FAIL; fi



    - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file scripts/config/self_test_basop_encoder.prm -v --update_ref 1 --html=report-selftest-basop-encoder.html --self-contained-html --junit-xml=report-junit-selftest-basop-encoder.xml || true

    - zero_errors=$(cat report-junit-selftest-basop-encoder.xml | grep -c 'errors="0"') || true

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered with scripts/config/self_test_basop_encoder.prm !"; exit $EXIT_CODE_FAIL; fi

    - python3 -m pytest tests/codec_be_on_mr_nonselection --param_file $PARAM_FILE --update_ref 1 --ref_encoder_path ivas-basop/IVAS_cod --ref_decoder_path ivas-basop/IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml || true

    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true

    - if [ $zero_errors != 1 ]; then echo "Run errors encountered with $PARAM_FILE !"; exit $EXIT_CODE_FAIL; fi



    - exit 0

  artifacts:

    name: "check-float-reference--sha-$CI_COMMIT_SHORT_SHA--results"

    name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"

    when: always

    expire_in: 1 month

    paths:

      - report-junit-selftest-full.xml

      - report-junit-selftest-basop-encoder.xml

      - report-selftest-full.html

      - report-selftest-basop-encoder.html

      - report-junit.xml

      - report.html



    expose_as: "check-float-reference results"

    reports:

      junit:

        - report-junit-selftest-full.xml

        - report-junit-selftest-basop-encoder.xml

        - report-junit.xml



check-compatibility-with-basop-reference-branch-stv:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    PARAM_FILE: scripts/config/self_test.prm

  script:

    - exit 0



check-compatibility-with-basop-reference-branch-ltv:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    PARAM_FILE: scripts/config/self_test_ltv.prm

  script:

    - exit 0



check-compatibility-with-basop-reference-branch-encoder-stv:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    PARAM_FILE: scripts/config/self_test_basop_encoder.prm

  script:

    - exit 0



check-compatibility-with-basop-reference-branch-encoder-ltv:

  extends:

    - .basop-ci-branch-compat-template

  variables:

    PARAM_FILE: scripts/config/self_test_ltv_basop_encoder.prm

  script:

    - exit 0



# ---------------------------------------------------------------

# Build jobs

ci/find_duplicates_in_prm_files.py

0 → 100644
+66 −0
Original line number Diff line number Diff line 
from collections import Counter

from pathlib import Path

import sys

import pprint



HERE = Path(__file__).parent

CONFIGS_DIR = HERE.parent.joinpath("scripts/config")



PRM_FILES = [

    CONFIGS_DIR.joinpath("self_test.prm"),

    CONFIGS_DIR.joinpath("self_test_ltv.prm"),

    CONFIGS_DIR.joinpath("self_test_basop_encoder.prm"),

    CONFIGS_DIR.joinpath("self_test_ltv_basop_encoder.prm"),

]





def main() -> bool:

    duplicates_found = False

    for prm_file in PRM_FILES:

        with open(prm_file) as f:

            contents = f.read()



        # first check if there are complete duplicates (i.e. comment line, encoder cmd and decoder cmd are the same)

        # for "basop_encoder" files, we only need to check for encoder command duplicates

        if "basop_encoder" in prm_file.name:

            lines = contents.split("\n")

            encoder_lines = [l for l in lines if "IVAS_cod" in l]

            encoder_counter = Counter(encoder_lines)

            testcase_duplicates = [l for l in encoder_counter if encoder_counter[l] > 1]

        else:

            blocks = contents.split("\n\n")

            blocks = [b for b in blocks if len(b.strip()) > 0]

            blocks_counter = Counter(blocks)

            testcase_duplicates = [

                b.split("\n")[0] for b in blocks_counter if blocks_counter[b] > 1

            ]



        # also check if there are duplicates in the command names only

        lines = contents.split("\n")

        commands_lines = [

            l

            for l in lines

            if l.strip().startswith("//")

            and "IVAS_cod" not in l

            and "IVAS_dec" not in l

        ]

        commands_counter = Counter(commands_lines)

        commands_duplicates = [l for l in commands_counter if commands_counter[l] > 1]



        if len(testcase_duplicates) > 0:

            duplicates_found = True

            print(f"Duplicate testcases in {prm_file.name} found:")

            pprint.pp(testcase_duplicates)



        if len(commands_duplicates) > 0:

            duplicates_found = True

            print(f"Duplicate command names in {prm_file.name} found:")

            pprint.pp(commands_duplicates)



        print()



    return duplicates_found





if __name__ == "__main__":

    sys.exit(int(main()))

scripts/config/self_test.prm

+4 −16
Original line number Diff line number Diff line
@@ -294,12 +294,12 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_dec STEREO 32 bit testv/stvST32c.wav_stereo_sw_32-32.tst



// stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, MONO out

//../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/stvST48n.wav bit

//../IVAS_dec MONO 48 bit testv/stvST48n.wav_stereo_sw_48-48_DTX_MONO.tst

../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/stvST48n.wav bit

../IVAS_dec MONO 48 bit testv/stvST48n.wav_stereo_sw_48-48_DTX_MONO.tst



// stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, EXT out

//../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/stvST48c.wav bit

//../IVAS_dec EXT 48 bit testv/stvST48c.wav_stereo_sw_48-48_DTX_EXT.tst

../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/stvST48c.wav bit

../IVAS_dec EXT 48 bit testv/stvST48c.wav_stereo_sw_48-48_DTX_EXT.tst





// 1 ISM with metadata at 13.2 kbps, 48 kHz in, 48 kHz out, EXT out
@@ -664,10 +664,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -sba 3 48000 32 testv/stv3OA32c.wav bit

../IVAS_dec BINAURAL_ROOM_IR 32 bit testv/stv3OA32c.wav_SBA_48000_32-32_BinauralRoom.tst



// SBA at 48 kbps, 32kHz in, 32kHz out, BINAURAL ROOM IR out

../IVAS_cod -sba 3 48000 32 testv/stv3OA32c.wav bit

../IVAS_dec BINAURAL_ROOM_IR 32 bit testv/stv3OA32c.wav_SBA_48000_32-32_BinauralRoom_Subframe.tst



// SBA at 48 kbps, 32kHz in, 32kHz out, BINAURAL ROOM IR out, HR

../IVAS_cod -sba 3 48000 32 testv/stv3OA32c.wav bit

../IVAS_dec -t testv/headrot.csv BINAURAL_ROOM_IR 32 bit testv/stv3OA32c.wav_SBA_48000_32-32_BinauralRoom_Headrot.tst
@@ -966,10 +962,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 24400 48 testv/stv1MASA1TC48c.wav bit

../IVAS_dec BINAURAL_ROOM_IR 48 bit testv/stv1MASA1TC48c.wav_24400_48-48_BinauralRoom.tst



// MASA 1dir 1TC at 24.4 kbps, 48kHz in, 48kHz out, BINAURAL ROOM IR out

../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 24400 48 testv/stv1MASA1TC48c.wav bit

../IVAS_dec BINAURAL_ROOM_IR 48 bit testv/stv1MASA1TC48c.wav_24400_48-48_BinauralRoom_Subframe.tst



// MASA 1dir 1TC at 24.4 kbps, 48kHz in, 48kHz out, BINAURAL ROOM IR out, HR

../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 24400 48 testv/stv1MASA1TC48c.wav bit

../IVAS_dec -t testv/headrot.csv BINAURAL_ROOM_IR 48 bit testv/stv1MASA1TC48c.wav_24400_48-48_BinauralRoom_Headrot.tst
@@ -1201,10 +1193,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -masa 2 testv/stv1MASA2TC48n.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/stv1MASA2TC48n.wav bit

../IVAS_dec 7_1 48 bit testv/stv1MASA2TC48n.wav_sw_48-48_7_1.tst



// MASA 1dir 2TC bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, MONO out

../IVAS_cod -masa 2 testv/stv1MASA2TC48n.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/stv1MASA2TC48n.wav bit

../IVAS_dec MONO 48 bit testv/stv1MASA2TC48n.wav_sw_48-48_MONO.tst



// MASA 2dir 2TC bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, 7_1 out

../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/stv2MASA2TC48c.wav bit

../IVAS_dec 7_1 48 bit testv/stv2MASA2TC48c.wav_sw_48-48_7_1.tst

scripts/config/self_test_basop_encoder.prm

+339 −447

File changed.

Preview size limit exceeded, changes collapsed.

scripts/config/self_test_ltv.prm

+4 −25
Original line number Diff line number Diff line
@@ -293,12 +293,12 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_dec STEREO 32 bit testv/ltv32_STEREO.wav_stereo_sw_32-32.tst



// stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, MONO out

//../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_STEREO.wav bit

//../IVAS_dec MONO 48 bit testv/ltv48_STEREO.wav_stereo_sw_48-48_DTX_MONO.tst

../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_STEREO.wav bit

../IVAS_dec MONO 48 bit testv/ltv48_STEREO.wav_stereo_sw_48-48_DTX_MONO.tst



// stereo bitrate switching from 13.2 kbps to 128 kbps, 48kHz in, 48kHz out, DTX on, EXT out

//../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_STEREO.wav bit

//../IVAS_dec EXT 48 bit testv/ltv48_STEREO.wav_stereo_sw_48-48_DTX_EXT.tst

../IVAS_cod -dtx -stereo ../scripts/switchPaths/sw_13k2_to_128k_10fr.bin 48 testv/ltv48_STEREO.wav bit

../IVAS_dec EXT 48 bit testv/ltv48_STEREO.wav_stereo_sw_48-48_DTX_EXT.tst





// 1 ISM with metadata at 13.2 kbps, 48 kHz in, 48 kHz out, EXT out
@@ -662,10 +662,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -sba 3 48000 32 testv/ltv32_HOA3.wav bit

../IVAS_dec BINAURAL_ROOM_IR 32 bit testv/ltv32_HOA3.wav_SBA_48000_32-32_BinauralRoom.tst



// SBA at 48 kbps, 32kHz in, 32kHz out, BINAURAL ROOM IR out

../IVAS_cod -sba 3 48000 32 testv/ltv32_HOA3.wav bit

../IVAS_dec BINAURAL_ROOM_IR 32 bit testv/ltv32_HOA3.wav_SBA_48000_32-32_BinauralRoom_Subframe.tst



// SBA at 48 kbps, 32kHz in, 32kHz out, BINAURAL ROOM IR out, HR

../IVAS_cod -sba 3 48000 32 testv/ltv32_HOA3.wav bit

../IVAS_dec -t testv/headrot.csv BINAURAL_ROOM_IR 32 bit testv/ltv32_HOA3.wav_SBA_48000_32-32_BinauralRoom_Headrot.tst
@@ -963,10 +959,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 24400 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec BINAURAL_ROOM_IR 48 bit testv/ltv48_MASA1TC.wav_24400_48-48_BinauralRoom.tst



// MASA 1TC at 24.4 kbps, 48kHz in, 48kHz out, BINAURAL ROOM IR out

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 24400 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec BINAURAL_ROOM_IR 48 bit testv/ltv48_MASA1TC.wav_24400_48-48_BinauralRoom_Subframe.tst



// MASA 1TC at 24.4 kbps, 48kHz in, 48kHz out, BINAURAL ROOM IR out, HR

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 24400 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec -t testv/headrot.csv BINAURAL_ROOM_IR 48 bit testv/ltv48_MASA1TC.wav_24400_48-48_BinauralRoom_Headrot.tst
@@ -1206,14 +1198,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 
../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec 7_1 48 bit testv/ltv48_MASA2TC.wav_sw_48-48_7_1.tst



// MASA 2TC bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, MONO out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec MONO 48 bit testv/ltv48_MASA2TC.wav_sw_48-48_MONO.tst



// MASA 2TC bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, 7_1 out

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_MASA2TC.wav bit

../IVAS_dec 7_1 48 bit testv/ltv48_MASA2TC.wav_sw_48-48_7_1.tst



// MASA 1TC at 256kbps, 48kHz in, 48kHz out, BINAURAL_ROOM_REVERB out default configuration

../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 256000 48 testv/ltv48_MASA1TC.wav bit

../IVAS_dec BINAURAL_ROOM_REVERB 48 bit testv/ltv48_MASA1TC.wav_BINAURAL_ROOM_REVERB_256000_48-48.tst
@@ -1632,11 +1616,6 @@ networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit 
networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0

../IVAS_dec -Tracefile tracefile_dec -VOIP MONO 32 netsimoutput testv/ltv48_MASA2TC.wav_sw_48-32_MONO_JBM5.tst



// MASA 2TC bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 16kHz out, BINAURAL out, JBM Prof 5

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_MASA2TC.wav bit

networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0

../IVAS_dec -Tracefile tracefile_dec -VOIP BINAURAL 16 netsimoutput testv/ltv48_MASA2TC.wav_sw_48-16_BINAURAL_JBM5.tst



// MASA 2TC bitrate switching from 13.2 kbps to 512 kbps, 48kHz in, 48kHz out, EXT out, JBM Prof 5

../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met ../scripts/switchPaths/sw_13k2_512k.bin 48 testv/ltv48_MASA2TC.wav bit

networkSimulator_g192 ../scripts/dly_error_profiles/dly_error_profile_5.dat bit netsimoutput tracefile_sim 2 0