Commit 6cebea00 authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch '830-Enable-parallelized-test-runs-in-coverage-test' into 'main' 

CI - Enable parallelized test runs in coverage test

See merge request !1171
parents d2b2dba7 7b2a8e46

.gitlab-ci.yml

+3 −3
Original line number Diff line number Diff line
@@ -1522,9 +1522,9 @@ coverage-test-on-main-scheduled: 
    - make GCOV=1 -j

    - cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script

    - python3 tests/create_short_testvectors.py

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n 0 --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n 0 --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

    - bash ci/smoke_test.sh coverage

    - python3 -m pytest -q -n auto tests/renderer/test_renderer_be_comparison.py

    - bash ci/ivas_voip_be_test.sh coverage

CMakeLists.txt

+2 −2
Original line number Diff line number Diff line
@@ -89,8 +89,8 @@ if(UNIX) 
  endif()

  # GCOV

  if(GCOV)

    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")

    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")

    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage -fprofile-update=atomic")

    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage -fprofile-update=atomic")

  endif()

  # STRIP

  if(STRIP)

Makefile

+2 −2
Original line number Diff line number Diff line
@@ -96,8 +96,8 @@ LDFLAGS += -g3 
endif



ifeq "$(GCOV)" "1"

CFLAGS  += -fprofile-arcs -ftest-coverage

LDFLAGS += -fprofile-arcs -ftest-coverage

CFLAGS  += -fprofile-arcs -ftest-coverage -fprofile-update=atomic

LDFLAGS += -fprofile-arcs -ftest-coverage -fprofile-update=atomic

endif



ifeq "$(STRIP)" "1"

ci/smoke_test.sh

+8 −10
Original line number Diff line number Diff line
@@ -43,10 +43,8 @@ if [ ! -d "lib_com" ]; then 
fi



if [ -z "$1" ] || [ "$1" == "test" ]; then

    WORKERS=""

    BUILD=1

elif [ "$1" == "coverage" ]; then

    WORKERS="-t 1"

    BUILD=0

else

    usage
@@ -85,21 +83,21 @@ echo "\n======================= 0. preparing combined format test inputs ======= 
ism_modes=$(./scripts/runIvasCodec.py -l | grep ^ISM)

non_ism_modes=$(./scripts/runIvasCodec.py -l | grep -v ^ISM)

echo "\n======================= 1. non-ism modes no FEC =======================\n\n"

./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg $WORKERS | tee smoke_test_output.txt

./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg | tee smoke_test_output.txt

echo "\n======================= 2. ism modes no FEC =======================\n\n"

./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $WORKERS $ism_md_cmd | tee smoke_test_output.txt

./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $ism_md_cmd | tee smoke_test_output.txt

# run the decoding again, but with 15% frame loss

echo "\n======================= 3. all modes with FEC =======================\n\n"

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg $WORKERS -D="-fec 15" --decoder_only | tee smoke_test_output_plc.txt

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg -D="-fec 15" --decoder_only | tee smoke_test_output_plc.txt



# run JBM modes - EXT is excluded as not supported yet

# OMASA disabled for now

modes_with_no_ext_out=$(./scripts/runIvasCodec.py -l | grep -v ^MASA | grep -v ^ISM | grep -v OMASA)

modes_with_ext_out=$(./scripts/runIvasCodec.py -l | grep 'MASA\|ISM' | grep -v ^ISM+ | grep -v OMASA)

echo "\n======================= 4. JBM, modes with no EXT =======================\n\n"

./scripts/runIvasCodec.py $verbosity_cmd -m $modes_with_no_ext_out -p $cfg $duration_arg $WORKERS --decoder_only --jbm_file $dly_profile | tee smoke_test_output_jbm_noEXT.txt

./scripts/runIvasCodec.py $verbosity_cmd -m $modes_with_no_ext_out -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile | tee smoke_test_output_jbm_noEXT.txt

echo "\n======================= 5. JBM, modes with EXT =======================\n\n"

./scripts/runIvasCodec.py  $verbosity_cmd -m $modes_with_ext_out -p $cfg $duration_arg $WORKERS --decoder_only --jbm_file $dly_profile --oc BINAURAL BINAURAL_ROOM_IR mono stereo FOA HOA3 5_1 7_1_4 | tee -a smoke_test_output_jbm_noEXT.txt

./scripts/runIvasCodec.py  $verbosity_cmd -m $modes_with_ext_out -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile --oc BINAURAL BINAURAL_ROOM_IR mono stereo FOA HOA3 5_1 7_1_4 | tee -a smoke_test_output_jbm_noEXT.txt



# run all modes with binaural output using external files

modes_with_bin_out="FOA HOA2 HOA3 PlanarFOA PlanarHOA2 PlanarHOA3 MASA MC ISM1 ISM2 ISM3 ISM4"
@@ -108,14 +106,14 @@ bin_out_modes="BINAURAL BINAURAL_ROOM_IR" 
echo "\n======================= 6. binaural out with HRTF files - WB =======================\n\n"

wb_modes=$(./scripts/runIvasCodec.py -l -C $modes_with_bin_out | grep _wb_)

hrtf_wb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin"

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg $WORKERS -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt



echo "\n======================= 7. binaural out with HRTF files - SWB =======================\n\n"

swb_modes=$(./scripts/runIvasCodec.py -l -C $modes_with_bin_out | grep _swb_)

hrtf_swb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin"

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg $WORKERS -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt



echo "\n======================= 8. binaural out with HRTF files - FB =======================\n\n"

fb_modes=$(./scripts/runIvasCodec.py -l -C $modes_with_bin_out | grep _fb_)

hrtf_fb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin"

./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg $WORKERS -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt
 
./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes | tee -a smoke_test_output_hrtf.txt