Merge branch 'main' into dettbarn_exp5_-_lib_dec (7171f292) · Commits · SA4 / Audio / IVAS BASOP

.gitlab-ci.yml

+193 −5

Original line number	Diff line number	Diff line
		variables:
		# note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this
		# see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information
		IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF 41186736915f5c0a857ac92e8ab438c434e5247d
		IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main

		include:
		- local: .gitlab-ci/variables.yml
		@@ -29,6 +29,12 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing
		variables:
		IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs'
		variables:
		IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs'
		variables:
		IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'
		variables:
		IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'
		@@ -62,6 +68,9 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'voip-be-test'
		variables:
		IVAS_PIPELINE_NAME: 'Voip BE test on $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'renderer-framesize-be'
		variables:
		IVAS_PIPELINE_NAME: 'Renderer framesize BE test on $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'peaq-enc-passthrough'
		variables:
		IVAS_PIPELINE_NAME: 'PEAQ encoder pass-through test: $CI_COMMIT_BRANCH'
		@@ -189,6 +198,7 @@ workflow:
		.test-job-linux-needs-testv-dir:
		extends: .test-job-linux
		before_script:
		- !reference [ .job-linux, before_script ]
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- if [ ! -d "$TESTV_DIR" ]; then mkdir -p $TESTV_DIR; fi
		- cp -r scripts/testv/* $TESTV_DIR/
		@@ -275,7 +285,7 @@ workflow:
		- if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)
		- echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous"
		- curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- unzip artifacts.zip -d previous_artifacts
		# This wildcard thingy relies on only one csv file being present per job
		- file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv"
		@@ -835,6 +845,8 @@ clang-format-check:
		# ensure that codec builds on linux
		build-codec-linux-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs
		@@ -852,6 +864,8 @@ build-codec-linux-make:
		# ensure that codec builds on linux with instrumentation active
		build-codec-linux-instrumented-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
		@@ -869,6 +883,8 @@ build-codec-linux-instrumented-make:

		build-codec-linux-debugging-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
		@@ -887,6 +903,8 @@ build-codec-linux-debugging-make:

		build-codec-windows-msbuild:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs
		@@ -1059,6 +1077,63 @@ check-regressions-short-dec--10db:
		DUT_ENCODER_PATH: "$REF_ENCODER_PATH"
		MERGE_TARGET_ENCODER_PATH: "$REF_ENCODER_PATH"

		check-be-between-renderer-framesizes:
		extends:
		- .test-job-linux
		# - .rules-pytest-to-main-short
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "renderer-framesize-be"
		stage: test
		needs: ["build-codec-linux-make"]
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/disable-limiter.sh
		- make clean
		- make -j
		- echo "$SHORT_TEST_SUITE"

		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-float-ref-and-dut-binaries.sh
		- python3 ci/remove_unsupported_testcases.py $PRM_FILES

		- exit_code=0
		- python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml \|\| exit_code=$?

		- exit_code5=0
		- exit_code10=0
		- python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only \|\| exit_code5=$?
		- python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path ./IVAS_cod_ref --ref_decoder_path ./IVAS_dec --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only \|\| exit_code10=$?

		- zero_errors5=$(cat report-junit-5ms.xml \| grep -c 'errors="0"') \|\| true
		- zero_errors10=$(cat report-junit-10ms.xml \| grep -c 'errors="0"') \|\| true
		- zero_errors=1

		- *print-results-banner
		- echo "!! Encoder command lines are in the 20ms log files only !!!"
		- if [ $exit_code -ne 0 ]; then echo "20 ms framesize run already failed with errors."; zero_errors=0; fi
		- if [ $zero_errors5 != 1 ]; then echo "run error with 5ms rendering encountered"; zero_errors=0 ; fi
		- if [ $zero_errors10 != 1 ]; then echo "run error with 10ms rendering encountered"; zero_errors=0 ; fi
		- if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi
		- if [ $exit_code5 -ne 0 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi
		- if [ $exit_code10 -ne 0 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi
		- if [ $exit_code -ne 0 ]; then exit $EXIT_CODE_FAIL; fi
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		expire_in: 1 week
		when: always
		expose_as: "pytest 5ms and 10ms vs 20ms results"
		paths:
		- report-junit-5ms.xml
		- report-5ms.html
		- report-junit-10ms.xml
		- report-10ms.html
		- report-junit-20ms.xml
		- report-20ms.html
		reports:
		junit:
		- report-junit-5ms.xml
		- report-junit-10ms.xml
		- report-junit-20ms.xml


		# ---------------------------------------------------------------
		# Short test jobs for running from web interface or schedule
		# ---------------------------------------------------------------
		@@ -1573,6 +1648,120 @@ voip-be-on-merge-request:
		- make -j >> /dev/null
		- python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py

		# To inject backup copy in manual trigger:
		# - Store logs backup in accessible folder, e.g. /usr/local/backup
		# - Set MANUAL_PIPELINE_TRIGGER to long-term-logs
		# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5
		# - Set LOGS_BACKUP_SOURCE_DIR to source folder, e.g. /usr/local/backup
		ivas-long-term-job-logs:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs"
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs"
		tags:
		- $RUNNER_TAG
		stage: maintenance
		timeout: "25 minutes"
		script:
		- !reference [ .job-linux, before_script ]
		- set -euxo pipefail
		- echo "Running on RUNNER_TAG = $RUNNER_TAG"
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

		# Clean up logs
		- if [ -d "logs" ]; then rm -rf "logs"; fi
		# Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set
		- if [ "$LOGS_BACKUP_SOURCE_DIR" != "" ]; then
		- cp_ret=0
		- cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ \|\| cp_ret=$?
		- if [ "$cp_ret" != 0 ]; then
		- echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!"
		- exit 1
		- fi
		- fi

		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)
		- echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"

		- if [ "$id_previous" != "-1" ]; then
		# Unzip artifacts to recover past logs dir
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		# If there is overlap between injected log and new log, use the new logs.
		- unzip -o artifacts.zip
		- else
		# create logs dir if it doesn't exist (should only happen in first run)
		- echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!"
		- mkdir logs
		- fi

		# create folder for today's results
		- TODAY=$(date +'%Y-%m-%d')
		- mkdir -p logs/$TODAY

		# Aggregate job logs
		- job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10"
		- for job_name in $job_names; do
		- echo "Getting job logs for $job_name"
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID)
		- echo "Job ID from variables - "$job_name", Job ID from script - $id_previous"
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- unzip artifacts.zip -d previous_artifacts
		- mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY
		- rm artifacts.zip
		- rm -r previous_artifacts
		- done
		- ls logs
		- exit 0

		artifacts:
		name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
		expire_in: 4 weeks
		when: always
		paths:
		- logs
		expose_as: "ivas long term job logs results"

		# To store backup copy:
		# - Prepare accessible folder for backup, e.g. /usr/local/backup
		# - Set MANUAL_PIPELINE_TRIGGER to backup-long-term-logs
		# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5
		# - Set LOGS_BACKUP_TARGET_DIR to source folder, e.g. /usr/local/backup
		backup-long-term-job-logs:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs"
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs"
		tags:
		- $RUNNER_TAG
		stage: maintenance
		timeout: "25 minutes"
		script:
		- !reference [ .job-linux, before_script ]
		- set -euxo pipefail
		- echo "Running on RUNNER_TAG = $RUNNER_TAG"
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

		# Find ID of last run of ivas-long-term-job-logs
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID)

		- if [ "$id_previous" != "-1" ]; then
		# Unzip artifacts to recover past logs dir
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- unzip artifacts.zip
		- else
		# create logs dir if it doesn't exist
		- mkdir logs
		- fi

		# Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set
		- if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then
		- cp -r logs $LOGS_BACKUP_TARGET_DIR
		- exit 0
		- else
		- echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!"
		- exit 1
		- fi


		# ---------------------------------------------------------------
		# Complexity measurement jobs
		@@ -1588,7 +1777,7 @@ voip-be-on-merge-request:
		# this is a testing/maintenance mechanism to force getting the log history from a specific job id
		# see below in the concrete complexity jobs
		- if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi
		- curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip
		- curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip
		- unzip -qq artifacts.zip \|\| true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html
		- public_dir="$CI_JOB_NAME-public"

		@@ -1618,7 +1807,7 @@ voip-be-on-merge-request:
		### 1.5.part: get the corresponding measurement from ivas-float-update
		- job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID)
		- echo $job_id
		- curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip
		- curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip
		- unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" \|\| true
		# add file to arguments only if the artifact could be retrieved to prevent error later.
		- if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi
		@@ -2042,7 +2231,6 @@ pages:
		- if: $UPDATE_PAGES
		script:
		- !reference [ .job-linux, before_script ]

		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- python3 ci/setup_pages.py

.gitlab-ci/variables.yml

+6 −0

Original line number	Diff line number	Diff line
		@@ -46,6 +46,9 @@ variables:
		FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"
		CUT_COMMIT_FILE: "CuT-git-sha.txt"
		MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"
		RUNNER_TAG: "ivas-basop-linux"
		LOGS_BACKUP_SOURCE_DIR: ""
		LOGS_BACKUP_TARGET_DIR: ""
		MANUAL_PIPELINE_TYPE:
		description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."
		value: 'default'
		@@ -63,4 +66,7 @@ variables:
		- 'complexity'
		- 'coverage'
		- 'voip-be-test'
		- 'renderer-framesize-be'
		- 'peaq-enc-passthrough'
		- 'long-term-logs'
		- 'backup-long-term-logs'

apps/encoder.c

+1 −6

Original line number	Diff line number	Diff line
		@@ -758,12 +758,7 @@ int main(
		}

		/* * Encode one frame * */
		if ( ( error = IVAS_ENC_EncodeFrameToSerial( hIvasEnc, pcmBuf, pcmBufSize, bitStream, &numBits
		#ifdef DBG_BITSTREAM_ANALYSIS
		,
		frame
		#endif
		) ) != IVAS_ERR_OK )
		if ( ( error = IVAS_ENC_EncodeFrameToSerial( hIvasEnc, pcmBuf, pcmBufSize, bitStream, &numBits ) ) != IVAS_ERR_OK )
		{
		fprintf( stderr, "\nencodeFrame failed: %s\n\n", IVAS_ENC_GetErrorMessage( error ) );
		goto cleanup;

apps/renderer.c

+100 −49

File changed.

Preview size limit exceeded, changes collapsed.

lib_com/basop32.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -3582,13 +3582,13 @@ Word16 i_mult_o( Word16 a, Word16 b, Flag *Overflow )
		#ifdef ORIGINAL_G7231
		return a * b;
		#else
		register Word32 c = a * b;
		register Word32 c = (Word32) a * b;
		return saturate_o( c, Overflow );
		#endif
		}
		Word16 i_mult( Word16 a, Word16 b )
		{
		return i_mult_o( a, b, NULL );
		return i_mult_sat( a, b );
		}
		Word16 i_mult_sat( Word16 a, Word16 b )
		{