Merge branch 'main' of ssh://forge.3gpp.org:29419/sa4/audio/ivas-basop into... (e0ceaf16) · Commits · SA4 / Audio / IVAS BASOP

.gitlab-ci.yml

+132 −4

Original line number	Diff line number	Diff line
		variables:
		# note: GitLab cannot reference variables defined by users in the include ref:, we need to use a YAML anchor for this
		# see https://docs.gitlab.com/ci/yaml/includes/#use-variables-with-include for more information
		IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF a31272de16bd1b556269a50bc179321a60f2a500
		IVAS_CODEC_CI_REF: &IVAS_CODEC_CI_REF main

		include:
		- local: .gitlab-ci/variables.yml
		@@ -29,6 +29,12 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'default' # for testing
		variables:
		IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'long-term-logs'
		variables:
		IVAS_PIPELINE_NAME: 'Aggregate long term logs: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs'
		variables:
		IVAS_PIPELINE_NAME: 'Backup long term logs: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'pytest-compare'
		variables:
		IVAS_PIPELINE_NAME: 'Run comparison tools against float ref: $CI_COMMIT_BRANCH'
		@@ -279,7 +285,7 @@ workflow:
		- if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)
		- echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous"
		- curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- unzip artifacts.zip -d previous_artifacts
		# This wildcard thingy relies on only one csv file being present per job
		- file_previous="previous_artifacts/mld--$CI_JOB_NAME-$id_previous--sha-*.csv"
		@@ -839,6 +845,8 @@ clang-format-check:
		# ensure that codec builds on linux
		build-codec-linux-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs
		@@ -856,6 +864,8 @@ build-codec-linux-make:
		# ensure that codec builds on linux with instrumentation active
		build-codec-linux-instrumented-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
		@@ -873,6 +883,8 @@ build-codec-linux-instrumented-make:

		build-codec-linux-debugging-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
		@@ -891,6 +903,8 @@ build-codec-linux-debugging-make:

		build-codec-windows-msbuild:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs
		@@ -1634,6 +1648,120 @@ voip-be-on-merge-request:
		- make -j >> /dev/null
		- python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py

		# To inject backup copy in manual trigger:
		# - Store logs backup in accessible folder, e.g. /usr/local/backup
		# - Set MANUAL_PIPELINE_TRIGGER to long-term-logs
		# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5
		# - Set LOGS_BACKUP_SOURCE_DIR to source folder, e.g. /usr/local/backup
		ivas-long-term-job-logs:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "long-term-logs"
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "long-term-logs"
		tags:
		- $RUNNER_TAG
		stage: maintenance
		timeout: "25 minutes"
		script:
		- !reference [ .job-linux, before_script ]
		- set -euxo pipefail
		- echo "Running on RUNNER_TAG = $RUNNER_TAG"
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

		# Clean up logs
		- if [ -d "logs" ]; then rm -rf "logs"; fi
		# Inject logs from backup copy if LOGS_BACKUP_SOURCE_DIR is set
		- if [ "$LOGS_BACKUP_SOURCE_DIR" != "" ]; then
		- cp_ret=0
		- cp -r $LOGS_BACKUP_SOURCE_DIR/logs ./ \|\| cp_ret=$?
		- if [ "$cp_ret" != 0 ]; then
		- echo "Error -- Copying to $LOGS_BACKUP_SOURCE_DIR failed!"
		- exit 1
		- fi
		- fi

		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$CI_JOB_NAME" $CI_PROJECT_ID)
		- echo "Job name from variables - "$CI_JOB_NAME", Job ID from script - $id_previous"

		- if [ "$id_previous" != "-1" ]; then
		# Unzip artifacts to recover past logs dir
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		# If there is overlap between injected log and new log, use the new logs.
		- unzip -o artifacts.zip
		- else
		# create logs dir if it doesn't exist (should only happen in first run)
		- echo "Could not find previous job, creating empty logs folder. If this is not the first run, an error likely happened!"
		- mkdir logs
		- fi

		# create folder for today's results
		- TODAY=$(date +'%Y-%m-%d')
		- mkdir -p logs/$TODAY

		# Aggregate job logs
		- job_names="ivas-pytest-compare_ref-long-enc ivas-pytest-compare_ref-long-enc-lev+10 ivas-pytest-compare_ref-long-enc-lev-10 ivas-pytest-compare_ref-long-dec ivas-pytest-compare_ref-long-dec-lev+10 ivas-pytest-compare_ref-long-dec-lev-10 ivas-pytest-compare_ref-long-fx-fx ivas-pytest-compare_ref-long-fx-fx-lev+10 ivas-pytest-compare_ref-long-fx-fx-lev-10"
		- for job_name in $job_names; do
		- echo "Getting job logs for $job_name"
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH "$job_name" $CI_PROJECT_ID)
		- echo "Job ID from variables - "$job_name", Job ID from script - $id_previous"
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- unzip artifacts.zip -d previous_artifacts
		- mv previous_artifacts/mld--"$job_name"-$id_previous--sha-*.csv logs/$TODAY
		- rm artifacts.zip
		- rm -r previous_artifacts
		- done
		- ls logs
		- exit 0

		artifacts:
		name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results"
		expire_in: 4 weeks
		when: always
		paths:
		- logs
		expose_as: "ivas long term job logs results"

		# To store backup copy:
		# - Prepare accessible folder for backup, e.g. /usr/local/backup
		# - Set MANUAL_PIPELINE_TRIGGER to backup-long-term-logs
		# - Set RUNNER_TAG to specific runner with this folder prepared, e.g. test-ericsson-linux-runner-5
		# - Set LOGS_BACKUP_TARGET_DIR to source folder, e.g. /usr/local/backup
		backup-long-term-job-logs:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs"
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $MANUAL_PIPELINE_TYPE == "backup-long-term-logs"
		tags:
		- $RUNNER_TAG
		stage: maintenance
		timeout: "25 minutes"
		script:
		- !reference [ .job-linux, before_script ]
		- set -euxo pipefail
		- echo "Running on RUNNER_TAG = $RUNNER_TAG"
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh

		# Find ID of last run of ivas-long-term-job-logs
		- id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH ivas-long-term-job-logs $CI_PROJECT_ID)

		- if [ "$id_previous" != "-1" ]; then
		# Unzip artifacts to recover past logs dir
		- curl --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$id_previous/artifacts" --output artifacts.zip
		- unzip artifacts.zip
		- else
		# create logs dir if it doesn't exist
		- mkdir logs
		- fi

		# Store logs as backup copy if LOGS_BACKUP_TARGET_DIR is set
		- if [ -d "$LOGS_BACKUP_TARGET_DIR" ]; then
		- cp -r logs $LOGS_BACKUP_TARGET_DIR
		- exit 0
		- else
		- echo "Error - LOGS_BACKUP_TARGET_DIR not set for backup job!"
		- exit 1
		- fi


		# ---------------------------------------------------------------
		# Complexity measurement jobs
		@@ -1649,7 +1777,7 @@ voip-be-on-merge-request:
		# this is a testing/maintenance mechanism to force getting the log history from a specific job id
		# see below in the concrete complexity jobs
		- if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi
		- curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip
		- curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip
		- unzip -qq artifacts.zip \|\| true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html
		- public_dir="$CI_JOB_NAME-public"

		@@ -1679,7 +1807,7 @@ voip-be-on-merge-request:
		### 1.5.part: get the corresponding measurement from ivas-float-update
		- job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID)
		- echo $job_id
		- curl --silent --show-error --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip
		- curl --silent --show-error --request GET "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip
		- unzip -qq -j artifacts_ref.zip "*latest_WMOPS.csv" \|\| true
		# add file to arguments only if the artifact could be retrieved to prevent error later.
		- if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi

.gitlab-ci/variables.yml

+5 −0

Original line number	Diff line number	Diff line
		@@ -46,6 +46,9 @@ variables:
		FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"
		CUT_COMMIT_FILE: "CuT-git-sha.txt"
		MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"
		RUNNER_TAG: "ivas-basop-linux"
		LOGS_BACKUP_SOURCE_DIR: ""
		LOGS_BACKUP_TARGET_DIR: ""
		MANUAL_PIPELINE_TYPE:
		description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."
		value: 'default'
		@@ -65,3 +68,5 @@ variables:
		- 'voip-be-test'
		- 'renderer-framesize-be'
		- 'peaq-enc-passthrough'
		- 'long-term-logs'
		- 'backup-long-term-logs'

lib_com/cnst.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -134,7 +134,7 @@
		#define ONE_IN_Q29 536870912
		#define ONE_IN_Q30 1073741824
		#define ONE_IN_Q31 0x7fffffff
		#define MINUS_ONE_IN_Q31 -2147483648
		#define MINUS_ONE_IN_Q31 (-2147483647 - 1)
		#define TWO_IN_Q29 1073741824
		#define FOUR_IN_Q28 1073741824

lib_com/ivas_masa_com_fx.c

+7 −4

Original line number	Diff line number	Diff line
		@@ -978,7 +978,7 @@ static Word16 quantize_phi_masa_fx(
		Word32 tmp32;
		Word16 tmp_e, delta_phi_e;

		delta_phi_fx = L_deposit_h( BASOP_Util_Divide1616_Scale( 360, n, &delta_phi_e ) );
		delta_phi_fx = BASOP_Util_Divide3232_Scale_newton( 360, n, &delta_phi_e );
		delta_phi_fx = L_shr( delta_phi_fx, sub( sub( 31, delta_phi_e ), 22 ) ); // Q22

		IF( EQ_16( n, 1 ) )
		@@ -1000,8 +1000,8 @@ static Word16 quantize_phi_masa_fx(
		}

		tmp32 = L_add( L_sub( phi_fx, dd_fx ), L_shr( delta_phi_fx, 1 ) );
		id_phi = BASOP_Util_Divide3232_Scale( tmp32, delta_phi_fx, &tmp_e );
		id_phi = shr( id_phi, sub( 15, tmp_e ) ); // Q0
		tmp32 = BASOP_Util_Divide3232_Scale_newton( tmp32, delta_phi_fx, &tmp_e );
		id_phi = extract_l( L_shr( tmp32, sub( 31, tmp_e ) ) ); // Q0

		if ( EQ_16( id_phi, n ) )
		{
		@@ -1014,7 +1014,10 @@ static Word16 quantize_phi_masa_fx(
		id_phi = sub( n, 1 );
		}

		*phi_hat_fx = L_add( L_shl( Mpy_32_16_1( delta_phi_fx, id_phi ), 15 ), dd_fx ); // q22 + q0 - 15 = q7 -> q7 + 15 = q22
		delta_phi_fx = BASOP_Util_Divide3232_Scale_newton( L_mult0( 360, id_phi ), n, &delta_phi_e );
		delta_phi_fx = L_shr( delta_phi_fx, sub( sub( 31, delta_phi_e ), 22 ) ); // Q22

		*phi_hat_fx = L_add( delta_phi_fx, dd_fx ); // Q22
		move32();

		return id_phi; // Q0

lib_com/ivas_rom_com_fx.c

+1 −13

Original line number	Diff line number	Diff line
		@@ -483,20 +483,8 @@ const Word32 dft_res_gains_q_fx[][2] = {

		};

		// Q13
		const Word16 McMASA_LFEGain_vectors_fx_q13[64] = {
		3112, 2703, 1556, 1638, -1310, -1802, -2867, -2785,
		4096, 4096, 4096, 4096, -6553, -3276, 8355, 819,
		-4096, -4096, -4096, -4096, -4587, -983, -6389, 11141,
		-8355, 9666, -4669, 2703, 5898, -9256, 7946, -5079,
		-7454, 7618, 8192, -9011, 14172, -1884, -6389, -6881,
		7782, -13107, -2785, 7618, 7127, 3850, -15564, 4259,
		5488, 11632, -7946, -10158, 6799, 4751, 4997, -16711,
		-6553, -12943, 6717, 11632, -17530, 2129, 6881, 8355
		};

		// Q25
		const Word32 McMASA_LFEGain_vectors_fx_q25[64] = {
		const Word32 McMASA_LFEGain_vectors_fx[64] = {
		12750684, 11072963, 6375342, 6710886, /* 1st bit */
		-5368709, -7381975, -11744051, -11408507,
		16777216, 16777216, 16777216, 16777216, /* 2nd bit*/