From 06a37541b3ca484d618c92b87b54fc23dd8cd45f Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 14 Jul 2022 16:19:33 +0200
Subject: [PATCH 01/24] add script and test job for be-to-evs-mono test

---
 .gitlab-ci.yml        | 26 ++++++++++++++++
 ci/run_evs_be_test.py | 70 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 ci/run_evs_be_test.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f734e43e01..9303e87c13 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,6 @@
 variables:
   TESTV_DIR: "/usr/local/testv"
+  EVS_BE_TEST_DIR: "/usr/local/evs_be_test"
 
 
 # prevent running two pipelines on pushes to merge request branches
@@ -97,6 +98,31 @@ codec-smoke-test:
       - out/logs
 
 
+# check bitexactness to EVS
+be-2-evs:
+  extends: .test-job-linux
+  tags:
+    - test-fhg-linux-runner1
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+  stage: test
+  needs: [ "build-codec-linux-cmake" ]
+  script:
+    - mkdir build
+    - cd build
+    - cmake ..
+    - make -j
+    - cd ..
+
+    # try with copy-over first
+    - cp -r $EVS_BE_TEST_DIR ./evs_be_test
+    - cp IVAS_cod ./evs_be_test/bin/EVS_cod
+    - cp IVAS_dec ./evs_be_test/bin/EVS_dec
+
+    - cd evs_be_test
+    - python3 ci/run_evs_be_check.py
+
+
 # compare bit exactness between target and source branch
 self-test-on-merge-request:
   extends: .test-job-linux
diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
new file mode 100644
index 0000000000..3c83b3185d
--- /dev/null
+++ b/ci/run_evs_be_test.py
@@ -0,0 +1,70 @@
+import subprocess
+import pathlib
+import sys
+import concurrent.futures
+from threading import Lock
+
+
+README_FILES_PARALLEL = [
+    "Readme_AMRWB_IO_enc.txt",
+    "Readme_AMRWB_IO_dec.txt",
+    "Readme_EVS_enc.txt",
+    "Readme_EVS_dec.txt",
+]
+README_FILES_JBM = ["Readme_JBM_dec.txt"]
+README_FILES = README_FILES_PARALLEL + README_FILES_JBM
+BINARY_PATHS = ["bin/EVS_cod", "bin/EVS_dec"]
+FOLDER_PATHS = ["switchPaths", "testv"]
+
+
+def main():
+
+    if not environment_is_correct():
+        return 1
+
+    result_dict = dict()
+    # run first part in parallel
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        executor.map(
+            run_file, README_FILES_PARALLEL, [result_dict] * len(README_FILES_PARALLEL)
+        )
+
+    # JBM test after that
+    run_file(README_FILES_JBM[0], result_dict)
+
+    return analyze_results(result_dict)
+
+
+def analyze_results(result_dict):
+    ret = 0
+
+    for filename, ret_code in result_dict.items():
+        print(filename, ret_code)
+        if ret_code != 0:
+            print(f"========= Test for {filename} failed! See log below: ==========")
+            with open(filename.replace(".txt", "log.txt")) as f:
+                print(f.read())
+            ret = 1
+
+    return ret
+
+
+def run_file(filename: str, result_dict: dict):
+    ret_code = subprocess.call(["bash", filename])
+    with Lock():
+        result_dict[filename] = ret_code
+
+
+def environment_is_correct():
+    ok = True
+
+    for path in README_FILES + BINARY_PATHS + FOLDER_PATHS:
+        if not pathlib.Path(path).exists():
+            print(f"Environment setup is incorrect - {path} not found.")
+            ok = False
+
+    return ok
+
+
+if __name__ == "__main__":
+    sys.exit(main())
-- 
GitLab


From b9c42b06c7f597bb71475c487af4467d68321633 Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 14 Jul 2022 17:33:58 +0000
Subject: [PATCH 02/24] fix EVS test dir

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9303e87c13..f19ff6924b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,6 @@
 variables:
   TESTV_DIR: "/usr/local/testv"
-  EVS_BE_TEST_DIR: "/usr/local/evs_be_test"
+  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"
 
 
 # prevent running two pipelines on pushes to merge request branches
-- 
GitLab


From b56ecef48694091e271e4316371c4ffd11424a57 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 08:42:10 +0200
Subject: [PATCH 03/24] fix path for copying

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f19ff6924b..6df10a2455 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -116,8 +116,8 @@ be-2-evs:
 
     # try with copy-over first
     - cp -r $EVS_BE_TEST_DIR ./evs_be_test
-    - cp IVAS_cod ./evs_be_test/bin/EVS_cod
-    - cp IVAS_dec ./evs_be_test/bin/EVS_dec
+    - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod
+    - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec
 
     - cd evs_be_test
     - python3 ci/run_evs_be_check.py
-- 
GitLab


From 23c40a735b87d562e6e0845f67e6f7cc22afccdb Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 06:54:07 +0000
Subject: [PATCH 04/24] fix path

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6df10a2455..dd0191ce2b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,7 +120,7 @@ be-2-evs:
     - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec
 
     - cd evs_be_test
-    - python3 ci/run_evs_be_check.py
+    - python3 ../ci/run_evs_be_check.py
 
 
 # compare bit exactness between target and source branch
-- 
GitLab


From 4e408dab40f9594fdcce9b81ed8dc6bfc517b044 Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 06:56:34 +0000
Subject: [PATCH 05/24] fix typo in filename

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dd0191ce2b..3fda028043 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,7 +120,7 @@ be-2-evs:
     - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec
 
     - cd evs_be_test
-    - python3 ../ci/run_evs_be_check.py
+    - python3 ../ci/run_evs_be_test.py
 
 
 # compare bit exactness between target and source branch
-- 
GitLab


From ad67b625709b0f8c0a2bd52e98a77adeaf2ad390 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 09:14:24 +0200
Subject: [PATCH 06/24] fix log file name

---
 ci/run_evs_be_test.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
index 3c83b3185d..43112580ae 100644
--- a/ci/run_evs_be_test.py
+++ b/ci/run_evs_be_test.py
@@ -29,7 +29,7 @@ def main():
             run_file, README_FILES_PARALLEL, [result_dict] * len(README_FILES_PARALLEL)
         )
 
-    # JBM test after that
+    # JBM test can not run concurrently with the others
     run_file(README_FILES_JBM[0], result_dict)
 
     return analyze_results(result_dict)
@@ -39,10 +39,9 @@ def analyze_results(result_dict):
     ret = 0
 
     for filename, ret_code in result_dict.items():
-        print(filename, ret_code)
         if ret_code != 0:
             print(f"========= Test for {filename} failed! See log below: ==========")
-            with open(filename.replace(".txt", "log.txt")) as f:
+            with open(filename.replace(".txt", "_log.txt")) as f:
                 print(f.read())
             ret = 1
 
@@ -56,14 +55,20 @@ def run_file(filename: str, result_dict: dict):
 
 
 def environment_is_correct():
-    ok = True
+    """
+    Check that the folder with the test resources is et up correctly:
+    - all Readme files there
+    - EVS binaris available in bin/
+    - testv and switchPaths folder exist - Content is not checked, though
+    """
+    ret = True
 
     for path in README_FILES + BINARY_PATHS + FOLDER_PATHS:
         if not pathlib.Path(path).exists():
             print(f"Environment setup is incorrect - {path} not found.")
-            ok = False
+            ret = False
 
-    return ok
+    return ret
 
 
 if __name__ == "__main__":
-- 
GitLab


From 0b0aab50af3d90aee5052bb1b36ef0683b84aae4 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 10:40:58 +0200
Subject: [PATCH 07/24] make script executable

---
 ci/run_evs_be_test.py | 1 +
 1 file changed, 1 insertion(+)
 mode change 100644 => 100755 ci/run_evs_be_test.py

diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
old mode 100644
new mode 100755
index 43112580ae..5e268054a4
--- a/ci/run_evs_be_test.py
+++ b/ci/run_evs_be_test.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import subprocess
 import pathlib
 import sys
-- 
GitLab


From 56e8fb5f8e614bc866463cd2a5ffd3d7e603187e Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 09:27:12 +0000
Subject: [PATCH 08/24] Run EVS BE check on pushes to main to not increase MR
 pipeline runtime

---
 .gitlab-ci.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6edc7373d5..a72b6a6945 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,9 +104,8 @@ be-2-evs:
   tags:
     - test-fhg-linux-runner1
   rules:
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_BRANCH == 'main'
   stage: test
-  needs: [ "build-codec-linux-cmake" ]
   script:
     - mkdir build
     - cd build
@@ -114,7 +113,7 @@ be-2-evs:
     - make -j
     - cd ..
 
-    # try with copy-over first
+    # copy over to never change the testvector dir
     - cp -r $EVS_BE_TEST_DIR ./evs_be_test
     - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod
     - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec
-- 
GitLab


From fbe8e3827a0885cb0efe4807177629f64280d5cf Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Fri, 15 Jul 2022 10:40:58 +0000
Subject: [PATCH 09/24] restrict BE comparison to FhG runner for debugging

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a72b6a6945..2a973d37ce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -125,6 +125,8 @@ be-2-evs:
 # compare bit exactness between target and source branch
 self-test-on-merge-request:
   extends: .test-job-linux
+  tags:
+    - test-fhg-linux-runner1
   stage: compare
   needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]
   rules:
-- 
GitLab


From 8779a64a0d90b0e84924b7f6eb5e9eaadd733e84 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Fri, 22 Jul 2022 09:33:10 +0200
Subject: [PATCH 10/24] change name of job

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f4a54fdb9d..0aecea17a9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -99,7 +99,7 @@ codec-smoke-test:
 
 
 # check bitexactness to EVS
-be-2-evs:
+be-2-evs-linux:
   extends: .test-job-linux
   tags:
     - test-fhg-linux-runner1
-- 
GitLab


From 8ef7a0e27f62011405876af5fbf65c6db670b40c Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 28 Jul 2022 17:41:56 +0200
Subject: [PATCH 11/24] adapt to using conformance package

---
 ci/run_evs_be_test.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
index 5e268054a4..109f9c2100 100755
--- a/ci/run_evs_be_test.py
+++ b/ci/run_evs_be_test.py
@@ -14,9 +14,9 @@ README_FILES_PARALLEL = [
 ]
 README_FILES_JBM = ["Readme_JBM_dec.txt"]
 README_FILES = README_FILES_PARALLEL + README_FILES_JBM
-BINARY_PATHS = ["bin/EVS_cod", "bin/EVS_dec"]
+BINARY_PATHS = ["./bin/EVS_cod", "./bin/EVS_dec"]
 FOLDER_PATHS = ["switchPaths", "testv"]
-
+BIN_PATHS = BINARY_PATHS * 2
 
 def main():
 
@@ -27,13 +27,13 @@ def main():
     # run first part in parallel
     with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
         executor.map(
-            run_file, README_FILES_PARALLEL, [result_dict] * len(README_FILES_PARALLEL)
+            run_file, README_FILES_PARALLEL, BIN_PATHS, [result_dict] * len(README_FILES_PARALLEL)
         )
 
     # JBM test can not run concurrently with the others
-    run_file(README_FILES_JBM[0], result_dict)
+    run_file(README_FILES_JBM[0], BINARY_PATHS[1], result_dict)
 
-    return analyze_results(result_dict)
+    return int(any(result_dict.values()))
 
 
 def analyze_results(result_dict):
@@ -49,17 +49,17 @@ def analyze_results(result_dict):
     return ret
 
 
-def run_file(filename: str, result_dict: dict):
-    ret_code = subprocess.call(["bash", filename])
+def run_file(filename: str, bin_path: str, result_dict: dict):
+    ret_code = subprocess.call(["bash", filename, bin_path])
     with Lock():
         result_dict[filename] = ret_code
 
 
 def environment_is_correct():
     """
-    Check that the folder with the test resources is et up correctly:
+    Check that the folder with the test resources is set up correctly:
     - all Readme files there
-    - EVS binaris available in bin/
+    - EVS binaries available in bin/
     - testv and switchPaths folder exist - Content is not checked, though
     """
     ret = True
-- 
GitLab


From 3b5e831f5ac0b2b7958c09f791afb4a29e0de462 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 28 Jul 2022 17:43:13 +0200
Subject: [PATCH 12/24] make run on push

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0aecea17a9..30ec110dcb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -103,8 +103,8 @@ be-2-evs-linux:
   extends: .test-job-linux
   tags:
     - test-fhg-linux-runner1
-  rules:
-    - if: $CI_COMMIT_BRANCH == 'main'
+  # rules:
+  #   - if: $CI_COMMIT_BRANCH == 'main'
   stage: test
   script:
     - mkdir build
-- 
GitLab


From acc6b53398f6b6d7edbf5099e9c60c84764d6e9e Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 28 Jul 2022 18:13:33 +0200
Subject: [PATCH 13/24] disable all other jobs to save time

---
 .gitlab-ci.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index afc38e13e3..3cdfcf3e34 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,7 +56,7 @@ stages:
       - 123
 
 
-build-codec-linux-make:
+.build-codec-linux-make:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -65,7 +65,7 @@ build-codec-linux-make:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-unittests-linux:
+.build-unittests-linux:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -74,7 +74,7 @@ build-unittests-linux:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-prerenderer-linux:
+.build-prerenderer-linux:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -83,7 +83,7 @@ build-prerenderer-linux:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-td-object-renderer-standalone-linux:
+.build-td-object-renderer-standalone-linux:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -92,7 +92,7 @@ build-td-object-renderer-standalone-linux:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-codec-linux-cmake:
+.build-codec-linux-cmake:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -105,7 +105,7 @@ build-codec-linux-cmake:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-codec-instrumented-linux:
+.build-codec-instrumented-linux:
   extends: .build-job-linux
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -113,7 +113,7 @@ build-codec-instrumented-linux:
     - bash ci/build_codec_instrumented_linux.sh
 
 # make sure that the codec builds with msan, asan and usan
-build-codec-sanitizers-linux:
+.build-codec-sanitizers-linux:
   extends: .build-job-linux
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -122,7 +122,7 @@ build-codec-sanitizers-linux:
 
 
 # test that runs all modes with 1s input signals
-codec-smoke-test:
+.codec-smoke-test:
   extends: .test-job-linux-needs-testv-dir
   # temporarily restrict this job to the only runner which (so far) seems to runit without problems
   timeout: "5 minutes"
-- 
GitLab


From d1a4c87a9130c8d901ba7c02e6aeaad5d53eb97f Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 28 Jul 2022 16:14:59 +0000
Subject: [PATCH 14/24] really disable all other jobs

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3cdfcf3e34..2d67342161 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -163,7 +163,7 @@ be-2-evs-linux:
     - cd evs_be_test
     - python3 ../ci/run_evs_be_test.py
 # code selftest testvectors with memory-sanitizer binaries
-msan-on-merge-request-linux:
+.msan-on-merge-request-linux:
   extends: .test-job-linux
   stage: test
   needs: [ "build-codec-sanitizers-linux" ]
@@ -178,7 +178,7 @@ msan-on-merge-request-linux:
 
 
 # code selftest testvectors with address-sanitizer binaries
-asan-on-merge-request-linux:
+.asan-on-merge-request-linux:
   extends: .test-job-linux
   stage: test
   needs: [ "build-codec-sanitizers-linux" ]
@@ -193,7 +193,7 @@ asan-on-merge-request-linux:
 
 
 # compare bit exactness between target and source branch
-self-test-on-merge-request:
+.self-test-on-merge-request:
   extends: .test-job-linux
   tags:
     - test-fhg-linux-runner1
-- 
GitLab


From 76e47bba12e5ce815ae49fada0811381c1fe2816 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 28 Jul 2022 18:19:03 +0200
Subject: [PATCH 15/24] adapt env check for new environment

---
 ci/run_evs_be_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
index 109f9c2100..ffcda6578b 100755
--- a/ci/run_evs_be_test.py
+++ b/ci/run_evs_be_test.py
@@ -15,7 +15,7 @@ README_FILES_PARALLEL = [
 README_FILES_JBM = ["Readme_JBM_dec.txt"]
 README_FILES = README_FILES_PARALLEL + README_FILES_JBM
 BINARY_PATHS = ["./bin/EVS_cod", "./bin/EVS_dec"]
-FOLDER_PATHS = ["switchPaths", "testv"]
+FOLDER_PATHS = ["testv"]
 BIN_PATHS = BINARY_PATHS * 2
 
 def main():
-- 
GitLab


From 834672083caef1474e87183f5717d80c144b2ef4 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 28 Jul 2022 19:20:11 +0200
Subject: [PATCH 16/24] run job only on main

---
 .gitlab-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2d67342161..a35d0eab10 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,7 +56,7 @@ stages:
       - 123
 
 
-.build-codec-linux-make:
+build-codec-linux-make:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -65,7 +65,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-unittests-linux:
+build-unittests-linux:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -74,7 +74,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-prerenderer-linux:
+build-prerenderer-linux:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -83,7 +83,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-td-object-renderer-standalone-linux:
+build-td-object-renderer-standalone-linux:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -92,7 +92,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-codec-linux-cmake:
+build-codec-linux-cmake:
   extends: .build-job-with-check-for-warnings
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -105,7 +105,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-codec-instrumented-linux:
+build-codec-instrumented-linux:
   extends: .build-job-linux
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -113,7 +113,7 @@ stages:
     - bash ci/build_codec_instrumented_linux.sh
 
 # make sure that the codec builds with msan, asan and usan
-.build-codec-sanitizers-linux:
+build-codec-sanitizers-linux:
   extends: .build-job-linux
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
@@ -122,7 +122,7 @@ stages:
 
 
 # test that runs all modes with 1s input signals
-.codec-smoke-test:
+codec-smoke-test:
   extends: .test-job-linux-needs-testv-dir
   # temporarily restrict this job to the only runner which (so far) seems to runit without problems
   timeout: "5 minutes"
@@ -145,8 +145,8 @@ be-2-evs-linux:
   extends: .test-job-linux
   tags:
     - test-fhg-linux-runner1
-  # rules:
-  #   - if: $CI_COMMIT_BRANCH == 'main'
+  rules:
+    - if: $CI_COMMIT_BRANCH == 'main'
   stage: test
   script:
     - mkdir build
@@ -163,7 +163,7 @@ be-2-evs-linux:
     - cd evs_be_test
     - python3 ../ci/run_evs_be_test.py
 # code selftest testvectors with memory-sanitizer binaries
-.msan-on-merge-request-linux:
+msan-on-merge-request-linux:
   extends: .test-job-linux
   stage: test
   needs: [ "build-codec-sanitizers-linux" ]
@@ -178,7 +178,7 @@ be-2-evs-linux:
 
 
 # code selftest testvectors with address-sanitizer binaries
-.asan-on-merge-request-linux:
+asan-on-merge-request-linux:
   extends: .test-job-linux
   stage: test
   needs: [ "build-codec-sanitizers-linux" ]
@@ -193,7 +193,7 @@ be-2-evs-linux:
 
 
 # compare bit exactness between target and source branch
-.self-test-on-merge-request:
+self-test-on-merge-request:
   extends: .test-job-linux
   tags:
     - test-fhg-linux-runner1
-- 
GitLab


From d150000fd94d0c176b8fe8150cdf4844fcef82ca Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Wed, 31 Aug 2022 20:13:19 +0200
Subject: [PATCH 17/24] temporarily only run one job for testing

---
 .gitlab-ci.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 29d436e68b..65d7455a7f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,7 +104,7 @@ stages:
 # Build jobs
 # ---------------------------------------------------------------
 
-build-codec-linux-make:
+.build-codec-linux-make:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -114,7 +114,7 @@ build-codec-linux-make:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-unittests-linux:
+.build-unittests-linux:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -124,7 +124,7 @@ build-unittests-linux:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-prerenderer-linux:
+.build-prerenderer-linux:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -134,7 +134,7 @@ build-prerenderer-linux:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-td-object-renderer-standalone-linux:
+.build-td-object-renderer-standalone-linux:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -144,7 +144,7 @@ build-td-object-renderer-standalone-linux:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-codec-linux-cmake:
+.build-codec-linux-cmake:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -158,7 +158,7 @@ build-codec-linux-cmake:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-build-codec-instrumented-linux:
+.build-codec-instrumented-linux:
   extends: 
     - .build-job-linux
     - .rules-basis
@@ -167,7 +167,7 @@ build-codec-instrumented-linux:
     - bash ci/build_codec_instrumented_linux.sh
 
 # make sure that the codec builds with msan, asan and usan
-build-codec-sanitizers-linux:
+.build-codec-sanitizers-linux:
   extends: 
     - .build-job-linux
     - .rules-basis
@@ -181,7 +181,7 @@ build-codec-sanitizers-linux:
 # ---------------------------------------------------------------
 
 # test that runs all modes with 1s input signals
-codec-smoke-test:
+.codec-smoke-test:
   extends: 
     - .test-job-linux-needs-testv-dir
     - .rules-merge-request
@@ -228,7 +228,7 @@ be-2-evs-linux:
 
 
 # code selftest testvectors with memory-sanitizer binaries
-msan-on-merge-request-linux:
+.msan-on-merge-request-linux:
   extends: 
     - .test-job-linux
     - .rules-merge-request
@@ -244,7 +244,7 @@ msan-on-merge-request-linux:
 
 
 # code selftest testvectors with address-sanitizer binaries
-asan-on-merge-request-linux:
+.asan-on-merge-request-linux:
   extends: 
     - .test-job-linux
     - .rules-merge-request
@@ -260,7 +260,7 @@ asan-on-merge-request-linux:
 
 
 # compare bit exactness between target and source branch
-self-test-on-merge-request:
+.self-test-on-merge-request:
   extends: 
     - .test-job-linux
     - .rules-merge-request
@@ -356,7 +356,7 @@ self-test-on-merge-request:
 # Test jobs for main branch
 # ---------------------------------------------------------------
 
-codec-comparison-on-main-push:
+.codec-comparison-on-main-push:
   extends:
     - .test-job-linux
     - .rules-main-push
-- 
GitLab


From a100828e3149a62d7d408b26769b5b8877926fa2 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 08:39:06 +0200
Subject: [PATCH 18/24] Revert "temporarily only run one job for testing"

This reverts commit d150000fd94d0c176b8fe8150cdf4844fcef82ca.
---
 .gitlab-ci.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 65d7455a7f..29d436e68b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,7 +104,7 @@ stages:
 # Build jobs
 # ---------------------------------------------------------------
 
-.build-codec-linux-make:
+build-codec-linux-make:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -114,7 +114,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-unittests-linux:
+build-unittests-linux:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -124,7 +124,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-prerenderer-linux:
+build-prerenderer-linux:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -134,7 +134,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-td-object-renderer-standalone-linux:
+build-td-object-renderer-standalone-linux:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -144,7 +144,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-codec-linux-cmake:
+build-codec-linux-cmake:
   extends: 
     - .build-job-with-check-for-warnings
     - .rules-basis
@@ -158,7 +158,7 @@ stages:
     # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
     - ci/check_for_warnings.py $BUILD_OUTPUT || exit $?
 
-.build-codec-instrumented-linux:
+build-codec-instrumented-linux:
   extends: 
     - .build-job-linux
     - .rules-basis
@@ -167,7 +167,7 @@ stages:
     - bash ci/build_codec_instrumented_linux.sh
 
 # make sure that the codec builds with msan, asan and usan
-.build-codec-sanitizers-linux:
+build-codec-sanitizers-linux:
   extends: 
     - .build-job-linux
     - .rules-basis
@@ -181,7 +181,7 @@ stages:
 # ---------------------------------------------------------------
 
 # test that runs all modes with 1s input signals
-.codec-smoke-test:
+codec-smoke-test:
   extends: 
     - .test-job-linux-needs-testv-dir
     - .rules-merge-request
@@ -228,7 +228,7 @@ be-2-evs-linux:
 
 
 # code selftest testvectors with memory-sanitizer binaries
-.msan-on-merge-request-linux:
+msan-on-merge-request-linux:
   extends: 
     - .test-job-linux
     - .rules-merge-request
@@ -244,7 +244,7 @@ be-2-evs-linux:
 
 
 # code selftest testvectors with address-sanitizer binaries
-.asan-on-merge-request-linux:
+asan-on-merge-request-linux:
   extends: 
     - .test-job-linux
     - .rules-merge-request
@@ -260,7 +260,7 @@ be-2-evs-linux:
 
 
 # compare bit exactness between target and source branch
-.self-test-on-merge-request:
+self-test-on-merge-request:
   extends: 
     - .test-job-linux
     - .rules-merge-request
@@ -356,7 +356,7 @@ be-2-evs-linux:
 # Test jobs for main branch
 # ---------------------------------------------------------------
 
-.codec-comparison-on-main-push:
+codec-comparison-on-main-push:
   extends:
     - .test-job-linux
     - .rules-main-push
-- 
GitLab


From 7951f104fda09cb2d86c3ea16dccf7b22b1050fd Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 07:49:03 +0000
Subject: [PATCH 19/24] test other runner

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 29d436e68b..1ece379701 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -209,7 +209,7 @@ be-2-evs-linux:
     - .test-job-linux
     - .rules-merge-request
   tags:
-    - test-fhg-linux-runner1
+    - test-fhg-linux-runner2
   stage: test
   script:
     - mkdir build
-- 
GitLab


From c0c4f8c772f73dab4ee2d58a015eee938c794770 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 10:40:51 +0200
Subject: [PATCH 20/24] temporarily change EVS output on purpose for testing

---
 lib_dec/evs_dec.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib_dec/evs_dec.c b/lib_dec/evs_dec.c
index b51ed8d6ea..661b73e0c9 100755
--- a/lib_dec/evs_dec.c
+++ b/lib_dec/evs_dec.c
@@ -949,6 +949,12 @@ ivas_error evs_dec(
         updt_dec_common( st, hq_core_type, concealWholeFrameTmp, output );
     }
 
+    // only temporary for testing the EVS-BE test -> purposely change BE for limited operating points
+    if ( st->total_brate == 48000 && st->bwidth == SWB )
+    {
+      output[0] = 0.0f;
+    }
+
 
 #ifdef DEBUG_MODE_INFO
     for ( i = 0; i < ( st->L_frame / L_SUBFR ); i++ )
-- 
GitLab


From f50511dad03ce5f32509072b12b72744927b1a26 Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 11:31:29 +0200
Subject: [PATCH 21/24] adapt error reporting to new scripts

---
 ci/run_evs_be_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
index ffcda6578b..9fa64877f2 100755
--- a/ci/run_evs_be_test.py
+++ b/ci/run_evs_be_test.py
@@ -33,7 +33,7 @@ def main():
     # JBM test can not run concurrently with the others
     run_file(README_FILES_JBM[0], BINARY_PATHS[1], result_dict)
 
-    return int(any(result_dict.values()))
+    return analyze_results(result_dict)
 
 
 def analyze_results(result_dict):
@@ -42,7 +42,7 @@ def analyze_results(result_dict):
     for filename, ret_code in result_dict.items():
         if ret_code != 0:
             print(f"========= Test for {filename} failed! See log below: ==========")
-            with open(filename.replace(".txt", "_log.txt")) as f:
+            with open(filename.replace("Readme", "Log")) as f:
                 print(f.read())
             ret = 1
 
-- 
GitLab


From bb56f0331d57012fde2116e5593eec4e4f76e3be Mon Sep 17 00:00:00 2001
From: knj <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 12:02:32 +0200
Subject: [PATCH 22/24] Revert "temporarily change EVS output on purpose for
 testing"

This reverts commit c0c4f8c772f73dab4ee2d58a015eee938c794770.
---
 lib_dec/evs_dec.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lib_dec/evs_dec.c b/lib_dec/evs_dec.c
index 661b73e0c9..b51ed8d6ea 100755
--- a/lib_dec/evs_dec.c
+++ b/lib_dec/evs_dec.c
@@ -949,12 +949,6 @@ ivas_error evs_dec(
         updt_dec_common( st, hq_core_type, concealWholeFrameTmp, output );
     }
 
-    // only temporary for testing the EVS-BE test -> purposely change BE for limited operating points
-    if ( st->total_brate == 48000 && st->bwidth == SWB )
-    {
-      output[0] = 0.0f;
-    }
-
 
 #ifdef DEBUG_MODE_INFO
     for ( i = 0; i < ( st->L_frame / L_SUBFR ); i++ )
-- 
GitLab


From 1d39ff85be70970374487aa0e05383c9cd2d26de Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 11:03:13 +0000
Subject: [PATCH 23/24] back to initial runner to test porting of package

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1ece379701..29d436e68b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -209,7 +209,7 @@ be-2-evs-linux:
     - .test-job-linux
     - .rules-merge-request
   tags:
-    - test-fhg-linux-runner2
+    - test-fhg-linux-runner1
   stage: test
   script:
     - mkdir build
-- 
GitLab


From 218d4ba6cfe80ab51f585ac78221c3e298f20664 Mon Sep 17 00:00:00 2001
From: kiene <jan.kiene@iis.fraunhofer.de>
Date: Thu, 1 Sep 2022 12:57:14 +0000
Subject: [PATCH 24/24] Update .gitlab-ci.yml file

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 29d436e68b..f812488954 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -207,9 +207,9 @@ codec-smoke-test:
 be-2-evs-linux:
   extends:
     - .test-job-linux
-    - .rules-merge-request
+    - .rules-main-push
   tags:
-    - test-fhg-linux-runner1
+    - be-2-evs-temp
   stage: test
   script:
     - mkdir build
-- 
GitLab