diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 56420ae02e377767c88b7f388b6ae77d379474bf..f812488954584e092f33b944bff66b1bd38be571 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 variables:
   TESTV_DIR: "/usr/local/testv"
   BUILD_OUTPUT: "build_output.txt"
+  EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"
 
 
 # This sets when pipelines are created. Jobs have more specific rules to restrict them.
@@ -202,6 +203,30 @@ codec-smoke-test:
     expose_as: 'Smoke test results'
 
 
+# check bitexactness to EVS
+be-2-evs-linux:
+  extends:
+    - .test-job-linux
+    - .rules-main-push
+  tags:
+    - be-2-evs-temp
+  stage: test
+  script:
+    - mkdir build
+    - cd build
+    - cmake ..
+    - make -j
+    - cd ..
+
+    # copy over to never change the testvector dir
+    - cp -r $EVS_BE_TEST_DIR ./evs_be_test
+    - cp build/IVAS_cod ./evs_be_test/bin/EVS_cod
+    - cp build/IVAS_dec ./evs_be_test/bin/EVS_dec
+
+    - cd evs_be_test
+    - python3 ../ci/run_evs_be_test.py
+
+
 # code selftest testvectors with memory-sanitizer binaries
 msan-on-merge-request-linux:
   extends: 
diff --git a/ci/run_evs_be_test.py b/ci/run_evs_be_test.py
new file mode 100755
index 0000000000000000000000000000000000000000..9fa64877f2ed44034c69bad4715e71aa3d40fa07
--- /dev/null
+++ b/ci/run_evs_be_test.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+import subprocess
+import pathlib
+import sys
+import concurrent.futures
+from threading import Lock
+
+
+README_FILES_PARALLEL = [
+    "Readme_AMRWB_IO_enc.txt",
+    "Readme_AMRWB_IO_dec.txt",
+    "Readme_EVS_enc.txt",
+    "Readme_EVS_dec.txt",
+]
+README_FILES_JBM = ["Readme_JBM_dec.txt"]
+README_FILES = README_FILES_PARALLEL + README_FILES_JBM
+BINARY_PATHS = ["./bin/EVS_cod", "./bin/EVS_dec"]
+FOLDER_PATHS = ["testv"]
+BIN_PATHS = BINARY_PATHS * 2
+
+def main():
+
+    if not environment_is_correct():
+        return 1
+
+    result_dict = dict()
+    # run first part in parallel
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        executor.map(
+            run_file, README_FILES_PARALLEL, BIN_PATHS, [result_dict] * len(README_FILES_PARALLEL)
+        )
+
+    # JBM test can not run concurrently with the others
+    run_file(README_FILES_JBM[0], BINARY_PATHS[1], result_dict)
+
+    return analyze_results(result_dict)
+
+
+def analyze_results(result_dict):
+    ret = 0
+
+    for filename, ret_code in result_dict.items():
+        if ret_code != 0:
+            print(f"========= Test for {filename} failed! See log below: ==========")
+            with open(filename.replace("Readme", "Log")) as f:
+                print(f.read())
+            ret = 1
+
+    return ret
+
+
+def run_file(filename: str, bin_path: str, result_dict: dict):
+    ret_code = subprocess.call(["bash", filename, bin_path])
+    with Lock():
+        result_dict[filename] = ret_code
+
+
+def environment_is_correct():
+    """
+    Check that the folder with the test resources is set up correctly:
+    - all Readme files there
+    - EVS binaries available in bin/
+    - testv and switchPaths folder exist - Content is not checked, though
+    """
+    ret = True
+
+    for path in README_FILES + BINARY_PATHS + FOLDER_PATHS:
+        if not pathlib.Path(path).exists():
+            print(f"Environment setup is incorrect - {path} not found.")
+            ret = False
+
+    return ret
+
+
+if __name__ == "__main__":
+    sys.exit(main())