Commit 63a9c4d4 authored by Jan Kiene's avatar Jan Kiene Committed by Manuel Jander
Browse files

adapt CI config to new versions of scripts

parent 5e63521c

.gitlab-ci.yml

+35 −56
Original line number Diff line number Diff line
@@ -49,9 +49,9 @@ variables: 
  FAILED_TESTCASES_LIST: "failed-testcases.txt"

  ERRORS_TESTCASES_LIST: "errors-testcases.txt"

  PYTEST_CACHE_ARTIFACT: "pytest_cache.zip"

  FLOAT_REF_COMMIT_FILE: "float-ref-git-sha.txt"

  CUT_COMMIT_FILE: "CuT-git-sha.txt"

  MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha.txt"

  FLOAT_REF_COMMIT_FILE: "float-ref-git-sha"

  CUT_COMMIT_FILE: "CuT-git-sha"

  MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha"

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."

    value: 'default'
@@ -169,10 +169,10 @@ stages: 
  - current_commit_sha=$(git rev-parse HEAD)

  ### build reference binaries

  - git checkout $FLOAT_REF_BRANCH

  - git pull origin $FLOAT_REF_BRANCH

  - git pull

  - *activate-debug-mode-info-if-set

  - make clean

  - make -j >> /dev/null

  - make -j

  - mv ./IVAS_cod ./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY

  - mv ./IVAS_dec ./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY

  - mv ./IVAS_rend ./IVAS_rend_ref
@@ -185,10 +185,10 @@ stages: 
  - current_commit_sha=$(git rev-parse HEAD)

  ### build merge target binaries

  - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

  - git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

  - git pull

  - *activate-debug-mode-info-if-set

  - make clean

  - make -j >> /dev/null

  - make -j

  - mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY

  - mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY

  - mv ./IVAS_rend ./IVAS_rend_merge_target
@@ -203,7 +203,7 @@ stages: 
### build dut binaries

  - *activate-debug-mode-info-if-set

  - make clean

  - make -j >> /dev/null

  - make -j



.build-and-create-float-ref-outputs: &build-and-create-float-ref-outputs

  - *build-float-ref-and-dut-binaries
@@ -233,7 +233,7 @@ stages: 
  - git fetch

  - git restore . # Just as a precaution

  - git checkout $BASOP_CI_BRANCH_PC_REPO

  - git pull origin $BASOP_CI_BRANCH_PC_REPO

  - git pull

  - cd -

  - cp -r $SCRIPTS_DIR/ci .

  - cp -r $SCRIPTS_DIR/scripts .
@@ -375,6 +375,7 @@ stages: 
    MERGED_CSV_ARTIFACT_NAME: "$CI_JOB_NAME--merged_csv--$CI_JOB_ID.csv"

    PAGES_HTML_ARTIFACT_NAME: "$CI_JOB_NAME-index.html"

    SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"

    SUMMARY_HTML_ARTIFACT_SPLIT: "summary_split_$CI_JOB_NAME.html"

    IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"

    IMAGES_ARTIFACT_SPLIT: "images_split_$CI_JOB_NAME"

  script:
@@ -398,12 +399,11 @@ stages: 
    -   BUILD_WITH_DEBUG_MODE_INFO="true"

    - fi



    - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")

    - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"

    - summary_args="MLD DIFF SSNR ODG"

    - REPORT_ARG=""

    - comp_args="--mld --ssnr --odg"



    - MEASURES_FOR_REPORT="MLD MAX_ABS_DIFF MIN_SSNR MIN_ODG"

    - if [ "$ENCODER_TEST" = "true" ]; then comp_args="${comp_args} --enc_stats"; fi

    - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; summary_args="${summary_args} DELTA_ODG"; REPORT_ARG="--delta_odg"; fi

    - if [ "$DELTA_ODG" = "true" ]; then comp_args="${comp_args} --odg_bin"; MEASURES_FOR_REPORT="$MEASURES_FOR_REPORT DELTA_ODG"; fi

    - if [ "$SPLIT_COMPARISON" = "true" ]; then comp_args="${comp_args} --split-comparison"; export PYTEST_ADDOPTS="-k \"not JBM and stereo and at\""; fi



    - *build-and-create-float-ref-outputs
@@ -421,30 +421,20 @@ stages: 
    - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true



    ### create histograms

    - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME $REPORT_ARG

    # if split comparison is done, we need to create two pages, one for the "whole file" values and one for the "split" values

    - python3 scripts/parse_xml_report.py report-junit.xml $CSV_ARTIFACT_NAME --split-csv-file $CSV_ARTIFACT_SPLIT



    # first for "whole" files comparison

    - python3 scripts/create_histograms.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $MEASURES_FOR_REPORT



    - if [ "$SPLIT_COMPARISON" = "true" ]; then

    # use -v flag ("inverse") to also have the header in the output file

    -   grep -v "_whole" $CSV_ARTIFACT_NAME > $CSV_ARTIFACT_SPLIT

    -   cp $CSV_ARTIFACT_NAME tmp.csv

    -   grep -v "_split" tmp.csv > $CSV_ARTIFACT_NAME

    -   mkdir $IMAGES_ARTIFACT_SPLIT

    -   for MEASURE in $summary_args;do

    -     python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".csv $IMAGES_ARTIFACT_SPLIT/summary_"$MEASURE".png --measure $MEASURE

    -   done

    -   python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $summary_args

    -   python3 scripts/create_histograms.py $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT

    -   python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_SPLIT $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_SPLIT --measures $MEASURES_FOR_REPORT

    - else

    # touch files to suppress warning for missing artifacts

    -   touch $CSV_ARTIFACT_SPLIT $IMAGES_ARTIFACT_SPLIT

    - fi



    # this part creates the "whole file" histograms

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in $summary_args;do

    -   python3 scripts/create_histogram_summary.py $CSV_ARTIFACT_NAME $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE;

    - done

    - python3 ci/basop-pages/create_summary_page.py $SUMMARY_HTML_ARTIFACT_NAME $CI_JOB_ID $CI_JOB_NAME $IMAGES_ARTIFACT_NAME --measures $summary_args



    - if [ $USE_LTV -eq 1 ] && [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then

    -    id_previous=$(python3 ci/get_id_of_last_job_occurence.py $CI_DEFAULT_BRANCH $CI_JOB_NAME $CI_PROJECT_ID)

    -    echo "Job ID from variables - $CI_JOB_ID, Job ID from script - $id_previous"
@@ -486,6 +476,7 @@ stages: 
      - $CSV_ARTIFACT_SPLIT

      - $MERGED_CSV_ARTIFACT_NAME

      - $SUMMARY_HTML_ARTIFACT_NAME

      - $SUMMARY_HTML_ARTIFACT_SPLIT

      - $IMAGES_ARTIFACT_NAME

      - $IMAGES_ARTIFACT_SPLIT

    expose_as: "pytest compare results"
@@ -536,7 +527,7 @@ stages: 
    - *build-float-ref-binaries

    - *build-merge-target-binaries

    - make clean

    - make -j >> /dev/null

    - make -j

    - *check-up-to-date-in-comparison-jobs



    - exit_code_target=0
@@ -544,7 +535,7 @@ stages: 


    - exit_code=0

    - rm -rf .pytest_cache || true

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?



    - if [ $exit_code -ne 0 ]; then

    -   exit_code=$EXIT_CODE_NON_BE
@@ -663,16 +654,13 @@ stages: 
    - echo $CI_MERGE_REQUEST_TITLE > tmp.txt

    - allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) || true



    - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")

    - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"



    ### run branch first

    # this per default builds the branch and the reference and creates the reference outputs

    - *build-and-create-float-ref-outputs

    - *check-up-to-date-in-comparison-jobs

    # need to restore cache again

    - *overwrite-pytest-cache-with-artifact

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH $comp_args --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH
@@ -685,12 +673,12 @@ stages: 


    ### run main now

    - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

    - git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME

    - git pull

    - make clean

    - make -j >> /dev/null

    - make -j

    # need to restore cache again

    - *overwrite-pytest-cache-with-artifact

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN $comp_args --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN



    ### compare the two csv files for regressions
@@ -786,7 +774,7 @@ stages: 
    -   make_args="$make_args IGNORELIST=1"

    - fi

    - make clean

    - make -j $make_args >> /dev/null

    - make -j $make_args

    - testcase_timeout_arg="--testcase_timeout $TESTCASE_TIMEOUT_LTV_SANITIZERS"

    # disable per-testcase timeout for msan to evaluate what is going on that it takes so long

    - if [[ $CLANG_NUM = 1 ]]; then
@@ -828,13 +816,10 @@ stages: 
    - fi

    - *build-float-ref-and-dut-binaries



    - INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")

    - comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"



    ### run pytest

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml $comp_args -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-ref.html --self-contained-html --junit-xml=report-junit-ref.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $REF_ENCODER_PATH --dut_decoder_path $REF_DECODER_PATH  --compare_to_input || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=report-dut.html --self-contained-html --junit-xml=report-junit-dut.xml --mld --ssnr --odg -n auto --testcase_timeout $testcase_timeout --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH  --compare_to_input || exit_code=$?

    - zero_errors_ref=$(cat report-junit-ref.xml | grep -c 'errors="0"') || true

    - zero_errors_dut=$(cat report-junit-dut.xml | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py report-junit-ref.xml report-ref.csv
@@ -1569,7 +1554,7 @@ coverage-test-on-main-scheduled: 
    - *copy-ltv-files-to-testv-dir

    - *build-float-ref-binaries

    # Build DuT binaries with GCOV

    - make clean >> /dev/null

    - make clean

    - make GCOV=1 -j

    - cp IVAS_rend IVAS_rend_ref # Copy to ensure instrumented renderer is run in the first pytest call
@@ -1627,7 +1612,7 @@ be-2-evs-26444: 
    - *print-common-info

    - *update-scripts-repo

    - sed -i".bak" "s/\(#define EVS_FLOAT\)/\/\/\1/" lib_com/options.h

    - make -j  >> /dev/null

    - make -j



    # copy over to never change the testvector dir

    - cp -r $EVS_BE_TEST_DIR_BASOP ./evs_be_test
@@ -1691,7 +1676,7 @@ voip-be-on-merge-request: 
  script:

    - *print-common-info

    - make clean

    - make -j >> /dev/null

    - make -j

    - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py
@@ -1780,19 +1765,13 @@ voip-be-on-merge-request: 
    - *update-ltv-repo

    - *build-float-ref-and-dut-binaries

    - *complexity-measurements-setup

    # delete previous jobs logfiles if present (-f flag ensures return calue of 0 even in first run where this folder is not present)

    - rm -rf COMPLEXITY/logs

    - which coan

  allow_failure:

    exit_codes:

      - 123

  artifacts:

    name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"

    when: always

    expire_in: 2 week

    paths:

      - $CI_JOB_NAME-public

      - COMPLEXITY/logs



complexity-stereo-in-stereo-out:

  extends: