Commit d5d0fb12 authored by vaclav's avatar vaclav
Browse files

- Merge remote-tracking branch 'remotes/origin/main' into OBJ_EDITING_formal_impr

parents 5ec905e5 78d0014c

.gitlab-ci.yml

+11 −5
Original line number Diff line number Diff line
@@ -228,12 +228,12 @@ stages: 
    - if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi



    # run encoder and decoder with 20ms renderer framesize first, use reference creation mode

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec || exit_code20=$?

    # for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option

    # set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active

    # we are only interested in runtime errors from the sanitizers and ignore the diffs

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only --abs_tol 100000 || exit_code5=$?

    - python3 -m pytest tests/codec_be_on_mr_nonselection $USE_LTV --param_file $SELF_TEST_PRM_FILE -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only --abs_tol 100000 || exit_code10=$?



    - if [ $exit_code20 -ne 0 ] || [ $exit_code10 -ne 0 ] || [ $exit_code5 -ne 0 ]; then exit 1; fi
@@ -570,6 +570,7 @@ codec-msan: 
    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with address-sanitizer binaries
@@ -580,6 +581,7 @@ codec-asan: 
    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  <<: *sanitizer-selftest-anchor



# code selftest testvectors with undefined-behaviour-sanitizer binaries
@@ -592,6 +594,7 @@ codec-usan: 
    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_STV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test.prm"

    - USE_LTV=""

  <<: *sanitizer-selftest-anchor



# compare bit-exactness between 5ms and 20 on the branch
@@ -1612,11 +1615,11 @@ test-long-self-test: 
    # create references

    - exit_code_ref=0

    - testcase_timeout=300

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout || exit_code_ref=$?



    ### run pytest self-test using long test vectors

    - exit_code=0

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout || exit_code=$?

    - zero_errors=$(cat report-junit-ltv.xml | grep -c 'errors="0"') || true



    - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
@@ -1654,6 +1657,7 @@ ltv-msan: 
    - CLANG_NUM=1

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"

    - USE_LTV="--use_ltv"

  <<: *sanitizer-selftest-anchor



# code selftest long testvectors with address-sanitizer binaries
@@ -1671,6 +1675,7 @@ ltv-asan: 
    - CLANG_NUM=2

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"

    - USE_LTV="--use_ltv"

  <<: *sanitizer-selftest-anchor



# code selftest long testvectors with undefined-behaviour-sanitizer binaries
@@ -1688,6 +1693,7 @@ ltv-usan: 
    - CLANG_NUM=3

    - SELFTEST_SANITY_TIMEOUT=$TESTCASE_TIMEOUT_LTV_SANITIZERS

    - SELF_TEST_PRM_FILE="scripts/config/self_test_ltv.prm"

    - USE_LTV="--use_ltv"

  <<: *sanitizer-selftest-anchor



.sanitizer-test-template:

apps/renderer.c

+0 −14
Original line number Diff line number Diff line
@@ -169,10 +169,8 @@ typedef struct 
    char customHrtfFilePath[RENDERER_MAX_CLI_ARG_LENGTH];

    char renderConfigFilePath[RENDERER_MAX_CLI_ARG_LENGTH];

    int8_t orientation_tracking;

#ifdef FIX_1135_EXT_RENDERER_HANDLES

    int16_t Opt_Headrotation;

    int16_t Opt_ExternalOrientation;

#endif

    int16_t nonDiegeticPan;

    float nonDiegeticPanGain;

    IVAS_REND_COMPLEXITY_LEVEL complexityLevel;
@@ -886,11 +884,7 @@ int main( 


    const int16_t frameSize_smpls = (int16_t) ( ( args.render_framesize ) * args.sampleRate * 5 / ( 1000 ) );



#ifdef FIX_1135_EXT_RENDERER_HANDLES

    if ( ( error = IVAS_REND_Open( &hIvasRend, args.sampleRate, args.outConfig.audioConfig, !isEmptyString( args.customHrtfFilePath ), args.nonDiegeticPan, args.nonDiegeticPanGain, args.Opt_Headrotation, args.Opt_ExternalOrientation, (int16_t) args.render_framesize ) ) != IVAS_ERR_OK )

#else

    if ( ( error = IVAS_REND_Open( &hIvasRend, args.sampleRate, args.outConfig.audioConfig, !isEmptyString( args.customHrtfFilePath ), args.nonDiegeticPan, args.nonDiegeticPanGain, (int16_t) args.render_framesize ) ) != IVAS_ERR_OK )

#endif

    {

        fprintf( stderr, "Error opening renderer handle: %s\n", ivas_error_to_string( error ) );

        exit( -1 );
@@ -2518,10 +2512,8 @@ static CmdlnArgs defaultArgs( 
    clearString( args.renderConfigFilePath );

    clearString( args.externalOrientationFilePath );



#ifdef FIX_1135_EXT_RENDERER_HANDLES

    args.Opt_Headrotation = 0;

    args.Opt_ExternalOrientation = 0;

#endif

    args.orientation_tracking = IVAS_HEAD_ORIENT_TRK_NONE;



    args.nonDiegeticPan = 0;
@@ -2611,9 +2603,7 @@ static void parseOption( 
            break;

        case CmdLnOptionId_trajFile:

            assert( numOptionValues == 1 );

#ifdef FIX_1135_EXT_RENDERER_HANDLES

            args->Opt_Headrotation = 1;

#endif

            strncpy( args->headRotationFilePath, optionValues[0], RENDERER_MAX_CLI_ARG_LENGTH - 1 );

            break;

        case CmdLnOptionId_outputMetadata:
@@ -2634,9 +2624,7 @@ static void parseOption( 
            break;

        case CmdLnOptionId_exteriorOrientationFile:

            assert( numOptionValues == 1 );

#ifdef FIX_1135_EXT_RENDERER_HANDLES

            args->Opt_ExternalOrientation = 1;

#endif

            strncpy( args->externalOrientationFilePath, optionValues[0], RENDERER_MAX_CLI_ARG_LENGTH - 1 );

            break;

        case CmdLnOptionId_customHrtfFile:
@@ -2761,12 +2749,10 @@ static CmdlnArgs parseCmdlnArgs( 
        exit( -1 ); /* Error printout handled by failing function */

    }



#ifdef FIX_1135_EXT_RENDERER_HANDLES

    if ( args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_CODED || args.outConfig.audioConfig == IVAS_AUDIO_CONFIG_BINAURAL_SPLIT_PCM )

    {

        args.Opt_Headrotation = 1;

    }

#endif



    return args;

}

ci/basop-pages/basop_index.html

+5 −0
Original line number Diff line number Diff line
@@ -15,4 +15,9 @@ 
    <li><a href="ivas-pytest-compare_ref-long-enc-lev+10-index.html">ivas-pytest-compare_ref-long-enc-lev+10</a></li>

    <li><a href="ivas-pytest-compare_ref-long-enc-lev-10-index.html">ivas-pytest-compare_ref-long-enc-lev-10</a></li>

  </ul>



  <h3>Complexity Reports</h3>



  {}



</body>

ci/complexity_measurements/index_complexity.html

+8 −8
Original line number Diff line number Diff line
@@ -58,14 +58,14 @@ 


  <div id="menu">

    <ul>

      <li><a href="#sec:graph-wmops">WMOPS Float</a>

      <li><a href="#sec:graph-wmops">WMOPS</a>

      <span class="symbols">

      <span class="trafficlight"><span id="wmops_tl_l">&#x25cf;</span><span id="wmops_tl_c">&#x25cf;</span><span id="wmops_tl_r">&#x25cf;</span></span>

      <span class="trend"><span id="wmops_trend">-</span></span>

      </span>

      </li>



      <li><a href="#sec:graph-wmops_per_op">WMOPS Float per OP</a></li>

      <li><a href="#sec:graph-wmops_per_op">WMOPS per OP</a></li>



      <!--

      <li><a href="#sec:graph-wmops-basop">WMOPS BASOP</a>
@@ -80,7 +80,7 @@ 
      -->



        

      <li><a href="#sec:graph-ram">RAM Float</a>

      <li><a href="#sec:graph-ram">RAM</a>

      <span class="symbols">

      <span class="trafficlight"><span id="ram_tl_l">&#x25cf;</span><span id="ram_tl_c">&#x25cf;</span><span id="ram_tl_r">&#x25cf;</span></span>

      <span class="trend"><span id="ram_trend">-</span></span>
@@ -96,7 +96,7 @@ 
      </li>

      -->



      <li><a href="#sec:graph-rom">ROM Float</a>

      <li><a href="#sec:graph-rom">ROM</a>

      <span class="symbols">

      <span class="trafficlight"><span id="rom_tl_l">&#x25cf;</span><span id="rom_tl_c">&#x25cf;</span><span id="rom_tl_r">&#x25cf;</span></span>

      <span class="trend"><span id="rom_trend">-</span></span>
@@ -107,7 +107,7 @@ 
  </div>



  <div id="content">

    <h1 id="sec:graph-wmops">IVAS FORMAT - Worst Case WMOPS Performance (Float, native SR)</h1>

    <h1 id="sec:graph-wmops">IVAS FORMAT - Worst Case WMOPS Performance</h1>



  <div class="graph-container">

  <div id="wmops-graph"></div>
@@ -129,7 +129,7 @@ 
  <hr />



    <h1 id="sec:graph-wmops_per_op">IVAS FORMAT - Worst Case WMOPS Performance

        per Operating Point (Float, native SR)</h1>

        per Operating Point</h1>



  <div class="graph-container">

  <div id="wmops_per_op-graph"></div>
@@ -192,7 +192,7 @@ 
  <hr />

  -->



  <h1 id="sec:graph-ram">IVAS FORMAT - Worst Case RAM Demand (Float)</h1>

  <h1 id="sec:graph-ram">IVAS FORMAT - Worst Case RAM Demand</h1>



  <div class="graph-container">

  <div id="ram-graph"></div>
@@ -251,7 +251,7 @@ 
  <hr />

  -->



  <h1 id="sec:graph-rom">IVAS FORMAT - Worst Case ROM Demand (Float)</h1>

  <h1 id="sec:graph-rom">IVAS FORMAT - Worst Case ROM Demand</h1>

  

  <div class="graph-container">

  <div id="rom-graph"></div>

ci/setup_pages.py

+20 −18
Original line number Diff line number Diff line
@@ -42,14 +42,15 @@ JOBS_FLOAT_REPO = { 
    # "timeless" jobs (not complexity)

    "coverage-test-on-main-scheduled": "Coverage",

}

JOBS_BASOP_REPO = [

    "ivas-pytest-compare_ref-long-dec",

    "ivas-pytest-compare_ref-long-dec-lev+10",

    "ivas-pytest-compare_ref-long-dec-lev-10",

    "ivas-pytest-compare_ref-long-enc",

    "ivas-pytest-compare_ref-long-enc-lev+10",

    "ivas-pytest-compare_ref-long-enc-lev-10",    

]

JOBS_BASOP_REPO = {

    "ivas-pytest-compare_ref-long-dec": "Pytest decoder compare to ref LTV",

    "ivas-pytest-compare_ref-long-dec-lev+10": "Pytest decoder compare to ref LTV +10dB",

    "ivas-pytest-compare_ref-long-dec-lev-10": "Pytest decoder compare to ref LTV -10dB",

    "ivas-pytest-compare_ref-long-enc": "Pytest encoder compare to ref LTV",

    "ivas-pytest-compare_ref-long-enc-lev+10": "Pytest encoder compare to ref LTV +10dB",

    "ivas-pytest-compare_ref-long-enc-lev-10": "Pytest encoder compare to ref LTV -10dB",

    "complexity-stereo-in-stereo-out": "Stereo in, Stereo out",

}



JOBS_FOR_PROJECT_ID = {

    PROJECT_ID_FLOAT_REPO: JOBS_FLOAT_REPO,
@@ -76,16 +77,12 @@ def main(): 
        sys.exit(1)



    index_html = PUBLIC_FOLDER.joinpath("index.html")

    if project_id == PROJECT_ID_FLOAT_REPO:

        create_landing_page_float_repo(jobs, index_html)

    elif project_id == PROJECT_ID_BASOP_REPO:

        src = pathlib.Path("ci/basop-pages/basop_index.html").absolute()

        shutil.move(src, index_html)

    create_landing_page(jobs, index_html, project_id)



    sys.exit(0)





def create_landing_page_float_repo(jobs, index_html):

def create_landing_page(jobs, index_html, project_id):

    # dynamically create the complexity links on the landing page

    link_html = list()

    link_html = ["<ul>"]
@@ -97,8 +94,13 @@ def create_landing_page_float_repo(jobs, index_html): 
    link_html.append("</ul>")

    link_html_text = "\n".join(link_html)



    if project_id == PROJECT_ID_FLOAT_REPO:

        index_template = "index-pages.html"

    elif project_id == PROJECT_ID_BASOP_REPO:

        index_template = "basop-pages/basop_index.html"



    index_pages_tmpl_path = (

        pathlib.Path(__file__).parent.joinpath("index-pages.html").absolute()

        pathlib.Path(__file__).parent.joinpath(index_template).absolute()

    )



    with open(index_pages_tmpl_path) as f:
@@ -130,7 +132,7 @@ def get_artifacts_for_jobs_and_return_num_failed( 
        print(f"{job_id} - {job}")

        try:

            with TemporaryDirectory() as tmp_dir:

                curl_for_artifacts(job_id, tmp_dir)

                curl_for_artifacts(job_id, project_id, tmp_dir)



                tmp_dir = pathlib.Path(tmp_dir)
@@ -147,12 +149,12 @@ def get_artifacts_for_jobs_and_return_num_failed( 
    return failed_count





def curl_for_artifacts(job_id: int, exdir: str):

def curl_for_artifacts(job_id: int, project_id: int, exdir: str):

    cmd = [

        "curl",

        "--request",

        "GET",

        API_URL_BASE.format(os.environ["CI_PROJECT_ID"]) + f"/{job_id}/artifacts",

        API_URL_BASE.format(project_id) + f"/{job_id}/artifacts",

        "--output",

        ARTIFACTS,

    ]