Commit ba27bddf authored by Jan Kiene's avatar Jan Kiene
Browse files

Merge branch 'ci/split-complexity-graph-by-levels' into 'main' 

[CI] Split complexity graphs by levels

See merge request !1971
parents 22dbe9c1 6e6c37d7

.gitlab-ci.yml

+19 −2
Original line number Diff line number Diff line
@@ -2407,8 +2407,25 @@ coverage-test-on-main-scheduled: 
  - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html

  - ls

  - public_dir="$CI_JOB_NAME-public"



  # if is needed to catch case when no artifact is there (first run), similarly as above

  - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/;  fi

  - if [[ -d $public_dir ]]; then

  -   mv $public_dir/* wmops/

  # check here if we have the split-by-levels files present - if not, fake them up with the existing global one

  # this is needed for the first run with split graphs on a branch where the global version did run previously

  # NOTE: checking only for level_1 file here as this should already be sufficient

  # NOTE2: also not chechking for RAM for same reason

  -   wmops_all_global="wmops/log_wmops_all.txt"

  -   ram_all_global="wmops/log_ram_all.txt"

  -   if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then

  -     declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw")

  -     for suffix in "${suffixes[@]}"; do

  -       cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt

  -       cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt

  -     done

  -   fi

  - fi



  - ls wmops

  - rm artifacts.zip

  - rm -rf $public_dir
@@ -2417,7 +2434,7 @@ coverage-test-on-main-scheduled: 
  &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory

  - public_dir="$CI_JOB_NAME-public"

  - mkdir $public_dir

  - mv -f wmops/log_*_all.txt ./*.js ${public_dir}/

  - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/

  # move logfiles for links

  - mkdir $public_dir/logs

  # first move logs

ci/complexity_measurements/check_for_changes.py

+13 −10
Original line number Diff line number Diff line
@@ -4,17 +4,20 @@ import sys 




THRESH = 0.01

COLS = [

    [3, 5, 7, 9],  # wmops_all

    [3, 5, 7, 8, 10, 12, 13, 15, 17],  # ram_all

    [3, 5, 7, 9, 11, 13, 15, 17, 19],  # rom_all

]

COLS_WMOPS = [3, 5, 7, 9]

COLS_RAM = [3, 5, 7, 8, 10, 12, 13, 15, 17]

COLS_ROM = [3, 5, 7, 9, 11, 13, 15, 17, 19]





def main(args):

    linewise_logfiles = [args.wmops_logfile, args.ram_logfile, args.rom_logfile]

    linewise_logfiles = [*args.wmops_logfiles, *args.ram_logfiles, args.rom_logfile]

    cols = (

        len(args.wmops_logfiles) * [COLS_WMOPS]

        + len(args.ram_logfiles) * [COLS_RAM]

        + [COLS_ROM]

    )

    changes_found_linewise = any(

        [check_linewise_logfile(f, c) for f, c in zip(linewise_logfiles, COLS)]

        [check_linewise_logfile(f, c) for f, c in zip(linewise_logfiles, cols)]

    )



    if changes_found_linewise:
@@ -59,9 +62,9 @@ def check_linewise_logfile(filepath, cols): 


if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("wmops_logfile")

    parser.add_argument("ram_logfile")

    parser.add_argument("rom_logfile")

    parser.add_argument("--wmops_logfiles", nargs="+")

    parser.add_argument("--ram_logfiles", nargs="+")

    parser.add_argument("--rom_logfile")

    args = parser.parse_args()



    sys.exit(main(args))

ci/complexity_measurements/genWebpageData.py

+57 −40
Original line number Diff line number Diff line
@@ -90,16 +90,16 @@ RUNS_LINE_IDX = { 


DISPLAY_IDS = {

    "wmops": [

        "3xEVS",

        "6xEVS",

        "10xEVS",

        # "3xEVS",

        # "6xEVS",

        # "10xEVS",

        "worst case codec",

        "worst case enc/dec",

        "worst case enc",

        "worst case dec",

    ],

    "rom": [

        "requirementRom",

        # "requirementRom",

        "TotalRomCodecScore",

        "maxPROMEncScore",

        "maxPROMDecScore",
@@ -111,7 +111,7 @@ DISPLAY_IDS = { 
        "maxTROMRendScore",

    ],

    "ram": [

        "requirementRam",

        # "requirementRam",

        "maxTotalRamCodecScore",

        "maxTotalRamEncScore",

        "maxTotalRamDecScore",
@@ -129,9 +129,12 @@ DISPLAY_IDS = { 
}

# the -1's are for reference lines

DISPLAY_LINE_IDX = {

    "wmops": [-1, -1, -1, 9, 3, 5, 7],

    "rom": [-1, 3, 5, 7, 9, 11, 13, 15, 17, 19],

    "ram": [-1, 3, 5, 7, 8, 10, 12, 13, 15, 17],

    # "wmops": [-1, -1, -1, 9, 3, 5, 7],

    # "rom": [-1, 3, 5, 7, 9, 11, 13, 15, 17, 19],

    # "ram": [-1, 3, 5, 7, 8, 10, 12, 13, 15, 17],

    "wmops": [9, 3, 5, 7],

    "rom": [3, 5, 7, 9, 11, 13, 15, 17, 19],

    "ram": [3, 5, 7, 8, 10, 12, 13, 15, 17],

    "wmops_per_op": [1, 2],

}

DISPLAY_LABELS = {
@@ -144,16 +147,16 @@ REF_COLOR_FOR_COMP_BARS = dict( 
)

LINE_COLORS = {

    "wmops": [

        REF_COLORS[0],

        REF_COLORS[1],

        REF_COLORS[2],

        # REF_COLORS[0],

        # REF_COLORS[1],

        # REF_COLORS[2],

        "#0080FF",

        "#FF8000",

        "#CF4B4B",

        "#008040",

    ],

    "rom": [

        REF_COLORS[0],

        # REF_COLORS[0],

        "#FF0000",

        "#FF8000",

        "#FFFF00",
@@ -165,7 +168,7 @@ LINE_COLORS = { 
        "#00FF00",

    ],

    "ram": [

        REF_COLORS[0],

        # REF_COLORS[0],

        "#FF0000",

        "#FF8000",

        "#FFFF00",
@@ -185,7 +188,6 @@ LINE_COLORS = { 
JS_FILE_TEMPLATE = """var {var_name} = {{

    {elem_name}: {{

        description: "{description}",

        direction: -1,

        ticks: [

            {ticks}

            ],
@@ -199,48 +201,57 @@ JS_FILE_TEMPLATE = """var {var_name} = {{ 
}};

"""



# static references commented out for now

FILE_DATA = {

    "wmops": {

        "var_name": "Graphs_WMOPS",

        "var_name": "Graphs_WMOPS{}",

        "elem_name": "wmops_worstcase",

        "description": "Worst Case WMOPS",

        "filename": "graphs_wmops_flc.js",

        "references": {

            "3xEVS": REF_WMOPS_3EVS,

            "6xEVS": REF_WMOPS_6EVS,

            "10xEVS": REF_WMOPS_10EVS,

        },

        "filename": "graphs_wmops_flc{}.js",

        # "references": {

        #     "3xEVS": REF_WMOPS_3EVS,

        #     "6xEVS": REF_WMOPS_6EVS,

        #     "10xEVS": REF_WMOPS_10EVS,

        # },

        "references": {},

    },

    "rom": {

        "var_name": "Graphs_ROM",

        "var_name": "Graphs_ROM{}",

        "elem_name": "rom_worstcase",

        "description": "ROM",

        "filename": "graphs_rom_flc.js",

        "references": {

            "requirementRom": 0,

        },

        "filename": "graphs_rom_flc{}.js",

        # "references": {

        #     "requirementRom": 0,

        # },

        "references": {},

    },

    "ram": {

        "var_name": "Graphs_RAM",

        "var_name": "Graphs_RAM{}",

        "elem_name": "ram_worstcase",

        "description": "Worst Case RAM",

        "filename": "graphs_ram_flc.js",

        "references": {

            "requirementRam": 0,

        },

        "filename": "graphs_ram_flc{}.js",

        # "references": {

        #     "requirementRam": 0,

        # },

        "references": {},

    },

    "wmops_per_op": {

        "var_name": "Graphs_WMOPS_perOP",

        "var_name": "Graphs_WMOPS_perOP{}",

        "elem_name": "wmops_worstcase_per_op",

        "description": "Worst Case WMOPS per OP",

        "filename": "graphs_wmops_flc_perOP.js",

        "filename": "graphs_wmops_flc_perOP{}.js",

        "references": {},

    },

}





def main(

    wmops_log, wmops_per_op_log, rom_log, ram_log, wmops_per_op_log_for_comparison

    wmops_log,

    wmops_per_op_log,

    rom_log,

    ram_log,

    wmops_per_op_log_for_comparison,

    suffix,

):

    FILE_DATA["wmops"]["log_file"] = wmops_log

    FILE_DATA["wmops_per_op"]["log_file"] = wmops_per_op_log
@@ -248,6 +259,10 @@ def main( 
    FILE_DATA["ram"]["log_file"] = ram_log



    for x, data in FILE_DATA.items():

        if data["log_file"] is None:

            print(f"Skipping '{x}' because no logfile given")

            continue



        with open(data["log_file"]) as f:

            log_lines = f.readlines()
@@ -279,14 +294,14 @@ def main( 
        ticks = ",\n".join(ticks)



        js_string = JS_FILE_TEMPLATE.format(

            var_name=data["var_name"],

            var_name=data["var_name"].format(suffix),

            elem_name=data["elem_name"],

            description=data["description"],

            runs=runs,

            displays=displays,

            ticks=ticks,

        )

        with open(data["filename"], "w") as f:

        with open(data["filename"].format(suffix), "w") as f:

            print(js_string, file=f)
@@ -444,11 +459,12 @@ if __name__ == "__main__": 
    parser = argparse.ArgumentParser(

        description="Generate Javascript code for the complexity webpage"

    )

    parser.add_argument("wmops_log")

    parser.add_argument("wmops_per_op_log")

    parser.add_argument("rom_log")

    parser.add_argument("ram_log")

    parser.add_argument("--wmops_log", default=None)

    parser.add_argument("--wmops_per_op_log", default=None)

    parser.add_argument("--rom_log", default=None)

    parser.add_argument("--ram_log", default=None)

    parser.add_argument("--wmops_per_op_log_for_comparison", default=None)

    parser.add_argument("--suffix", default="")



    args = parser.parse_args()
@@ -458,4 +474,5 @@ if __name__ == "__main__": 
        args.rom_log,

        args.ram_log,

        args.wmops_per_op_log_for_comparison,

        args.suffix,

    )

ci/complexity_measurements/getWmops.sh

+46 −16
Original line number Diff line number Diff line
@@ -28,6 +28,8 @@ 
# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and

# the United Nations Convention on Contracts on the International Sales of Goods.



set -euxo pipefail



function usage {

  echo "Usage: $0 \"ivas-format(s)\" \"output-format(s)\" \"mode{full(default)|mem_only}\" \"repo{float(default)|basop}\" [wmops_ref_logfile]"

  exit 1
@@ -94,31 +96,59 @@ fi 
./scripts/IvasBuildAndRunChecks.py $mode_arg_script $repo_arg_script -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc} -C $ivas_format $mode_arg -f ${ep} --oc $output_format

ret_val=$?



# TODO: only split wmops and ram files

./ci/complexity_measurements/split_by_levels.py $(ls ${wmopsFilenameFlc}*.csv)



# for wmops and RAM graphs, split by levels

split_suffixes=("_rate_sw" "_level_1" "_level_2" "_level_3")

for suffix in "${split_suffixes[@]}"; do

  echo "$suffix"



  wmopsFilenameFlc_split="${wmopsFilenameFlc}_WMOPS${suffix}.csv"

  wmopsFilenameFlcLast_split="${wmopsFilenameFlcLast}_WMOPS${suffix}.csv"

  log_wmops_all_split="${destDir}/wmops/log_wmops_all${suffix}.txt"

  latest_wmops_split="${wmopsFilenameFlcDir}/latest_WMOPS${suffix}.csv"



  heapFilenameFlc_split="${wmopsFilenameFlc}_HEAP${suffix}.csv"

  stackFilenameFlc_split="${wmopsFilenameFlc}_STACK${suffix}.csv"

  ramFilenameFlc_split="${wmopsFilenameFlc}_RAM${suffix}.csv"

  ramFilenameFlcLast_split="${wmopsFilenameFlcLast}_RAM${suffix}.csv"

  log_ram_all_split="${destDir}/wmops/log_ram_all${suffix}.txt"



  # get the info on worst-case operating point: WMOPS number, enc-operating mode, dec-operating mode

  ### WMOPS

${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcLast}_WMOPS.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_wmops_all.txt

  ${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc_split} ${wmopsFilenameFlcLast_split} ${commit_sha} ${shortDate} ${fullDate} >>${log_wmops_all_split}



  # create copy of WMOPS csv file with special name to easily get it from artifacts when comparing BASOP to float numbers

cp ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcDir}/latest_WMOPS.csv

  cp ${wmopsFilenameFlc_split} ${latest_wmops_split}



  # get memory info for webpage

  ### RAM

${scriptDir}/mergeNewsletterRam.py ${wmopsFilenameFlc}_HEAP.csv ${wmopsFilenameFlc}_STACK.csv >${wmopsFilenameFlc}_RAM.csv

${scriptDir}/parseNewsletterRam.py ${wmopsFilenameFlc}_HEAP.csv ${wmopsFilenameFlc}_STACK.csv ${wmopsFilenameFlcLast}_RAM.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_ram_all.txt

  ${scriptDir}/mergeNewsletterRam.py ${heapFilenameFlc_split} ${stackFilenameFlc_split} >${ramFilenameFlc_split}

  ${scriptDir}/parseNewsletterRam.py ${heapFilenameFlc_split} ${stackFilenameFlc_split} ${ramFilenameFlcLast_split} ${commit_sha} ${shortDate} ${fullDate} >>${log_ram_all_split}



  # generate javascript code from split log files

  python3 ci/complexity_measurements/genWebpageData.py --wmops_log ${log_wmops_all_split} --ram_log ${log_ram_all_split} --suffix ${suffix}

done



# for ROM and wmops_per_op graphs, no split is done

promFilenameFlc="${wmopsFilenameFlc}_PROM.csv"

tromFilenameFlc="${wmopsFilenameFlc}_TROM.csv"

romFilenameFlc="${wmopsFilenameFlc}_ROM.csv"

romFilenameFlcLast="${wmopsFilenameFlcLast}_ROM.csv"

log_rom_all="${destDir}/wmops/log_rom_all.txt"



### ROM

${scriptDir}/mergeNewsletterRom.py ${wmopsFilenameFlc}_PROM.csv ${wmopsFilenameFlc}_TROM.csv >${wmopsFilenameFlc}_ROM.csv

${scriptDir}/parseNewsletterRom.py ${wmopsFilenameFlc}_PROM.csv ${wmopsFilenameFlc}_TROM.csv ${wmopsFilenameFlcLast}_ROM.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_rom_all.txt

${scriptDir}/mergeNewsletterRom.py ${promFilenameFlc} ${tromFilenameFlc} >${romFilenameFlc}

${scriptDir}/parseNewsletterRom.py ${promFilenameFlc} ${tromFilenameFlc} ${romFilenameFlcLast} ${commit_sha} ${shortDate} ${fullDate} >>${log_rom_all}



# generate javascript code from log files

# pass the ref log for comparison only if arg is given

if [ "$wmops_ref_logfile" != "" ]; then

  python3 ci/complexity_measurements/genWebpageData.py ${destDir}/wmops/log_wmops_all.txt ${wmopsFilenameFlc}_WMOPS.csv ${destDir}/wmops/log_rom_all.txt ${destDir}/wmops/log_ram_all.txt --wmops_per_op_log_for_comparison $wmops_ref_logfile

  python3 ci/complexity_measurements/genWebpageData.py --wmops_per_op_log ${wmopsFilenameFlc}_WMOPS.csv --rom_log ${log_rom_all} --wmops_per_op_log_for_comparison $wmops_ref_logfile

else

  python3 ci/complexity_measurements/genWebpageData.py ${destDir}/wmops/log_wmops_all.txt ${wmopsFilenameFlc}_WMOPS.csv ${destDir}/wmops/log_rom_all.txt ${destDir}/wmops/log_ram_all.txt

  python3 ci/complexity_measurements/genWebpageData.py --wmops_per_op_log ${wmopsFilenameFlc}_WMOPS.csv --rom_log ${log_rom_all}

fi



python3 ${scriptDir}/check_for_changes.py ${destDir}/wmops/log_wmops_all.txt ${destDir}/wmops/log_ram_all.txt ${destDir}/wmops/log_rom_all.txt

python3 ${scriptDir}/check_for_changes.py --wmops_logfiles $(ls ${destDir}/wmops/log_wmops_all*.txt) --ram_logfiles $(ls ${destDir}/wmops/log_ram_all*.txt) --rom_logfile ${destDir}/wmops/log_rom_all.txt

if [ "$?" != "0" ]; then

  ret_val=1

fi

ci/complexity_measurements/index_complexity.html

+175 −451

File changed.

Preview size limit exceeded, changes collapsed.