Dekker1/on-restart-benchmarks
Archived
1
0
Fork 0
Code

Update analysis scripts

Browse Source
This commit is contained in:
Jip J. Dekker 2021-07-09 14:07:26 +10:00
parent ac89bcb3da
commit 9c494bac60
No known key found for this signature in database
GPG Key ID: 517DF4A00618C9C3
2 changed files with 107 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
analyse_chuffed.py
View File

@ -13,6 +13,8 @@ import os
import re import re
import sys import sys
from statistics import stdev
def compute_area(file, time): def compute_area(file, time):
area = -1 area = -1
@ -30,74 +32,124 @@ def compute_area(file, time):
if match: if match:
objectives.append(int(match.group(1))) objectives.append(int(match.group(1)))
continue continue
match = re.match(r"%\stime elapsed:\s(\d+)\sms", line) match = re.match(r"%\stime elapsed:\s(\d+\.\d+)\ss", line)
if match: if match:
times.append(int(match.group(1))) times.append(float(match.group(1)))
continue continue
# Not proven optimal
if len(times) == len(objectives):
times.append(time) times.append(time)
assert len(objectives) > 0 assert len(objectives) > 0
assert len(objectives) + 1 == len(times) assert len(objectives) + 1 == len(times)
area = 0 area = 0
for i in range(len(objectives)): for i in range(len(objectives)):
area += ((times[i + 1] - times[i]) / 1000) * objectives[i] area += (times[i + 1] - times[i]) * objectives[i]
return int(area) return int(area)
folder = sys.argv[1] folder = sys.argv[1]
stats = {} statistics = dict()
for root, dirs, files in os.walk(folder): instances = set()
for config in ["original", "restart"]:
for root, dirs, files in os.walk(folder + "/" + config):
for name in files: for name in files:
if name.endswith(".sol"): if name.endswith(".sol"):
components = name[:-(4)].split(".")
data = components[0]
instances.add(data)
seed = 1 seed = 1
match = re.search(r"\.(\d+)\.sol", name) if len(components) > 1:
if match: assert len(components) == 2
seed = int(match.group(1)) seed = components[1]
if data not in statistics:
statistics[data] = dict()
if config not in statistics[data]:
statistics[data][config] = []
with open(os.path.join(root, name)) as f: with open(os.path.join(root, name)) as f:
contents = f.readlines() contents = f.readlines()
statistics = {} nodes = None
print(contents) solvetime = None
restarts = None
objective = None
for line in contents: for line in contents:
# Nodes # Nodes
match = re.search(r"nodes=(\d+)", line) match = re.search(r"%%%mzn-stat: nodes=(\d+)", line)
if match: if match:
statistics["nodes"] = int(match.group(1)) nodes = int(match.group(1))
continue continue
# Solve time # Solve time
match = re.search(r"solveTime=(\d+\.\d+)", line) match = re.search(r"%%%mzn-stat: solveTime=(\d+\.\d+)", line)
if match: if match:
statistics["search_time"] = int(float(match.group(1)) * 1000) solvetime = float(match.group(1))
continue continue
# Restarts # Restarts
match = re.search(r"restart count:\s+(\d+)", line) match = re.search(r"%%%mzn-stat: restarts=(\d+)", line)
if match: if match:
statistics["restarts"] = int(match.group(1)) restarts = int(match.group(1))
continue continue
for line in contents[::-1]: for line in contents[::-1]:
# Best objective # Best objective
match = re.match(r"objective\s=\s(\d+)", line) match = re.match(r"%%%mzn-stat: objective=(-?\d+)", line)
if match: if match:
statistics["objective"] = int(match.group(1)) objective = int(match.group(1))
break break
# Area # Area
area = compute_area(contents, statistics["search_time"]) area = compute_area(contents, solvetime)
stats[name[:-(4)].replace(".", ",")] = ( statistics[data][config].append(
(
area, area,
statistics["objective"], objective,
statistics["search_time"], solvetime,
statistics["restarts"], restarts,
statistics["nodes"], nodes,
)
) )
sorted_stats = sorted(stats.items()) for data in instances:
a = sorted_stats[0][0][: sorted_stats[0][0].find(",")] for config in ["original", "restart"]:
for key, val in sorted_stats: stats = statistics[data][config]
if key[: key.find(",")] != a: cumulative = stats[0]
print("\n\n") for i in range(1, len(stats)):
a = key[: key.find(",")] cumulative = tuple(map(sum, zip(cumulative, stats[i])))
print("%s,%s" % (key, ",".join([v.__str__() for v in val]))) avg = tuple([x / len(stats) for x in cumulative])
dev = stdev([x[1] for x in stats]) if len(stats) > 1 else 0
# (avg area, avg objective, stdev objective)
statistics[data][config] = (avg[0], avg[1], dev)
exit(1) # Print header
print(
"""
\\begin{tabular}{l|rr|rr|rr}
\\toprule
& \multicolumn{2}{c|}{Chuffed} & \multicolumn{2}{c|}{Chuffed Restart} \\\\
Instance & $\intobj$ & $\minobj$ & $\intobj$ & $\minobj$ \\\\
\midrule
"""
)
sorted_instances = sorted(instances)
for data in sorted_instances:
print(f"{data}", end="")
for config in ["original", "restart"]:
print(
f" & {int(statistics[data][config][0] / 1000) }k & {int(statistics[data][config][1])}",
end="",
)
if statistics[data][config][2] != 0:
print("^{", end="")
print(
int(statistics[data][config][2] / statistics[data][config][1] * 100),
# int(statistics[data][config][2]),
end="",
)
print("}", end="")
print(" \\\\")
# Print footer
print("\n\\bottomrule\n\end{tabular}")

1
analyse_gecode.py
View File

@ -8,7 +8,6 @@ area which can be found by adding
'constraint trace("% init_area = \(ub(objective));\n", true);' 'constraint trace("% init_area = \(ub(objective));\n", true);'
to the model in question. to the model in question.
""" """
import csv
import os import os
import re import re
import sys import sys