Dekker1/on-restart-benchmarks
Archived
1
0
Fork 0
Code

Update analysis script for Gecode

Browse Source
This commit is contained in:
Jip J. Dekker 2021-06-21 10:17:16 +10:00
parent d23f68f293
commit 9071b2aa0f
No known key found for this signature in database
GPG Key ID: 517DF4A00618C9C3
1 changed files with 105 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
analyse_gecode.py
View File

@ -13,15 +13,15 @@ import os
import re import re
import sys import sys
from statistics import stdev
def compute_area(file): def compute_area(file):
area = -1
objectives = [] objectives = []
times = [0] times = [0]
timeout = -1
objectives.append(0) objectives.append(0)
for line in contents: for line in file:
# match = re.match(r'%\sinit_area\s=\s(\d+)', line)
# match = re.match(r'%\sinit_area\s=\s(\d+)', line) # match = re.match(r'%\sinit_area\s=\s(\d+)', line)
# if match: # if match:
# objectives.append(int(match[1])) # objectives.append(int(match[1]))
@ -30,7 +30,7 @@ def compute_area(file):
if match: if match:
objectives.append(int(match.group(1))) objectives.append(int(match.group(1)))
continue continue
match = re.match(r"%\stime elapsed:\s(\d\.\d+)\ss", line) match = re.match(r"%\stime elapsed:\s(\d+\.\d+)\ss", line)
if match: if match:
times.append(float(match.group(1))) times.append(float(match.group(1)))
continue continue
@ -43,65 +43,111 @@ def compute_area(file):
assert len(objectives) + 1 == len(times) assert len(objectives) + 1 == len(times)
area = 0 area = 0
for i in range(len(objectives)): for i in range(len(objectives)):
area += ((times[i + 1] - times[i]) / 1000) * objectives[i] area += (times[i + 1] - times[i]) * objectives[i]
return int(area) return int(area)
folder = sys.argv[1] folder = sys.argv[1]
statistics = {} statistics = dict()
for root, dirs, files in os.walk(folder): instances = set()
for name in files: for config in ["original", "restart", "replay"]:
if name.endswith(".sol"): for root, dirs, files in os.walk(folder + "/" + config):
seed = 1 for name in files:
match = re.search(r"\.(\d+)\.sol", name) if name.endswith(".sol"):
if match: components = name[:-(4)].split(".")
seed = int(match.group(1)) data = components[0]
with open(os.path.join(root, name)) as f: instances.add(data)
contents = f.readlines() seed = 1
# Area if len(components) > 1:
area = compute_area(contents) assert len(components) == 2
seed = components[1]
objective = "UNSAT" if data not in statistics:
for line in contents[::-1]: statistics[data] = dict()
# Best objective if config not in statistics[data]:
match = re.match(r"objective\s=\s(\d+)", line) statistics[data][config] = []
if match:
objective = int(match.group(1))
break
nodes = -1 with open(os.path.join(root, name)) as f:
solvetime = -1 contents = f.readlines()
restarts = -1 # Area
for line in contents: area = compute_area(contents)
# Evaluation time
match = re.search(r"copies:\s+(\d+)", line) objective = "UNSAT"
if match: for line in contents[::-1]:
nodes = int(match.group(1)) # Best objective
continue match = re.match(r"objective\s=\s(\d+)", line)
# Solve time if match:
match = re.search(r"solveTime=(\d+(.\d+)?)", line) objective = int(match.group(1))
if match: break
solvetime = float(match.group(1))
continue nodes = -1
# Restarts solvetime = -1
match = re.search(r"restarts=(\d+)", line) restarts = -1
if match: for line in contents:
restarts = int(match.group(1)) # Evaluation time
continue match = re.search(r"copies:\s+(\d+)", line)
statistics[name[:-(4)].replace(".", ",")] = ( if match:
area, nodes = int(match.group(1))
objective, continue
solvetime, # Solve time
restarts, match = re.search(r"solveTime=(\d+(.\d+)?)", line)
nodes, if match:
solvetime = float(match.group(1))
continue
# Restarts
match = re.search(r"restarts=(\d+)", line)
if match:
restarts = int(match.group(1))
continue
statistics[data][config].append(
(
area,
objective,
solvetime,
restarts,
nodes,
)
)
for data in instances:
for config in ["original", "restart", "replay"]:
stats = statistics[data][config]
cumulative = stats[0]
for i in range(1, len(stats)):
cumulative = tuple(map(sum, zip(cumulative, stats[i])))
avg = tuple([x / len(stats) for x in cumulative])
dev = stdev([x[1] for x in stats]) if len(stats) > 1 else 0
# (avg area, avg objective, stdev objective)
statistics[data][config] = (avg[0], avg[1], dev)
# Print header
print(
"""
\\begin{tabular}{l|rr|rr|rr}
\\toprule
& \multicolumn{2}{c|}{Gecode} & \multicolumn{2}{c|}{Gecode Restart} & \multicolumn{2}{c}{Gecode Replay}\\\\
Instance & $\intobj$ & $\minobj$ & $\intobj$ & $\minobj$ & $\intobj$ & $\minobj$ \\\\
\midrule
"""
)
sorted_instances = sorted(instances)
for data in sorted_instances:
print(f"{data}", end="")
for config in ["original", "restart", "replay"]:
print(
f" & {int(statistics[data][config][0] / 1000) }k & {int(statistics[data][config][1])}",
end="",
)
if statistics[data][config][2] != 0:
print("^{", end="")
print(
int(statistics[data][config][2] / statistics[data][config][1] * 100),
# int(statistics[data][config][2]),
end="",
) )
print("}", end="")
print(" \\\\")
sorted_stats = sorted(statistics.items()) # Print footer
a = sorted_stats[0][0][: sorted_stats[0][0].find(",")] print("\n\\bottomrule\n\end{tabular}")
for key, val in sorted_stats:
if key[: key.find(",")] != a:
print("\n\n")
a = key[: key.find(",")]
print("%s,%s" % (key, ",".join([v.__str__() for v in val])))
exit(1)