Jip J. Dekker
f2a1c4e389
git-subtree-dir: software/mza git-subtree-split: f970a59b177c13ca3dd8aaef8cc6681d83b7e813
164 lines
7.3 KiB
Python
164 lines
7.3 KiB
Python
import timeit, re, sys, os
|
|
import utils, json_config
|
|
|
|
## TODO Keyline/value dictionaries: entries == json_config.s_CommentKey are ignored. Make it a parameter
|
|
|
|
class Output:
|
|
def __init__( self ):
|
|
stdout = ""
|
|
stderr = ""
|
|
|
|
def on_terminate(proc):
|
|
print("process {} terminated with exit code {}".format(proc, proc.returncode))
|
|
|
|
##############################################################################################
|
|
###################### MZN instance execution + result parsing low-level #####################
|
|
##############################################################################################
|
|
## runCmdCmdline using system(). Actually used for checking as well.
|
|
## run the specified shell command string and return the time.
|
|
## can add ulimit etc.
|
|
## s1, s2: filenames for strout, stderr
|
|
## dictCmd: commands for Win and non-Win
|
|
## meml: list of 2 values, soft & hard limits as N bytes
|
|
def runCmdCmdline( s_Cmd, s1, s2, dictCmd, timeo, bVerbose=False, meml=None ):
|
|
tm = timeit.default_timer()
|
|
setCmd = dictCmd["windows"] if "win" in sys.platform and "cygwin" not in sys.platform else dictCmd["non-windows"]
|
|
sCmd = setCmd["runVerbose" if bVerbose else "runSilent"].format(meml[1], timeo, s_Cmd, s1, s2)
|
|
print( "\n RUNNING:", sCmd )
|
|
os.system(sCmd)
|
|
tm = timeit.default_timer() - tm
|
|
return tm
|
|
|
|
|
|
## runCmd using psutils. Actually should be used for checking as well.
|
|
## run the specified shell command string and return the popen result.
|
|
## TODO catch intermediate solutions if needed
|
|
## meml: list of 2 values, soft & hard limits as N bytes
|
|
def runCmd( s_Cmd, b_Shell=False, timeo=None, meml=None ):
|
|
import psutil, shlex, subprocess, resource
|
|
if b_Shell:
|
|
l_Cmd = s_Cmd
|
|
else:
|
|
l_Cmd = shlex.split( s_Cmd )
|
|
tm = timeit.default_timer()
|
|
################# In the following, the subprocess.RUN method fails to kill shell calls on Linux
|
|
#try:
|
|
# completed = subprocess.run( l_Cmd, shell=b_Shell,
|
|
# universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeo )
|
|
#except subprocess.TimeoutExpired as te:
|
|
# completed = te
|
|
################# Using psutils
|
|
proc = psutil.Popen(l_Cmd, shell=b_Shell,
|
|
universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if None!=meml:
|
|
if hasattr( psutil, 'RLIMIT_AS' ): ## TODO move into preexec_fn for parallel tests?
|
|
proc.rlimit( resource.RLIMIT_AS, ( meml[0]*1000, meml[1]*1000 ) )
|
|
else:
|
|
print( " ... but the OS doesn't support RLIMIT_AS." )
|
|
completed = Output()
|
|
try:
|
|
completed.stdout, completed.stderr = proc.communicate(timeout=timeo)
|
|
except subprocess.TimeoutExpired:
|
|
print ( " soft_kill. ", end='' )
|
|
procs = psutil.Process().children(recursive=True)
|
|
for p in procs:
|
|
p.terminate()
|
|
try:
|
|
completed.stdout, completed.stderr = proc.communicate(timeout=1)
|
|
except subprocess.TimeoutExpired as te:
|
|
print ( " hard_kill. ", end='' )
|
|
procs = psutil.Process().children(recursive=True)
|
|
for p in procs:
|
|
p.kill()
|
|
completed.stdout, completed.stderr = proc.communicate()
|
|
### Clean up: (does psutil.Process.communicate() wait for all descendants?) -----------------------------------
|
|
procs = psutil.Process().children(recursive=True)
|
|
for p in procs:
|
|
p.kill()
|
|
### OR: even Queue?
|
|
# with psutil.Popen(["ifconfig"], stdout=subprocess.PIPE) as proc:
|
|
# log.write(proc.stdout.read())
|
|
#
|
|
# procs = psutil.Process().children()
|
|
# for p in procs:
|
|
# p.terminate()
|
|
# gone, still_alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate)
|
|
# for p in still_alive:
|
|
# p.kill()
|
|
tm = timeit.default_timer() - tm
|
|
return completed, tm
|
|
|
|
def parseStderr( f, result, mapKL, mapKV ):
|
|
# result["ProbSense"] = None
|
|
# result["TimeFlt"] = None
|
|
for line in f:
|
|
line = line.strip()
|
|
checkKeylines( line, mapKL, result )
|
|
checkKeyvalues( line, mapKV, result )
|
|
|
|
## Puts feasible solutions into solList if it's not None
|
|
def parseStdout( f, result, mapKL, mapKV, solList ):
|
|
l_SolLast = ""
|
|
n_SolStatus = 0
|
|
result["Number_Solutions"] = 0
|
|
for line in f:
|
|
line = line.rstrip() ## To remove \n and spaces on the right
|
|
res00 = {} ## A temporary to see if we get a feasible solution separator
|
|
checkKeylines( line, mapKL, res00 )
|
|
utils.mergeDict( result, res00 )
|
|
checkKeyvalues( line, mapKV, result )
|
|
## See if it's a solution status
|
|
if "Sol_Status" in res00:
|
|
if 1==res00[ "Sol_Status" ][0]:
|
|
result["Number_Solutions"] += 1
|
|
## result["Solution_Last"] = l_SolLast ## Or save it? Need from a file then but might be great to have here
|
|
if None!=solList:
|
|
solList.append( l_SolLast )
|
|
l_SolLast = "" ## Clean up
|
|
else:
|
|
l_SolLast += line
|
|
l_SolLast += '\n'
|
|
|
|
## Check if any keylines of the given 2-level dictionary equal the given line
|
|
## The 1st level gives variable name, 2nd level gives the line->value map
|
|
def checkKeylines( line, dict2, result ):
|
|
assert dict==type(dict2) ## What if we use OrderedDict??? TODO
|
|
assert isinstance(result, dict)
|
|
for key in dict2:
|
|
if json_config.s_CommentKey!=key:
|
|
val1 = dict2[key]
|
|
assert dict==type(val1), "checkKeylines: key '%s': '%s'==type (%s)" % \
|
|
( key, type(val1), val1.__str__() )
|
|
if line in val1:
|
|
result[ key ] = [ val1[ line ], line ]
|
|
|
|
## Check if any search pattern of the given dict values is in the given line
|
|
## The key gives variable name, value gives the line->value mapping
|
|
def checkKeyvalues( line, dictVal, result ):
|
|
assert dict==type(dictVal) ## What if we use OrderedDict??? TODO
|
|
assert isinstance(result, dict)
|
|
for key in dictVal:
|
|
if json_config.s_CommentKey!=key:
|
|
paramArray = dictVal[ key ]
|
|
assert 3<=len( paramArray ), \
|
|
"Key '%s': param list should have >=3 elements, now: %s" % (key, paramArray.__str__())
|
|
if None!=re.search( paramArray[0], line ):
|
|
try:
|
|
lineSubst = re.sub( paramArray[1], ' ', line )
|
|
except:
|
|
print(" WARNING: failed to substitute regex '", paramArray[1],
|
|
"' by ' ' in string '", line, "': ", sys.exc_info()[0:2]
|
|
, sep='')
|
|
else:
|
|
lSL = lineSubst.split()
|
|
if len( lSL )>paramArray[2]-1:
|
|
s_Val = lSL[ paramArray[2]-1 ]
|
|
## d_Val = try_float( s_Val )
|
|
result[key] = s_Val ### [ d_Val, s_Val ] Need here?
|
|
## print( " checkKeyval: result[{}] = '{}'".format(key, s_Val) )
|
|
else:
|
|
print( "ERROR: Parsing output line ", lSL,
|
|
": regex key '", paramArray[0],
|
|
"' found, but the split line too short for index ", paramArray[2],
|
|
sep='' )
|