#!/usr/bin/env python3

##  Author: Gleb Belov@monash.edu  2017
##  This program runs MiniZinc solvers over a set of instances,
##  checks solutions and compares to given solution logs.

##  TODO type errors etc. when checking?
##  TODO continuous output dumping as option
##  TODO CPU/user time limit, proper memory limit (setrlimit not working)

import sys, io, re as regex, traceback
import os.path, platform
##import numpy
import math
import json, argparse
import datetime
from collections import OrderedDict

import utils, json_config, json_log, mzn_exec, cmp_result_logs
from json_config import s_CommentKey, s_AddKey

s_ProgramDescr = 'MiniZinc testing automation. (c) 2018 Monash University, gleb.belov@monash.edu'
s_ProgramDescrLong = ( "Allows checking of MiniZinc solutions by configurable checker profiles. The solutions can be input from previous logs or produced by a chosen solver profile, for 1 or several instances, with result comparison and some ranking. New solutions' summary logs, detailed stdout/err outputs, and statistics are saved in subfolder mzn-test/LOGS, /OUTPUTS, and /STATS, resp.")

###########################   GENERAL CONFIG. Could be in the JSON config actually    #########################
sDirResults = "mzn-test"
sFlnSolUnchk = sDirResults + "/sol__unchk.json"      ### Logfile to append immediate results
sFlnSolCheckBase = sDirResults + "/LOGS/sol__{}.json"        ### Logfile to append checked results. {} replaced by datetime
sFlnSolFailBase = sDirResults + "/LOGS/sol__{}__FAIL.json"      ### To save failed solutions
sFlnSolLastDzn = sDirResults + "/sol_last.dzn"     ### File to save the DZN solution for checking
sFlnStdoutBase = sDirResults + "/OUTPUTS/{}.stdout.txt"      ### File name base to dump stdout for any backend call
sFlnStderrBase = sDirResults + "/OUTPUTS/{}.stderr.txt"
sFlnStatLog = sDirResults + "/STATS/stat_log__{}.txt"              ### The instance list name(s) will be inserted, if any

sDZNOutputAgrs = "--output-mode dzn --output-objective"    ## The flattener arguments to produce DZN-compatible output facilitating solution checking
                             ## Remove --output-objective in MZN Release <=2.1.7
sFlatOptChecker = "--allow-multiple-assignments -Dmzn_ignore_symmetry_breaking_constraints=true"

s_UsageExamples = ( 
    "\nUSAGE EXAMPLES:"
    "\n(1)  \"mzn-test.py model.mzn data.dzn [--checkDZN stdout.txt [--checkStderr stderr.txt]] [--chkPrf MINIZINC-CHK --chkPrf FZN-GECODE-CHK] [--tCheck 15] [--addSolverOption \"--fzn-flags '-D fPureCircuit=true'\"]\"                  ::: check the instance's solutions, optionally reading them from a DZN-formatted file (otherwise solving first), optionally overriding default checker list etc."
    "\n(2)  \"mzn-test.py --solver CPLEX -t 300 -l instList1.txt -l instList2.txt --name CPLEXTest_003 --result newLog00.json prevLog1.json prevLog2.json --failed failLog.json\""
    "             ::: solve instances using the specified solver profile and wall time limit 300 seconds. The instances are taken from the list files. The test is aliased CPLEXTest_003. Results are saved to newLog00.json and compared/ranked to those in prevLog's. (Probably) incorrect solutions are saved to failLog.json."
    "\n(3)  \"mzn-test.py [-l instList1.txt] -c prevLog1.json -c prevLog2.json [--runAndCmp]\"                  ::: compare existing logs, optionally limited to the given instances, optionally running new tests. USE SINGLE QUOTES ONLY INSIDE ARGUMENTS PASSED TO THE BACKENDS when running backends through shell."
  )
##############################################################################################
################ Parameters of MZN-Test, including config and command-line
##############################################################################################
class MZT_Param:
    def parseCmdLine(self):
        parser = argparse.ArgumentParser(
          description=s_ProgramDescr + '\n' + s_ProgramDescrLong,
          epilog=s_UsageExamples)
        parser.add_argument('instanceFiles', nargs='*', metavar='<instanceFile>',
          help='model instance files, if no instance lists supplied, otherwise existing solution logs to compare with')
        parser.add_argument('-l', '--instanceList', dest='l_InstLists', action='append', metavar='<instanceListFile>',
            help='file with a list of instance input files, one instance per line,'
              ' instance file types specified in config. Used for running tests or for instance selection in comparison mode')
        parser.add_argument('--name', '--testName', metavar='<string>', help='alias of this test run, defaults to result log file name')
        parser.add_argument('-c', '--compare', action="append", metavar='<logfile>',
                            help='summarize and compare results to existing <logfile>. Only compares the logs and does not run tests, unless --runAndCmp. The flags -c can be omitted if -l is used')
        parser.add_argument('--runAndCmp', '--runAndCompare', '--run', action='store_true',
                            help='even if other logs are provided by -c, do run the tests and compare')
        parser.add_argument('--solver', '--solverPrf', '--solverProfile', metavar='<prf_name>',
                            help='solver profile from those defined in config section \"SOLVER_PROFILES\"')
        parser.add_argument('--call', '--solverCall', metavar='"<exe+flags or shell command(s) if --shellSolve 1>"',
                            help='solver backend call, should be quoted. Insert %%s where instance files need to be. Flatten with \''
                             + sDZNOutputAgrs + '\' to enable solution checking, unless the model has a suitable output definition.'
                             " Pass '--output-time' to the output filter (e.g., solns2out) to enable ranking by time")
        parser.add_argument('-t', '--tSolve',
                            type=float,
                            metavar='<sec>', help='solver backend wall-time limit, default: '+
                              str(self.cfgDefault["BACKEND_DEFS"]["__BE_SOLVER"]["EXE"]["n_TimeoutRealHard"][0]))
        parser.add_argument('--result', default=sFlnSolCheckBase, metavar='<file>',
                            help='save result log to <file>, default: \''+sFlnSolCheckBase+'\'')
        parser.add_argument('--resultUnchk', default=sFlnSolUnchk, metavar='<file>', help='save unchecked result log to <file>')
        parser.add_argument('--chkPrf', '--checkerPrf', '--checkerProfile', metavar='<prf_name>', action='append',
                            help='checker profile from those defined in config section \"CHECKER_PROFILES\", can be a few')
        parser.add_argument('--tCheck',
                            type=float,
                            metavar='<sec>', help='checker backend wall-time limit, default: '+
                              str(self.cfgDefault["BACKEND_DEFS"]["__BE_CHECKER"]["EXE"]["n_TimeoutRealHard"][0]))
        parser.add_argument('--nCheckMax', '--nCheck', '--nCheckedMax', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
                            type=int,
                            metavar='<N>', help='max number of solutions checked per instance.'
                              ' Negative means checking starts from the last obtained solution. Default: '+
                              str(self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0]))
        parser.add_argument('--nFailedSaveMax', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
                            type=int,
                            metavar='<N>', help='max number of failed solution reports saved per instance, default: '+
                              str(self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0]))
        parser.add_argument('--failed', ## default=self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
                            metavar='<file>', help='save failed check reports to <file>, default: \''+
                              self.cfgDefault["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0]+'\'')
        parser.add_argument('--checkDZN', '--checkStdout', metavar='<stdout_file>',
                            help='for a single instance, check DZN-formatted solutions from a solver\'s std output dumped to <stdout_file>. The DZN format is produced, e.g., if the model is flattened with \'' + sDZNOutputAgrs + '\'')
        parser.add_argument('--checkStderr', metavar='<stderr_file>',
                            help='with --checkDZN, read a solver\'s stderr log from <stderr_file> (not essential)')
        parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='tee backend\'s stderr to screen, in addition to the instance\'s output dumpfile in mzn-test/OUTPUTS. Only works for --shellSolve 1')
        parser.add_argument('--vc', '--verbose-check', dest='vc', action='store_true', help='same for checking')
     ## parser.add_argument('--no-feature', dest='feature', action='store_false')
     ## parser.set_defaults(feature=True)
        parser.add_argument('--debug', '--printcall', type=int, metavar='<bitfield>', help='bit 1: print full solver call commands, bit 2: same for checker')
        parser.add_argument('--shellSolve', type=int, metavar='0/1', help='backend call through shell when using psutils')
        parser.add_argument('--psutils', type=int, metavar='0/1', help='backend call through psutils (seems buggy in 3.4.2)')
        ## parser.add_argument('--fullPaths', action='store_true',
        ##                    help='use full paths in instance identifiers. By default, it\'s the pure base filenames')
        
        parser.add_argument('--mergeCfg', action="append", metavar='<file>', help='merge config from <file>')
        parser.add_argument('--saveCfg', metavar='<file>', help='save internal config to <file>. Can be useful to modify some parameters and run with --mergeCfg')
        parser.add_argument('--saveSolverCfg', metavar='<file>', help='save the final solver backend config to <file>')
        parser.add_argument('--saveCheckerCfg', metavar='<file>', help='save the final checker backend config to <file>')
        parser.add_argument('--addOption', '--addOptions', action="append", metavar='<text>', type=str, help='add <text> to any solver / checker call')
        parser.add_argument('--addSolverOption', '--addSolverOptions', action="append", metavar='<text>', type=str, help='add <text> to the solver call')
        parser.add_argument('--addCheckerOption', '--addCheckerOptions', action="append", metavar='<text>', type=str, help='add <text> to a checker call')
        parser.add_argument('--useJoinedName', action="append", metavar='<...%s...>', type=str, help='add this to the call, with %%s being replaced by'
                            ' a joined filename from all the input filenames, e.g., "--writeModel MODELS/%%s.mps"')
        self.args = parser.parse_args()
        # print( "ARGS:\n", self.args )
        ## Solver backend and checker backend list
        self.slvBE = None
        self.chkBEs = None

    ## Get parameters from config and command line, merge values
    def obtainParams(self):
        self.parseCmdLine()
        self.mergeValues()
        
    def initCfgDefault(self):
        ddd = {
          s_CommentKey: [
            "The default config structure for mzn-test.py. ", s_ProgramDescr,
            "You can export this by --saveCfg and modify -> --mergeCfg,",
            "even having only partial JSON subtree in a merged file(s).",
            "Structure: COMMON_OPTIONS, SOLVER_/CHECKER_PROFILES, BACKEND_DEFS.",
            "Solver and checkers are selected from pre-defined profiles,",
            "which are in turn built from sequences of 'basic backend definitions'.",
            "Comments are either separate keys or added as list elements ('/// ...')",
            "in pre-selected positions; then, overriding items should keep that order."
          ],
          "COMMON_OPTIONS": {
            s_CommentKey: [ "'Solvers' and 'Checkers' select the profiles to use for solving and checking.",
                            "At the moment only the 1st solver is used from Solvers."
                            "The selected profiles must be known in SOLVER_PROFILES / CHECKER_PROFILES, resp."
                          ],
            "Solvers": [ "MINIZINC",
                        #"MZN-CPLEX",
                        "/// At the moment only the 1st element is used for solving" ],
            "SOLUTION_CHECKING": {
              "Checkers": ["GECODE-CHK", "GUROBI-CHK", "CPLEX-CHK", "ORTOOLS-CHK" ],
              "n_CheckedMax": [ -10, "/// Negative value means it's that many last solutions" ],
              "n_FailedSaveMax": [ 3, "/// After that many failed solutions, stop checking the instance" ],
              "s_FailedSaveFile": [ sFlnSolFailBase, "/// Filename to save failed solutions" ],
            },
            "Instance_List": {
              s_CommentKey: [ "Params for instance lists.",
                              "Instance list is a file containing an instance's model files,",
                              "at most 1 instance per line.",
                              "InstanceFileExt: only files with this extensions from a list file",
                              "will be taken on each line" ],
              "InstanceFileExt": [".mzn", ".dzn"]     ##   Add json?   TODO
            },
            "runCommand": {
              "windows": {
                "runSilent": "echo \"WARNING. No timeout on Windows.\" & {2} 1>{3} 2>{4}",
                "runVerbose": "echo \"WARNING. No timeout on Windows.\" & {2} 3>&1 1>{3} 2>&3 | tee {4} & echo >>{4}"
              }
              ,"non-windows": {
                "runSilent": "ulimit -v {0}; timeout -k 1 {1} bash -c \"{2}\" 1>{3} 2>{4}",
                "runVerbose": "ulimit -v {0}; timeout -k 1 {1} bash -c \"{2}\" 3>&1 1>{3} 2>&3 | tee {4}; echo >>{4}"
              }
            }
          },
          "SOLVER_PROFILES": {
            s_CommentKey: [ "Similar to CHECKER_PROFILES." ],
            "MINIZINC": [ "__BE_COMMON", "__BE_SOLVER", "BE_MINIZINC" ],
            "FZN-GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-GUROBI" ],
            "FZN-CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CPLEX" ],
            "FZN-CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CBC" ],
            "GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_GUROBI" ],
            "CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_CPLEX" ],
            "XPRESS": [ "__BE_COMMON", "__BE_SOLVER", "BE_XPRESS" ],
            "SCIP": [ "__BE_COMMON", "__BE_SOLVER", "BE_SCIP" ],
            "CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_CBC" ],
            "GECODE": [ "__BE_COMMON", "__BE_SOLVER", "BE_GECODE" ],
            "CHUFFED": [ "__BE_COMMON", "__BE_SOLVER", "BE_CHUFFED" ],
            "ORTOOLS": [ "__BE_COMMON", "__BE_SOLVER", "BE_ORTOOLS" ] #,
            # "MZN-GUROBI": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-GUROBI" ],
            # "MZN-CPLEX": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-CPLEX" ],
            # "MZN-CBC": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-CBC" ],
            # "MZN-GECODE": [ "__BE_COMMON", "__BE_SOLVER", "BE_MZN-GECODE" ],
            # "FZN-CHUFFED": [ "__BE_COMMON", "__BE_SOLVER", "BE_FZN-CHUFFED" ]
          },
          "CHECKER_PROFILES": {
            s_CommentKey: [ "Each profile gives a list of backend defs to use.",
              "Later backends in the list can override/add options, ",
              "for example for values to be read from the outputs.",
              "Adding is only possible if the key is prefixed by '"+s_AddKey+"'"
            ],
            "MINIZINC-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_MINIZINC" ],
            "GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_GECODE" ],
            "ORTOOLS-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_ORTOOLS" ],
            "GUROBI-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_GUROBI" ],
            "CPLEX-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_CPLEX" ],
            "CHUFFED-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_CHUFFED" ],
            "MZN-GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-GECODE" ],
            "MZN-GUROBI-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-GUROBI" ],
            "MZN-CPLEX-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_MZN-CPLEX" ],
            "FZN-GECODE-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_FZN-GECODE" ],
            "FZN-GECODE-SHELL-CHK": [ "__BE_COMMON", "__BE_CHECKER_OLDMINIZINC", "BE_FZN-GECODE_SHELL" ],
            "FZN-CHUFFED-CHK": [ "__BE_COMMON", "__BE_CHECKER", "BE_FZN-CHUFFED" ]
          },
          "BACKEND_DEFS": {
            s_CommentKey: [ "__BE_COMMON initializes a basic backend structure.",
              "Each further backend in a profile list overrides or adds options."
            ],
            "__BE_COMMON": {
              s_CommentKey: [ "THE INITIALIZING BACKEND." ],
              "EXE": {
                s_CommentKey: [ "Solver call parameters" ],
                "s_SolverCall" : ["minizinc -v -s -i " + sDZNOutputAgrs + " %s",
                  "/// The 1st element defines the call line. %s is replaced by the instance filename(s)."],
                "s_ExtraCmdline" : ["", "/// Only for __BE_SOLVER/__BE_CHECKER... subprofiles."
                                        " The 1st element gives extra cmdline arguments to the call"],
                "b_ThruShell"  : [True, "/// Set True to call solver thru shell."
                  " Then you can do shell tricks but Ctrl+C may not kill all subprocesses etc."],
                "n_TimeoutRealHard": [150, "/// Real-time timeout per instance, seconds,"
                  " for all solution steps together. Use mzn/backend options for CPU time limit."],
                "n_VMEMLIMIT_SoftHard": [8000000, 8000000, "/// 2 limits, soft/hard, in KB. Platform-dependent in Python 3.6. Default 8 GB"],
              },
              "Stderr_Keylines": {
                s_CommentKey: [ "A complete line in stderr will be interpreted accordingly.",
                  " Format: <outvar> : { <line>: <value>, ... }"
                  " You can add own things here (use '"+s_AddKey+"' before new var name)",
                  " which will be transferred into results" ]
              },
              "Stderr_Keyvalues": {
                s_CommentKey: [ "Numerical values to be extracted from a line in stderr.",
                  " { <outvar>: [ <regex search pattern>, <regex to replace by spaces>, <value's pos in the line>] }."
                ], 
                ### The %%%mzn-stat values appear in stdout (as of May 2019) but leave them here just in case
                "Time_Flt": [ "%%%mzn-stat: flatTime", "[:=]", 3, "/// E.g., 'Flattening done, 3s' produces 3."
                                " !!! This is interpreted as successful flattening by the checker" ],
                "ObjVal_Solver":   [ "%%%mzn-stat: objective=", "[,:/=]", 3,        ## Need = to avoid mixup with the bound
                                        "/// The objval as reported by solver."],
                "DualBnd_Solver":   [ "%%%mzn-stat: objectiveBound", "[,:/=]", 3 ],
                "CPUTime_Solver":   [ "%%%mzn-stat: solveTime", "[,:/=]", 3 ],
                "NNodes_Solver":   [ "%%%mzn-stat: nodes", "[,:/=]", 3 ],
              },
              "Stdout_Keylines": {
                s_CommentKey: [ "Similar to Stderr_Keylines"],
                "Sol_Status": {
                  "----------": 1,
                  "==========": 2,
                  "=====UNSATISFIABLE=====": -1,
                  "=====UNBOUNDED=====": -2,
                  "=====UNKNOWN=====": 0,
                  "=====UNSATorUNBOUNDED=====": -3,
                  "=====ERROR=====": -4
                },
                "Problem_Sense": {
                  "%%%mzn-stat: method=\"maximize\"": 1,
                  "%%%mzn-stat: method=\"minimize\"": -1,
                  "%%%mzn-stat: method=\"satisfy\"": 0,
                }
              },
              "Stdout_Keyvalues": {
                s_CommentKey: ["Similar to Stderr_Keyvalues." ],
                "Time_Flt": [ "%%%mzn-stat: flatTime", "[:=]", 3, "/// E.g., 'Flattening done, 3s' produces 3."
                                " !!! This is interpreted as successful flattening by the checker" ],
                "ObjVal_Solver":   [ "%%%mzn-stat: objective=", "[,:/=]", 3,        ## Need = to avoid mixup with the bound
                                        "/// The objval as reported by solver."],
                "DualBnd_Solver":   [ "%%%mzn-stat: objectiveBound", "[,:/=]", 3 ],
                "CPUTime_Solver":   [ "%%%mzn-stat: solveTime", "[,:/=]", 3 ],
                "NNodes_Solver":   [ "%%%mzn-stat: nodes", "[,:/=]", 3 ],
                "ObjVal_MZN":   [ "_objective", "[():=;%]", 2,
                                    "/// The objective value as evaluated by MZN." ],
                "RealTime_Solns2Out": [ "% time elapsed:", " ", 4 ],
              }
            },
            "__BE_SOLVER": {
              s_CommentKey: ["Specializations for a general solver" ],
              "EXE": {
                "s_ExtraCmdline" : ["-i"],
                "b_ThruShell"  : [True],
                "n_TimeoutRealHard": [150],
                #  "n_VMEMLIMIT_SoftHard": [8000100000, 8100000000]
              }
            },
            "__BE_CHECKER": {
              s_CommentKey: ["Specializations for a general checker" ],
              "EXE": {
                "s_ExtraCmdline" : [sFlatOptChecker],
                "b_ThruShell"  : [True],
                "n_TimeoutRealHard": [15],
                #  "n_VMEMLIMIT_SoftHard": [8100000000, 8100000000]
              }
            },
            "__BE_CHECKER_OLDMINIZINC": {
              s_CommentKey: ["Specializations for a general checker using the 1.6 MiniZinc driver" ],
              "EXE": {
                "s_ExtraCmdline" : ["--mzn2fzn-cmd 'mzn2fzn -v -s --output-mode dzn " + sFlatOptChecker + "'"],
                "b_ThruShell"  : [True],
                "n_TimeoutRealHard": [15],
                #  "n_VMEMLIMIT_SoftHard": [8100000000, 8100000000]
              }
            },
            "BE_MINIZINC": {
              s_CommentKey: [ "------------------- Specializations for pure minizinc driver" ],
              "EXE":{
                "s_SolverCall": [ "minizinc --mzn2fzn-cmd 'mzn2fzn -v -s " + sDZNOutputAgrs + "' -s %s"], # _objective fails for checking
                "b_ThruShell"  : [True],
              },
            },
            "BE_FZN-GUROBI": {
              s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ],
              "EXE":{
                "s_SolverCall" : ["mzn2fzn -v -s -G linear " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-gurobi -v -s tmp.fzn"],  ## works without solns2out for now. Need thus when using shell call with system()  TODO?
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {
                s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ],
                s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ],
                s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ]
              },
            },
            "BE_GUROBI": {
              s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ],
              "EXE":{
                "s_SolverCall" : ["minizinc -v -s --solver gurobi --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {
                s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ],
                s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ],
                s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ]
              },
            },
            "BE_MZN-GUROBI": {
              s_CommentKey: [ "------------------- Specializations for Gurobi solver instance" ],
              "EXE":{
                "s_SolverCall" : ["mzn-gurobi -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {
                s_AddKey+"Preslv_Rows": [ "Presolved:", " ", 2 ],
                s_AddKey+"Preslv_Cols": [ "Presolved:", " ", 4 ],
                s_AddKey+"Preslv_Non0": [ "Presolved:", " ", 6 ]
              },
            },
            "BE_FZN-CPLEX": {
              s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ],
              "EXE": {
                "s_SolverCall" : ["mzn2fzn -v -s -G linear " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-cplex -v -s tmp.fzn"]
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {
                s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
                s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
                s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
              },
            },
            "BE_CPLEX": {
              s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ],
              "EXE": {
                "s_SolverCall" : ["minizinc -v -s --solver cplex --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {
                s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
                s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
                s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
              },
            },
            "BE_XPRESS": {
              s_CommentKey: [ "------------------- Specializations for FICO XPRESS solver instance" ],
              "EXE": {
                "s_SolverCall" : ["minizinc -v -s --solver xpress --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {   ## Is different for XPRESS and -v fails at the moment anyway
                s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
                s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
                s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
              },
            },
            "BE_MZN-CPLEX": {
              s_CommentKey: [ "------------------- Specializations for IBM ILOG CPLEX solver instance" ],
              "EXE": {
                "s_SolverCall" : ["mzn-cplex -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
                #"opt_writeModel": ["--writeModel"]
              },
              "Stderr_Keyvalues": {
                s_AddKey+"Preslv_Rows": [ "Reduced MIP has [0-9]+ rows,", " ", 4 ],
                s_AddKey+"Preslv_Cols": [ "Reduced MIP has [0-9]+ rows,", " ", 6 ],
                s_AddKey+"Preslv_Non0": [ "Reduced MIP has [0-9]+ rows,", " ", 9 ]
              },
            },
            "BE_FZN-CBC": {
              s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ],
              "EXE": {
                "s_SolverCall" : ["mzn-cbc -v -s -G linear --output-time " + sDZNOutputAgrs + " %s --fzn tmp.fzn --ozn tmp.ozn && mzn-cplex -v -s tmp.fzn"], 
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
              },
            },
            "BE_SCIP": {
              s_CommentKey: [ "------------------- Specializations for SCIP solver instance" ],
              "EXE": {
                "s_SolverCall" : ["minizinc -v -s --solver scip --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
              },
            },
            "BE_CBC": {
              s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ],
              "EXE": {
                "s_SolverCall" : ["minizinc -v -s --solver osicbc --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
              },
            },
            "BE_MZN-CBC": {
              s_CommentKey: [ "------------------- Specializations for COIN-OR Branch&Cut solver instance" ],
              "EXE": {
                "s_SolverCall" : ["mzn-cbc -v -s -G linear --output-time " + sDZNOutputAgrs + " %s"], # _objective fails for checking
                #"s_SolverCall" : ["./run-mzn-cplex.sh %s"],
                #"b_ThruShell"  : [True],
              },
            },
            "BE_GECODE": {
              s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
              "EXE": {
#                "s_SolverCall" : ["minizinc -s --solver gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
                "s_SolverCall" : ["minizinc -v -s --solver gecode " + sDZNOutputAgrs
                     + " %s"],    #  --time 300000
                "b_ThruShell"  : [True],
              }
            },
            "BE_MZN-GECODE": {
              s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
              "EXE": {
#                "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
                "s_SolverCall" : ["mzn-gecode -v -s -G gecode " + sDZNOutputAgrs
                     + " %s"],    #  --time 300000
                "b_ThruShell"  : [True],
              }
            },
            "BE_FZN-GECODE": {
              s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
              "EXE": {
#                "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
                "s_SolverCall" : ["minizinc -s -G gecode -f fzn-gecode --mzn2fzn-cmd 'mzn2fzn -v -s " + sDZNOutputAgrs + "' %s"],
                "b_ThruShell"  : [True],
              }
            },
            "BE_FZN-GECODE_SHELL": {
              s_CommentKey: [ "------------------- Specializations for Gecode FlatZinc interpreter" ],
              "EXE": {
#                "s_SolverCall" : ["mzn-fzn -s -G gecode --solver fzn-gecode " + sDZNOutputAgrs + " %s"], # _objective fails for checking TODO
                "s_SolverCall" : ["mzn2fzn -v -s -G gecode " + sDZNOutputAgrs
                     + sFlatOptChecker + " %s --fzn tmp.fzn --ozn tmp.ozn && fzn-gecode tmp.fzn | solns2out tmp.ozn"],
                "b_ThruShell"  : [True],
                "s_ExtraCmdline" : [""],
              }
            },
            "BE_CHUFFED": {
              s_CommentKey: [ "------------------- Specializations for Chuffed FlatZinc interpreter" ],
              "EXE": {
                "s_SolverCall" : ["minizinc -v -s --solver chuffed -f --output-time "
                                    + sDZNOutputAgrs + " %s"], # _objective fails for checking
              }  ##  --fzn-flags --time-out --fzn-flags 300
            }
            , "BE_ORTOOLS": {
              s_CommentKey: [ "------------------- Specializations for OR-Tools FlatZinc interpreter" ],
              "EXE": {
                "s_SolverCall" : ["minizinc -v -s --solver ortools -f --output-time "
                                    + sDZNOutputAgrs + " %s"], # _objective fails for checking
              }  ##  --fzn-flags --time-out --fzn-flags 300
            }
            , "BE_FZN-CHUFFED": {
              s_CommentKey: [ "------------------- Specializations for Chuffed FlatZinc interpreter" ],
              "EXE": {
                "s_SolverCall" : ["mzn-fzn -v -s -G chuffed --solver fzn-chuffed --fzn-flags -f --output-time "
                                    + sDZNOutputAgrs + " %s"], # _objective fails for checking
              }  ##  --fzn-flags --time-out --fzn-flags 300
            }
          }
        }
        return ddd
    ##        self.nNoOptAndAtLeast2Feas = 0
    
    ## Read a cfg file, instead of/in addition to the default cfg
    def mergeCfg(self, fln):
        ddd1 = None
        with open( fln, 'r' ) as rf:
            ddd1 = json.load( rf )
        self.cfg = mergeJSON( self.cfg, ddd1 )
      
    ## Merge cmdline values with cfg, performing some immediate actions
    ## And compile the backends constituting solver and checker(s)
    def mergeValues(self):
        if None!=self.args.mergeCfg:               ### MERGE CFG FROM EXTRA FILES
            for eCfg in self.args.mergeCfg:
                self.mergeCfg( eCfg )
        ################ Update some explicit cmdline params -- AFTER MERGING CFG FILES.
        if None!=self.args.failed:
            self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0] = self.args.failed
        if None!=self.args.nCheckMax:
            self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] = self.args.nCheckMax
        if None!=self.args.nFailedSaveMax:
            self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] = self.args.nFailedSaveMax
        ################ SAVE FINAL CFG
        if None!=self.args.saveCfg:
            with utils.openFile_autoDir( self.args.saveCfg, 'w' ) as wf:
                print( "Saving final config to", self.args.saveCfg )
                json.dump( self.cfg, wf, sort_keys=True, indent=json_config.n_JSON_Indent )
        ### COMPILE THE SOLVER BACKEND
        if None!=self.args.solver:
            self.cfg["COMMON_OPTIONS"]["Solvers"][0] = self.args.solver
        slvPrfName = self.cfg["COMMON_OPTIONS"]["Solvers"][0]
        slvPrf = self.cfg["SOLVER_PROFILES"][slvPrfName]
        assert len(slvPrf)>0, "Solver profile '%s' should use at least a basic backend" % slvPrfName
        self.slvBE = self.cfg["BACKEND_DEFS"][slvPrf[0]]
        for i in range( 1, len( slvPrf ) ):
            self.slvBE = json_config.mergeJSON( self.slvBE, self.cfg["BACKEND_DEFS"][slvPrf[i]] )
        if None!=self.args.tSolve:
            self.slvBE["EXE"]["n_TimeoutRealHard"][0] = self.args.tSolve
        assert None==self.args.solver or None==self.args.call, "ERROR: both solver call and a solver profile specified."
        if None!=self.args.call:    ## After the compilation
            self.slvBE["EXE"]["s_SolverCall"][0] = self.args.call
        if None!=self.args.shellSolve:
            self.slvBE["EXE"]["b_ThruShell"][0] = self.args.shellSolve!=0
        print ( "\nSolver/checker configurations:\n     SLV_CFG: ", json.dumps( self.slvBE["EXE"] ) )
        ### COMPILE THE CHECKER BACKENDS
        if None!=self.args.chkPrf and 0<len( self.args.chkPrf ):
            self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["Checkers"] = self.args.chkPrf
        chkPrfList = self.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["Checkers"]
        self.chkBEs = []
        for chkPrfName in chkPrfList:
            chkPrf = self.cfg["CHECKER_PROFILES"][chkPrfName]
            assert len(chkPrf)>0, "Checker profile '%s' should use at least a basic backend" % chkPrfName
            self.chkBEs.append( self.cfg["BACKEND_DEFS"][chkPrf[0]] )
            for i in range( 1, len( chkPrf ) ):
                self.chkBEs[-1] = json_config.mergeJSON( self.chkBEs[-1], self.cfg["BACKEND_DEFS"][chkPrf[i]] )
            if None!=self.args.tCheck:
                self.chkBEs[-1]["EXE"]["n_TimeoutRealHard"][0] = self.args.tCheck
            print ( "     CHK_CFG: ", json.dumps( self.chkBEs[-1]["EXE"] ) )
        print( "" )
        ### SAVE THE SOLVER BACKEND
        if None!=self.args.saveSolverCfg:
            with utils.openFile_autoDir( self.args.saveSolverCfg, 'w' ) as wf:
                print( "Saving solver config to", self.args.saveSolverCfg )
                json.dump( self.slvBE, wf, sort_keys=True, indent=json_config.n_JSON_Indent )
        ### SAVE THE CHECKER BACKENDS
        if None!=self.args.saveCheckerCfg:
            with utils.openFile_autoDir( self.args.saveCheckerCfg, 'w' ) as wf:
                print( "Saving checker config to", self.args.saveCheckerCfg )
                json.dump( self.chkBE, wf, sort_keys=True, indent=json_config.n_JSON_Indent )
        self.sThisName = self.args.result
        if None!=self.args.name:
            self.sThisName = self.args.name
    
    def __init__(self):
        self.cfgDefault = self.initCfgDefault()
        self.cfg = self.cfgDefault
        ### Further parameters
        self.args = {}
        
    def __str__(self):
        s_Out = json.dumps( self.cfg, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n'
        s_Out += str(self.args) + '\n'
        s_Out += "\n >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> SOLVER BACKEND:\n"
        s_Out += json.dumps( self.slvBE, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n'
        s_Out += "\n >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> CHECKER BACKENDS:\n"
        for chkBE in self.chkBEs:
            s_Out += json.dumps( chkBE, sort_keys=True, indent=json_config.n_JSON_Indent ) + '\n'
        return s_Out
                

##############################################################################################
################### The MZNTest class
##############################################################################################
class MznTest:

    ## Produce a pair: first, a string identifying the instance which is given as a string of the instance files
    ## Second, same without paths and file extensions for short printing
    ## the elements are sorted according to the extensions list, then alphabetically
    def getIName( self, s_Inst ):
        lId = s_Inst.split()     ## list of instance files
        lExt = self.params.cfg["COMMON_OPTIONS"]["Instance_List"]["InstanceFileExt"]
        lId = sorted( lId, key = lambda nm:
                ( lExt.index( os.path.splitext(nm)[1] ) \
                        if os.path.splitext(nm)[1] in lExt else len(lExt), os.path.basename( nm ) ) )
        lId2 = [ os.path.splitext( os.path.basename( fln ) )[0] for fln in lId ];
        return ' '.join(lId), ' '.join(lId2);

    def obtainParams( self ):
        self.params.obtainParams()

    ## If an instance was specified in cmdline, or model list(s) supplied
    def compileExplicitModelLists(self):
        ## Get cmdline filenames or from the --instList arguments
        ## Can compile the list from log files, see below
        self.params.instList = []
        ## Only if -l not used, take the pos args
        if (self.params.args.l_InstLists is None or 0==len( self.params.args.l_InstLists )) \
                and self.params.args.instanceFiles is not None and 0<len( self.params.args.instanceFiles ):
            self.params.instList.append( " ".join( self.params.args.instanceFiles ) )
        ## Mode "compare only" if (comparison lists and not run option) or no instances
        self.bCmpOnly = True if (not self.params.args.runAndCmp and (( \
                self.params.args.compare is not None and 0<len(self.params.args.compare)) or \
              ( self.params.args.l_InstLists is not None and 0<len( self.params.args.l_InstLists ) \
              and self.params.args.instanceFiles is not None and 0<len( self.params.args.instanceFiles ) ))) \
          else False
        ## If -l used, compile the inst list files
        if None!=self.params.args.l_InstLists and 0<len( self.params.args.l_InstLists ):
            for sFln in self.params.args.l_InstLists:
                with open( sFln, 'r' ) as rf:
                    for line in rf:
                        s_ModelFiles = ""
                        for wrd in line.split():
                            fIsMF = False
                            for ext in self.params.cfg["COMMON_OPTIONS"]["Instance_List"]["InstanceFileExt"]:
                                if wrd.endswith( ext ):
                                    fIsMF = True
                                    break
                            if fIsMF:
                                s_ModelFiles += ' ' + wrd
                        if 0<len( s_ModelFiles ):
                          self.params.instList.append( s_ModelFiles )
        
    ## Result logs can be used to extract the instance list, if wished
    def compileResultLogs(self):
        self.cmpRes = cmp_result_logs.CompareLogs()
        cmpFileList = []
        if None!=self.params.args.compare:     ### If -c used
            cmpFileList += self.params.args.compare
        ### If -l used, interpret pos arguments as comparison logs
        if None!=self.params.args.l_InstLists and 0<len( self.params.args.l_InstLists ):
            cmpFileList += self.params.args.instanceFiles
        for sFlnLog in cmpFileList:
            nEntries = 0
            logCurrent, lLogNames = self.cmpRes.addLog( sFlnLog )
            print( "Reading result log '", sFlnLog, "'...  ", sep='', end='', flush=True )
            with open( sFlnLog, 'r' ) as rf:
                while True:
                    chJ = json_log.readLogJSONChunk( rf )
                    if None==chJ:
                        break
                    try:
                        logCurrent[ self.getIName( chJ["Inst_Files"] )[0] ] = chJ
                        if "TEST_NAME" in chJ:
                            lLogNames[1] = chJ[ "TEST_NAME" ]
                            print( "  TEST NAME: '", chJ[ "TEST_NAME" ],
                                  "'... ", sep='', end='', flush=True )
                    except:
                        print( "\n   WARNING: unrecognized result chunk in file '", sFlnLog,
                              "' before position", rf.tell(), ", doesn't contain all keys")
                    else:
                        nEntries += 1
            print( len(logCurrent), "different instances among", nEntries, "recognized entries."  )
        
    def runTheInstances(self):
        logCurrent, lLogNames = self.cmpRes.addLog( self.params.args.result )
        if self.params.sThisName!=lLogNames[0]:
            lLogNames[1] = self.params.sThisName
        ## TRUNCATING files first, then "a" - better on Win??
        sNow = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
        self.sStartTime = sNow
        self.fileSol00 = utils.openFile_autoDir( self.params.args.resultUnchk, "w" )
        self.fileSol = utils.openFile_autoDir(self.params.args.result.format(sNow + 
          ("" if self.params.args.name is None else "__"+utils.flnfy(self.params.args.name))), "w" )
        self.fileFailName = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0].format(sNow)
        self.fileFail = None      ## Not opening yet
        self.nCheckedInstances = 0
        self.nChecksFailed = 0
        self.cmpRes.initListComparison()
        for i_Inst in range( len(self.params.instList) ):
            s_Inst = self.params.instList[ i_Inst ]
            self.initInstance( i_Inst, s_Inst )
            self.solveOriginal( s_Inst )
            try:
                if self.ifShouldCheck():
                    self.checkOriginal( s_Inst )
                else:
                    print( "NO CHECK,  total check-failed instances:", self.nChecksFailed,
                           "from", self.nCheckedInstances )
            except:
                print( "  WARNING: failed to check instance solution. ", sys.exc_info() )
            ## Saving to the main log:
            self.saveSolution( self.fileSol )
            ## Ranking:
            sSet_Inst = self.getIName( self.result["Inst_Files"] )
            logCurrent[ sSet_Inst[0] ] = self.result
            try:
                self.cmpRes.compareInstance( sSet_Inst )
                if i_Inst < len(self.params.instList)-1:
                    print( "STATS:  ", end='' )
                    self.cmpRes.summarizeCurrent( lLogNames )
                    print( "" )
            except:
                print( "  WARNING: failed to compare/rank instance. ", sys.exc_info() )

    def compareLogs(self):
        self.cmpRes.initListComparison()
        theList = [ lfn for lfn in self.params.instList ]
        if 0==len( theList ):
            theList = self.cmpRes.getInstanceUnion()
        for sInst in theList:
            try:
                self.cmpRes.compareInstance( self.getIName( sInst ) )
            except:
                print( "  ------  WARNING: failed to compare/rank instance. ",  )
                traceback.print_exc()
        self.cmpRes.summarizeCmp()
        print( self.cmpRes.summarizeFinalHdr(), end='' )

    def summarize(self):
        try:
            ### Printing summary
            print ('')
            print ('='*50)
            print('            SUMMARY')
            print ('='*50)
            stats = self.cmpRes.summarize()
            print (stats)
        except Exception as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            print(exc_type, exc_obj, fname, exc_tb.tb_lineno)
        if not self.bCmpOnly:
            print( "Printing stats log to: ", end="" )
            sFlnSL = sFlnStatLog.format( utils.flnfy(
                " ".join(self.params.args.l_InstLists if self.params.args.l_InstLists is not None else []) ) )
            print( sFlnSL )
            with utils.openFile_autoDir( sFlnSL, "a" ) as wf:
                wf.write( "\nRUN  " + self.sStartTime + "--" + datetime.datetime.now().__str__() + ":  " )
                wf.write( sys.argv.__str__() )
                wf.write( "\n" )
                wf.write( stats )
            print( "\nResult logs saved to '",  self.params.args.result,
               "', with the unchecked log in '", self.params.args.resultUnchk, "'; failed solutions saved to '",
               self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["s_FailedSaveFile"][0],
               "' in an \"appendable JSON\" format, cf. https://github.com/pvorb/jsml."
               ## no more for shell: "\nSolver/checker stdout(err) outputs saved to '" + sDirResults + "/last_stdout(err)_...txt'."
               , sep='')

##############################################################################################
######################### MZNTest average-level #############################

    ## init
    def initInstance( self, i_Inst, s_Inst ):
        print( "\n--------------------- INSTANCE %d of %d:  '" % (i_Inst+1, len( self.params.instList )),
              s_Inst, "' ----------------------", sep='' )
        self.result = OrderedDict()
        if 0==i_Inst and None!=self.params.args.name:
            self.result["TEST_NAME"] = self.params.args.name
        self.result[ "Inst_Index_Of" ] = [ (i_Inst+1), len( self.params.instList ) ]
        self.result["Inst_Files"] = s_Inst
        self.solList = []                   ## The solution list
        ## TODO: instance name source?

    ## Solve the original instance
    def solveOriginal( self, s_Inst ):
        self.result["__SOLVE__"] = self.solveInstance( s_Inst, self.params.slvBE, '_SOLVING', self.solList )
        # print( "   RESULT:\n", self.result )
        self.saveSolution( self.fileSol00 )
        
    ## Solve a given MZN instance and return the result map
    ## Arguments: instance files in a string, backend parameter dictionary
    ## slvName is used for screen output and last_std(out/err)_... naming, so better no spaces
    ## solList provided <=> this is solving (not checking) and will use --checkDZN if opted

    ##  TODO
    ## Because we cannot limit memory of the subprocesses directly AND there seem to be bugs in the Python 3.4 impl,
    ## could replace the subprocess call by a call to an external which would run under given memory/time limits
    ## and save output to given files.
    ## OR: update from Python 3.4.2?
    ## NAming output / model files: sort input filenames, replace spaces/punctuation
    ### USE smth like
    ## keepcharacters = (' ','.','_')
    ## "".join(c for c in filename if c.isalnum() or c in keepcharacters else 'I').strip()
    ## PARAMETERS
    ##   : solList: if not None, we are solving originally ( not checking )
    def solveInstance(self, s_Inst, slvBE, slvName, solList=None):
        resSlv = OrderedDict()
        bChkDZN = True if None!=solList and None!=self.params.args.checkDZN else False
        if bChkDZN:
            print( "_PARSING '", self.params.args.checkDZN, sep='', end="'... " )
            with open( self.params.args.checkDZN, 'r' ) as ro:
                mzn_exec.parseStdout( ro, resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )
                print( ro.tell(), "bytes", end='' )
            if None!=self.params.args.checkStderr:
                print( "  and '", self.params.args.checkStderr, sep='', end="'... " )
                with open( self.params.args.checkStderr, 'r' ) as re:
                    mzn_exec.parseStderr( re, resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] )
                    print( re.tell(), "bytes", end='' )
        else:      #### Solving oneself
            print( slvName, "... ", sep='', end='', flush=True )
            s_Call = slvBE["EXE"]["s_SolverCall"][0] % s_Inst \
              + ' ' + slvBE["EXE"]["s_ExtraCmdline"][0]
            if self.params.args.addOption is not None:
                for sOpt in self.params.args.addOption:
                    s_Call += ' ' + sOpt
            if solList is not None:
                if self.params.args.addSolverOption is not None:
                    for sOpt in self.params.args.addSolverOption:
                        s_Call += ' ' + sOpt
            else:
                if self.params.args.addCheckerOption is not None:
                    for sOpt in self.params.args.addCheckerOption:
                        s_Call += ' ' + sOpt
            s_InstMerged = s_Inst.strip()
            ## s_InstMerged = regex.sub( r"[.\\/:~]", "", s_InstMerged );
            ## s_InstMerged = regex.sub( r"[ ]", "-", s_InstMerged );
            keepcharacters = ('-','_')
            s_InstMerged = "".join(c if c.isalnum() or c in keepcharacters else 'I' for c in s_InstMerged).strip()
            
            if solList is not None and self.params.args.useJoinedName is not None:
                for sUseJN in self.params.args.useJoinedName:
                    s_UsingOpt = sUseJN % s_InstMerged
                    s_Call += ' ' + s_UsingOpt
            if solList is not None:      ## solving the original instance
                sFlnStdout = sFlnStdoutBase.format( s_InstMerged )
                sFlnStderr = sFlnStderrBase.format( s_InstMerged )
                if self.params.args.debug is not None and ( self.params.args.debug & 1 ):
                    print( "  CALL: \"", s_Call, "\"", sep='', flush=True )
            else:
                sFlnStdout = 'last_stdout' + slvName + '.txt'
                sFlnStderr = 'last_stderr' + slvName + '.txt'
                if self.params.args.debug is not None and ( self.params.args.debug & 2 ):
                    print( "  CALL: \"", s_Call, "\"", sep='', flush=True )
            resSlv["Solver_Call"] = s_Call
            resSlv["DateTime_Start"] = datetime.datetime.now().__str__()
            if 1==self.params.args.psutils:
                completed, tmAll = \
                    mzn_exec.runCmd(
                      s_Call,
                      slvBE["EXE"]["b_ThruShell"][0],
                      slvBE["EXE"]["n_TimeoutRealHard"][0],
                      slvBE["EXE"]["n_VMEMLIMIT_SoftHard"]
                    )
                with utils.openFile_autoDir( sFlnStdout, "w" ) as tf:
                    tf.write( completed.stdout )
                with utils.openFile_autoDir( sFlnStderr, "w" ) as tf:
                    tf.write( completed.stderr )
                print( "STDOUT/ERR: ", len(completed.stdout), '/',
                      len(completed.stderr), " bytes", sep='', end=', ' )
                mzn_exec.parseStderr( io.StringIO( completed.stderr ), resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] )
                mzn_exec.parseStdout( io.StringIO( completed.stdout ), resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )
                ## Adding the outputs to the log
                ## resSlv["StdErr"] = completed.stderr
                ## resSlv["StdOut"] = completed.stdout
            else:          ## use the 'system' call
                with utils.openFile_autoDir( sFlnStdout, "w" ) as tf:
                    tf.write( "% EMPTY\n" )
                with utils.openFile_autoDir( sFlnStderr, "w" ) as tf:
                    tf.write( "% EMPTY" )
                tmAll = mzn_exec.runCmdCmdline(
                      s_Call,
                      sFlnStdout, sFlnStderr,
                      self.params.cfg["COMMON_OPTIONS"]["runCommand"],
                      slvBE["EXE"]["n_TimeoutRealHard"][0],
                      (self.params.args.verbose) if solList is not None else (self.params.args.vc),
                      slvBE["EXE"]["n_VMEMLIMIT_SoftHard"]
                )
                with open( sFlnStderr, "r" ) as rf:
                    mzn_exec.parseStderr( rf, resSlv, slvBE["Stderr_Keylines"], slvBE["Stderr_Keyvalues"] )
                with open( sFlnStdout, "r" ) as rf:
                    mzn_exec.parseStdout( rf, resSlv, slvBE["Stdout_Keylines"], slvBE["Stdout_Keyvalues"], solList )

            print( " t: {:.3f}".format( tmAll ), end=' s, ' )
            resSlv["DateTime_Finish"] = datetime.datetime.now().__str__()
            resSlv["TimeReal_All"] = tmAll
            resSlv["TimeReal_LastStatus"] = 0
            resSlv["Hostname"] = platform.uname()[1]
        ### For all cases, some postprocessing #############################
        if "Sol_Status" not in resSlv:
            resSlv["Sol_Status"] = [-50, "   !!!!!  STATUS TOTALLY UNKNOWN - NO STATUS LINE PARSED."]
        if "Time_Flt" not in resSlv or utils.try_float( resSlv.get( "Time_Flt" ) ) is None:
            resSlv["NOFZN"] = ["      !!!!! No flattening finish time registered or successfully parsed"]
        dTmLast = utils.try_float( resSlv.get( "RealTime_Solns2Out" ) )
        if None!=dTmLast:
            resSlv["TimeReal_LastStatus"] = dTmLast
            resSlv.pop( "RealTime_Solns2Out" )
        ## if "SolutionLast" in resSlv:
        ##     print( "   SOLUTION_LAST:\n", resSlv["SolutionLast"], sep='' )
        if None!=solList:
            print( "  Nsol:", len(solList), end=',' )
        print( "   STATUS:", resSlv["Sol_Status"] )
        return resSlv
      
    ## Append the unchecked solution of the instance to a temp. file
    def saveSolution(self, wf):
        json_log.writeLogChunk( wf, json.dumps( self.result, indent=json_config.n_JSON_Indent ) )
    
    ## Return the necessity to check solutions.
    ## Can be false if we only want to compare different solvers.
    def ifShouldCheck(self):
        return \
          0<self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0] and \
          0!=self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0] and \
          0<len( self.params.chkBEs ) and \
          self.ifCheckableStatus( self.result["__SOLVE__"]["Sol_Status"][0] )
    
    ##
    def ifCheckableStatus( self, status ):
        return status>0
    
    ## Check selected solutions of the instance
    def checkOriginal( self, s_Inst ):
        nCM = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_CheckedMax"][0]
        nFSM = self.params.cfg["COMMON_OPTIONS"]["SOLUTION_CHECKING"]["n_FailedSaveMax"][0]
        assert 0!=nCM
        assert 0<len(self.solList)
        rng = None
        if 0<nCM:
            rng = range( 0, min( nCM, len(self.solList ) ) )
        else:
            rng = range( -1, max( nCM-1, -len(self.solList)-1 ), -1 )
        self.result["SOLUTION_CHECKS_DONE"] = 0
        self.result["SOLUTION_CHECKS_FAILED"] = 0
        fFailed = 0
        ## Iterate over the solution range
        for iSol in rng:
            ## Try modify
            # self.solList[ iSol ] = self.solList[iSol][:20] + '1' + self.solList[iSol][21:]
            print ( "CHK SOL ", iSol if iSol<0 else (iSol+1), '/', len(rng), sep='', end='... ' )
            with utils.openFile_autoDir( sFlnSolLastDzn, "w" ) as wf:          ## Save the selected solution
                wf.write( self.solList[ iSol ] )
            s_IC = s_Inst + ' ' + sFlnSolLastDzn
            bCheckOK = True
            chkFlSt = []
            # self.result["__CHECKS__"] = []
            for iChk in range ( len ( self.params.chkBEs ) ):
                chkBE = self.params.chkBEs[ iChk ]
                chkRes = self.solveInstance( s_IC, chkBE, '__Checker_'+str(iChk+1) )
                # self.result["__CHECKS__"].append( chkRes )
                if ( ## -51==chkRes["Sol_Status"][0] or               ## NOFZN? No, flattener should report INFEAS.
                    ( 0>chkRes["Sol_Status"][0] and -3<=chkRes["Sol_Status"][0] ) ):   ## INFEAS
                    bCheckOK = False
                    chkFlSt = chkRes["Sol_Status"]
            self.result["SOLUTION_CHECKS_DONE"] += 1
            if not bCheckOK:
                fFailed = 1
                self.result["SOLUTION_CHECKS_FAILED"] += 1
                self.result["SOLUTION_FAILED_LAST"] = self.solList[ iSol ]
                # self.result["SOLUTION_FAILED_LAST__CHKSTATUS"] = chkRes["Sol_Status"]
                if self.fileFail is None:
                    self.fileFail = utils.openFile_autoDir( self.fileFailName, "w" )
                self.saveSolution( self.fileFail )
                if nFSM<=self.result["SOLUTION_CHECKS_FAILED"]:
                    print ( self.result["SOLUTION_CHECKS_FAILED"], "failed solution(s) saved, go on" )
                    break
              
        self.nCheckedInstances += 1
        self.nChecksFailed += fFailed
        print( "   CHECK FAILS on this instance: ", self.result["SOLUTION_CHECKS_FAILED"],
                ",  total check-failed instances: ", self.nChecksFailed,
                " from ", self.nCheckedInstances, sep='' )
            
    def __init__(self):
        ## Default params
        self.params = MZT_Param()

##############################################################################################
######################### MZNTest public #############################
##############################################################################################
    def run(self):
        self.obtainParams()
        self.compileExplicitModelLists()
        self.compileResultLogs()
        if not self.bCmpOnly:
            self.runTheInstances()
        else:
            self.compareLogs()
        self.summarize()
        
#  def main
mznTst = MznTest()
mznTst.run()