#!/bin/bash
# vim: ft=sh ts=4 sw=4 et
#
# usage: run-test <prog> <filename>
#
# Ralph Becket <rafe@csse.unimelb.edu.au>
#
#
# Run program <prog> on <filename> and compare the results against the
# expected outputs.  The solver is run with default time and memory limits
# which an be configured by setting TIMELIMIT (default 30 seconds) and
# MEMLIMIT (default 1000 megabytes).
#
# If G12PATH is set and $G12PATH/<prog> is executable then this file is
# run, otherwise PATH is used.
#
# <filename> should end in
# - .fzn for FlatZinc tests
# - .mzn for MiniZinc tests
# - .zinc for Zinc tests
# - .dzn for Zinc or MiniZinc tests (the prefix of <filename> up to
# the first '.' is used to identify the corresponding .mzn or .zinc model).
#
#
# TEST CASES
#
# A test case consists of the following:
#
# - a <model>.zinc or <model>.mzn or <model>.fzn file;
#
# - zero or more <model>.<testid>.dzn files (as an option for .zinc and .mzn
# tests which separate the data from the model);
#
# - an optional <model>.opt file containing command line options to be
# passed to <prog> for <model>;
#
# - an optional <model>.<prog>.opt file containing command line options
# specific to <prog> for <model>;
#
# - [where a test is not specified as a .dzn file] one or more <model>*.exp*
# or <model>*.err_exp* files specifying the expected output or error report
#
# - [where a test is specified as a .dzn file] one or more;
# <model>.<testid>*.exp* or <model>.<testid>*.err_exp* files specifying the
# expected output or error report.
#
#
# HOW TEST CASES ARE RUN
#
# The contents of any <model>.opts and <model>.<prog>.opts files are
# collected in $OPTS.
#
# Where a test is specified as a <model>.[fzn|mzn|zinc] file:
#   $OUT is defined as <model>.<prog>.out
#   $ERR is defined as <model>.<prog>.err
#   The test is run under time-and-mem-limit as:
#   <prog> $OPTS <model>.[fzn|mzn|zinc] >$OUT 2>$ERR
#
# Where a test is specified as a <model>.<testid>.dzn file:
#   $OUT is defined as <model>.<testid>.<prog>.out
#   $ERR is defined as <model>.<testid>.<prog>.err
#   The test is run under time-and-mem-limit as:
#   <prog> $OPTS -d <model>.<testid>.dzn <model>.[mzn|zinc] >$OUT 2>$ERR
#
# If the $OUT file matches a <model>*.exp* or <model>.<testid>.*exp* file
# or the $ERR file matches a <model>*.err_exp* or <model>.<testid>.*err_exp*
# file then the test is deemed to have passed and $OUT and $ERR are deleted.
# Otherwise an error message is printed and $OUT and $ERR are left for later
# review.



# Uncomment the next line for debugging:
# set -x

# Set default time and memory limits (seconds and megabytes respectively).
#
TIMELIMIT=${TIMELIMIT:-30}
MEMLIMIT=${MEMLIMIT:-8000}

# The name of this program as invoked.
#
THIS=$(basename $0)

# Parse the command line.
#
if [ $# -ne 2 ]
then
    echo "usage: $THIS <prog> <filename>" >&2
    echo "  <filename> must be either " >&2
    echo "     <model>.<testid>.dzn " >&2
    echo "  or <model>.mzn" >&2
    echo "  or <model>.fzn" >&2
    echo "  or <model>.zinc" >&2
    echo "  Environmnent vars TIMELIMIT (default $TIMELIMIT seconds)" >&2
    echo "  and MEMLIMIT (default $MEMLIMIT megabytes) are recognised." >&2
    exit 1
fi

PROG=$1
FILENAME=$2

if test "$G12PATH" -a -x "$G12PATH/$PROG"; then
    PROG_EXEC="$G12PATH/$PROG"
else
    PROG_EXEC="$PROG"
fi

# cd to the directory of the problem file if it is not present in the
# current working directory.
#
DIR=$(dirname $FILENAME)
if [ -n "$DIR" ]
then
    cd $DIR >/dev/null 2>&1
    FILENAME=$(basename $FILENAME)
fi

# get_exp_out_err_files X sets the OPTS, EXPS, ERREXPS, OUT and ERR variables
# for problem X.  If no expected output files are found then exit.
#
function get_exp_out_err_files () {

    X=$1

    # Find the .exp files.
    #
    EXPS=$(find . -maxdepth 1 -name $X'*.exp*')
    ERREXPS=$(find . -maxdepth 1 -name $X'*.err_exp*')

    # If no .exp file exists for this model then do nothing.
    #
    if [ -z "$EXPS" -a -z "$ERREXPS" ]
    then
        echo "$THIS: skipping $FILENAME since it has no .exp files" >&2
        exit 0
    fi

    # Read any special options for this model.
    #
    [ -e $X.opt ]       && OPTS=$(cat $X.opt)
    [ -e $X.$PROG.opt ] && OPTS=$OPTS" "$(cat $X.$PROG.opt)

    # Construct the command to run the solver.
    #
    OUT=$X.$PROG.out
    ERR=$X.$PROG.err

}

# Decide what to do with the given file.
# 
case "$FILENAME" in

    *.fzn)
        X=$(basename $FILENAME .fzn)
        FZN=$X.fzn

        get_exp_out_err_files $X

        # Construct the command to run the solver.
        #
        CMD="$PROG_EXEC $OPTS $FZN"

        ;;

    *.mzn)
        X=$(basename $FILENAME .mzn)
        MZN=$X.mzn

        # If a .dzn file exists for this model then do nothing.
        #
        if ls $X*.dzn >/dev/null 2>&1
        then
            exit 0
        fi

        get_exp_out_err_files $X

        # Construct the command to run the solver.
        #
        CMD="$PROG_EXEC $OPTS $MZN"

        ;;

    *.zinc)
        X=$(basename $FILENAME .zinc)
        ZINC=$X.zinc

        # If a .dzn file exists for this model then do nothing.
        #
        if ls $X*.dzn >/dev/null 2>&1
        then
            exit 0
        fi

        get_exp_out_err_files $X

        # Construct the command to run the solver.
        #
        CMD="$PROG_EXEC $OPTS $ZINC"

        ;;

    *.dzn)
        X=$(basename $FILENAME .dzn)
        DZN=$X.dzn
        MZN=$(echo $X | sed 's/[.].*//').mzn
        ZINC=$(echo $X | sed 's/[.].*//').zinc

        # Find the corresponding .mzn or .zinc file.  Report an error if it
        # doesn't exist.
        #
        MODEL=""
        [ -e $MZN  ] && MODEL=$MZN
        [ -e $ZINC ] && MODEL=$ZINC
        if [ -z $MODEL ]
        then
            echo "$THIS: no .mzn or .zinc file corresponding to $DZN" >&2
            exit 1
        fi

        get_exp_out_err_files $X

        # Construct the command to run the solver (give priority to
        # zinc models).
        #
        CMD="$PROG_EXEC $OPTS -d $DZN $MODEL"

        ;;

    *)
        # Don't recognise this file, just ignore it.
        #
        echo "$THIS: ignoring unrecognised file $FILENAME" >&2
        exit 1

        ;;

esac

# Run the solver and generate the .out and .err files.
#
echo -n "$THIS: $CMD"
time-and-mem-limit $TIMELIMIT $MEMLIMIT $CMD >$OUT 2>$ERR
#ulimit -d $MEMLIMITKB -m $MEMLIMITKB -v $MEMLIMITKB -t $TIMELIMIT
#$CMD >$OUT 2>$ERR

# Compare the solver output against the expected output.
#
PASSFAIL=fail

if [ -n "$EXPS" ]
then
    for EXP in $EXPS
    do
        if diff --brief --ignore-all-space \
          -I 'minizinc.exe: flattening killed by signal.*' \
          $OUT $EXP >/dev/null 2>&1
        then
            PASSFAIL=pass
            break
        fi
    done
fi

if [ -n "$ERREXPS" ]
then
    for ERREXP in $ERREXPS
    do
        if diff -I "^\(IBM \)\?ILOG.*CPLEX.*" --brief --ignore-all-space \
           -I 'minizinc.exe: flattening killed by signal.*' \
                $ERR $ERREXP >/dev/null 2>&1
        then
            PASSFAIL=pass
            break
        fi
    done
fi

# Clean up after success or report failure.
#
if [ "$PASSFAIL" = "pass" ]
then
    echo " ... passed"
    rm -f $OUT $ERR
else
    echo " ... failed"
    echo "$THIS: no match against expected outputs"
    echo "$THIS: ---- $OUT ----"
    cat $OUT
    echo "$THIS: ---- $ERR ----"
    cat $ERR
    echo "$THIS: ---- end of output ----"
    exit 1
fi