#!/usr/bin/env sh

# Copyright (c) 2007-2020. The SimGrid Team. All rights reserved.

# This program is free software; you can redistribute it and/or modify it
# under the terms of the license (GNU LGPL) which comes with this package.

export LD_LIBRARY_PATH="/usr/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"

SIMGRID_VERSION="SimGrid version 3.25"
SIMGRID_GITHASH=""

DEFAULT_LOOPBACK_BANDWIDTH="498000000Bps"
DEFAULT_LOOPBACK_LATENCY="0.000004s"
DEFAULT_NETWORK_BANDWIDTH="$((26 * 1024 * 1024))Bps"
DEFAULT_NETWORK_LATENCY="0.000005s"
DEFAULT_NUMPROCS="4"
DEFAULT_SPEED="100flops"

LOOPBACK_BANDWIDTH="${DEFAULT_LOOPBACK_BANDWIDTH}"
LOOPBACK_LATENCY="${DEFAULT_LOOPBACK_LATENCY}"
NETWORK_BANDWIDTH="${DEFAULT_NETWORK_BANDWIDTH}"
NETWORK_LATENCY="${DEFAULT_NETWORK_LATENCY}"
SPEED="${DEFAULT_SPEED}"

PRIVATIZE="--cfg=smpi/privatization:${SMPI_PRIVATIZATION:-1}"

SIMOPTS="--cfg=surf/precision:1e-9 --cfg=network/model:SMPI"

#usage to print the way this script should be called
usage () {
    cat <<EOF
Usage: $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> program [program-options]
       $0 [OPTIONS] -platform <xmldesc> -hostfile <hostfile> -replay <tracefile> [program] [program-options]
Options:
  -keep-temps                # don't remove the generated files after execution
  -wrapper <command>         # use command to run the program (e.g. "valgrind" or "gdb --args")
  -gdb                       # run within GDB (-wrapper "gdb --args" -keep-temps)
  -lldb                      # run within LLDB (-wrapper "lldb --" -keep-temps)
  -vgdb                      # run within Valgrind+GDB (-wrapper "valgrind --vgdb=yes --vgdb-error=0" -keep-temps)
  -map                       # display the machine on which each process rank is mapped
  -np <numprocs>             # use that amount of processes from the hostfile.
                             # By default, all processes of the hostfile are used.
  -no-privatize              # Disable the globals privatization, that is activated by default
  -trace-ti                  # activate time independent tracing (for replay, default in smpi_simgrid.txt)
  -trace                     # activate tracing (Paje, default in smpi_simgrid.trace)
  -trace-comment <comment>   # put a comment on the top of the trace file
  -trace-comment-file <file> # put file contents on the top of the trace file as comment
  -trace-grouped             # group MPI processes by location
  -trace-resource            # trace resource utilization
  -trace-file <tracefile>    # name of the tracefile (simgrid_smpi.trace)
  -replay <tracefile>        # replays a trace instead of actually executing an application

  -version                   # Displays the SimGrid version (human readable)
  -git-version               # Displays the git hash of SimGrid

or (deprecated usage):
  $0 [-keep-temps] [-np <numprocs>] [-bandwidth <bytes/sec>] [-latency <secs>] program [program-options]

EOF
}

#check if we have at least one parameter
if [ $# -eq 0 ]
then
    usage
    exit
fi

WRAPPER=""
HOSTFILE=""
HOSTFILETMP=0
MAPOPT=0
REPLAY=0
QUIET=""

unset pid

trapped_signals="HUP INT QUIT ILL ABRT SEGV FPE ALRM TERM USR1 USR2 BUS"

die () {
    printf '[%s] ** error: %s. Aborting.\n' "$(basename $0)" "$*" >&2
    exit 1
}

smpirun_cleanup()
{
  if [ -z "${KEEP}" ] ; then
      if [ -z "${PLATFORM}" -a -n "$PLATFORMTMP" ]; then
        rm -f ${PLATFORMTMP}
        PLATFORMTMP=""
      fi
      if [ ${HOSTFILETMP} = 1 -a -n "$HOSTFILE" ] ; then
          rm -f ${HOSTFILE}
          HOSTFILE=""
      fi
      if [ ${UNROLLEDHOSTFILETMP} = 1 -a -n "$UNROLLEDHOSTFILE" ] ; then
          rm -f ${UNROLLEDHOSTFILE}
          UNROLLEDHOSTFILE=""
      fi
      if [ -n ${APPLICATIONTMP} ]; then
        rm -f ${APPLICATIONTMP}
        APPLICATIONTMP=""
      fi
  fi
}

smpirun_trap() {
  local sig
  sig="$1"

  # Cleanup and kill the child process:
  smpirun_cleanup
  if ! [ -z "$pid" ]; then
    kill -TERM $pid
  fi
  unset pid

  # Raise the same signal again (remove the traps first):
  trap - $trapped_signals
  kill -$sig $$

  # This should never happen:
  kill -ABRT $$
  kill -TERM $$
}

for s in $trapped_signals; do
  trap "smpirun_trap $s" $s
done

while true; do
    case "$1" in
        "-np" | "-n")
            NUMPROCS="$2"
            shift 2
            ;;
        "-bandwidth")
            NETWORK_BANDWIDTH="$2"
            shift 2
            ;;
        "-latency")
            NETWORK_LATENCY="$2"
            shift 2
            ;;
        "-platform")
	    PLATFORM="$2"
            if [ ! -f "${PLATFORM}" ]; then
                die "the file '${PLATFORM}' does not exist"
            fi
	    shift 2
            ;;
        "-hostfile" | "-machinefile")
	    HOSTFILE="$2"
            if [ ! -f "${HOSTFILE}" ]; then
                die "the file '${HOSTFILE}' does not exist"
            fi
	    shift 2
            ;;
        "-replay")
            APP_TRACES="$2"
            shift 2
            ;;
	"-no-privatize")
	    PRIVATIZE="--cfg=smpi/privatization:no"
	    shift 1
	    ;;
        "-map")
	    MAPOPT=1
            shift 1
            ;;
        "-trace")
            TRACE_ACTIVE="true"
            shift 1
            ;;
        "-trace-ti")
            TRACE_ACTIVE="true"
            TRACE_TI_ACTIVE="true"
            shift 1
            ;;
        "-trace-comment")
            TRACE_COMMENT="$2"
            shift 2
            ;;
        "-trace-comment-file")
            TRACE_COMMENT_FILE="$2"
            shift 2
            ;;
        "-trace-file")
            TRACE_FILENAME="$2"
            shift 2
            ;;
        "-trace-grouped")
            TRACE_GROUPED="true"
            shift 1
            ;;
        "-trace-resource")
            TRACE_RESOURCE="true"
            shift 1
            ;;
        "-keep-temps")
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
	"-quiet")
	    QUIET="true"
	    shift 1
	    ;;
        "-wrapper")
            WRAPPER="$2"
            shift 2
            ;;
        "-gdb")
            WRAPPER="gdb --args"
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-vgdb")
            WRAPPER="valgrind --vgdb=yes --vgdb-error=0"
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-lldb")
            WRAPPER="lldb --"
            KEEP="true"
            SIMOPTS="$SIMOPTS --cfg=smpi/keep-temps:yes"
            shift 1
            ;;
        "-help" | "--help" | "-h")
            usage
            exit 0
            ;;
        "-version" | "--version" | "-v")
            printf '%b\n' "$SIMGRID_VERSION"
            exit 0
            ;;
        "-git-version" | "--git-version")
            printf '%b\n' "$SIMGRID_GITHASH"
            exit 0
            ;;
        "--cfg="*|"--log="*)
            for OPT in ${1#*=}
            do
                SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
            done
            shift 1
            ;;
        "-foreground")
            # Nothing to do, compatibility.
            shift 1
            ;;
        *)
            break
            ;;
    esac
done

if [ -n "${APP_TRACES}" ] ; then
    if [ $# -eq 0 ] ; then
        EXEC="/usr/lib/simgrid/smpireplaymain"
    else
        EXEC="$1"
        shift
    fi
else
    # check if we still have at least one parameter beyond options
    if [ $# -eq 0 ]
    then
        echo "Error: no program to execute!"
        usage
        exit
    fi

    EXEC="$1"
    shift
fi

# steal --cfg and --logs options
while [ $# -gt 0 ]; do
    case "$1" in
        "--cfg="*|"--log="*)
            for OPT in ${1#*=}
            do
                SIMOPTS="$SIMOPTS ${1%%=*}=$OPT"
            done
            shift 1
            ;;
        *)
            PROC_ARGS="${PROC_ARGS:+$PROC_ARGS }$1"
            shift
            ;;
    esac
done

if [ -z "${HOSTFILE}" ] && [ -z "${PLATFORM}" ] ; then
    echo "No hostfile nor platform specified."
    usage
    exit 1
fi

if [ -z "${HOSTFILE}" ] ; then
    HOSTFILETMP=1
    HOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
    /usr/bin/python3 -c '
import xml.etree.ElementTree as ET
import sys
import re

tree = ET.parse(sys.stdin)

for elem in tree.findall(".//host"):
    print(elem.attrib["id"])

for elem in tree.findall(".//cluster"):
    prefix = elem.attrib["prefix"]
    radical = elem.attrib["radical"]
    suffix = elem.attrib["suffix"]
    for r in radical.split(","):
        m = re.match("^([^-]*?)-([^-]*)$", r)
        if m:
            for i in range(int(m.group(1)), int(m.group(2))):
                print(prefix + str(i) + suffix)
        else:
            print(prefix + r + suffix)
            ' < ${PLATFORM} > ${HOSTFILE}
fi
UNROLLEDHOSTFILETMP=0

# parse if our lines are terminated by :num_process
if grep -q ':' $HOSTFILE ; then
    UNROLLEDHOSTFILETMP=1
    UNROLLEDHOSTFILE="$(mktemp smpitmp-hostfXXXXXX)"
    /usr/bin/python3 -c '
import sys
import re

for line in sys.stdin:
    m = re.match("(.*):(.*)", line)
    if m:
        for i in range(0, int(m.group(2))):
            print(m.group(1))
    else:
        print(line.strip())
' < ${HOSTFILE}  > ${UNROLLEDHOSTFILE}
    if [ ${HOSTFILETMP} = 1 ] ; then
        rm ${HOSTFILE}
        HOSTFILETMP=0
    fi
    HOSTFILE=$UNROLLEDHOSTFILE
fi

# Don't use wc -l to compute it to avoid issues with trailing \n at EOF
hostfile_procs=$(grep -c "[a-zA-Z0-9]" $HOSTFILE)
if [ ${hostfile_procs} = 0 ] ; then
    die "the hostfile '${HOSTFILE}' is empty"
fi

if [ -z "${NUMPROCS}" ] ; then
    # Use the amount of processes in the hostfile as default value for the -np parameter
    NUMPROCS=$hostfile_procs
fi

if [ ${NUMPROCS} -gt ${hostfile_procs} ] ; then
    echo "You requested to use ${NUMPROCS} ranks, but there is only ${hostfile_procs} processes in your hostfile..." >&2
fi

##-------------------------------- DEFAULT or SPECIFIED PLATFORM --------------------------------------
if [ -z "${PLATFORM}" ]; then
    PLATFORMTMP="$(mktemp smpitmp-platfXXXXXX)"

    cat > ${PLATFORMTMP} <<PLATFORMHEAD
<?xml version='1.0'?>
<!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
<platform version="4.1">
<zone id="AS0" routing="Full">
PLATFORMHEAD

    i=${NUMPROCS}
    while [ $i -gt 0 ]; do
        echo "  <host id=\"host$i\" speed=\"${SPEED}\"/>" >> ${PLATFORMTMP}
        echo "  <link id=\"loop$i\" bandwidth=\"${LOOPBACK_BANDWIDTH}\" latency=\"${LOOPBACK_LATENCY}\"/>" >> ${PLATFORMTMP}
        echo "  <link id=\"link$i\" bandwidth=\"${NETWORK_BANDWIDTH}\" latency=\"${NETWORK_LATENCY}\"/>" >> ${PLATFORMTMP}
        i=$((i - 1))
    done

    i=${NUMPROCS}
    while [ $i -gt 0 ]; do
        j=${NUMPROCS}
        while [ $j -gt 0 ]; do
            if [ $i -eq $j ]; then
                echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"loop$i\"/></route>" >> ${PLATFORMTMP}
            else
                echo "  <route src=\"host$i\" dst=\"host$j\"><link_ctn id=\"link$i\"/><link_ctn id=\"link$j\"/></route>" >> ${PLATFORMTMP}
            fi
            j=$((j - 1))
        done
        i=$((i - 1))
    done

    cat >> ${PLATFORMTMP} <<PLATFORMFOOT
</zone>
</platform>
PLATFORMFOOT

else
    PLATFORMTMP=${PLATFORM}
fi
##-------------------------------- end DEFAULT or SPECIFIED PLATFORM --------------------------------------
##-------------------------------- DEFAULT APPLICATION --------------------------------------
APPLICATIONTMP="$(mktemp smpitmp-appXXXXXX)"
#APPLICATIONTMP="app.xml"

cat > ${APPLICATIONTMP} <<APPLICATIONHEAD
<?xml version='1.0'?>
<!DOCTYPE platform SYSTEM "https://simgrid.org/simgrid.dtd">
<platform version="4.1">
APPLICATIONHEAD

##---- cache hostnames of hostfile---------------
if [ -n "${HOSTFILE}" ] && [ -f ${HOSTFILE} ]; then
    hostnames=$(cat ${HOSTFILE} | tr '\n\r' '  ')
fi

if [ -n "${APP_TRACES}" ]; then
    if [ -f "${APP_TRACES}" ]; then
        hosttraces=$(cat ${APP_TRACES} | tr '\n\r' '  ' )
        NUMTRACES=$(cat ${APP_TRACES} | wc -l)
        REPLAY=1
    else
        printf "File not found: %s\n" "${APP_TRACES:-\${APP_TRACES\}}" >&2
        exit 1
    fi
fi

##----------------------------------------------------------
##  generate application.xml with hostnames from hostfile:
##  the name of host_i (1<=i<=p, where -np p) is the line i in hostfile (where -hostfile hostfile), or "host$i" if
##  hostfile has less than i lines.
##----------------------------------------------------------

HAVE_SEQ="$(which seq 2>/dev/null)"

if [ -n "${HAVE_SEQ}" ]; then
    SEQ=$(${HAVE_SEQ} 0 $(( NUMPROCS - 1)))
else
    cnt=0
    while [ $cnt -lt ${NUMPROCS} ] ; do
        SEQ="$SEQ $cnt"
        cnt=$((cnt + 1));
    done
fi

set -- $hostnames

##---- generate <actor> tags------------------------------
#prepare arguments at once
for ARG in $PROC_ARGS; do
  XML_ARGS="${XML_ARGS}""<argument value=\"${ARG}\"/>
"
done

for i in ${SEQ}
do
    j=$(( i % hostfile_procs + 1 ))
    host=$(eval "echo \${$j}")

    ##---- optional display of ranks to actor mapping
    if [ ${MAPOPT} = 1 ]; then
      echo "[rank $i] -> $host"
    fi

    echo "  <actor host=\"${host}\" function=\"$i\"> <!-- function name used only for logging -->
    <prop id=\"instance_id\" value=\"smpirun\"/>
    <prop id=\"rank\" value=\"$i\"/>" >> ${APPLICATIONTMP}
    if [ ${REPLAY} = 1 ]; then
        echo "    <prop id=\"smpi_replay\" value=\"true\"/>" >> ${APPLICATIONTMP}
        if  [ ${NUMTRACES} -gt 1 ]; then
            echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f$j)\"/>" >> ${APPLICATIONTMP}
        else
            echo "    <argument value=\"$(echo $hosttraces|cut -d' ' -f1)\"/>" >> ${APPLICATIONTMP}
        fi
    else
    echo ${XML_ARGS} >> ${APPLICATIONTMP}
    fi
    echo "  </actor>" >> ${APPLICATIONTMP}
done

cat >> ${APPLICATIONTMP} <<APPLICATIONFOOT
</platform>
APPLICATIONFOOT
##-------------------------------- end DEFAULT APPLICATION --------------------------------------
##---------------------- SMPI TRACING OPTIONS ---------------------------------
if [ -n "${TRACE_ACTIVE}" ]; then
    #define trace filename
    if [ -n "${TRACE_TI_ACTIVE}" ]; then
        if [ -z "${TRACE_FILENAME}" ]; then
            TRACE_FILENAME="smpi_simgrid.txt"
        fi
        TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes --cfg=tracing/smpi/format:TI --cfg=tracing/smpi/computing:yes"
    else
        if [ -z "${TRACE_FILENAME}" ]; then
            TRACE_FILENAME="smpi_simgrid.trace"
        fi
        TRACEOPTIONS="--cfg=tracing:yes --cfg=tracing/filename:${TRACE_FILENAME} --cfg=tracing/smpi:yes"
    fi

    if [ -n "${TRACE_COMMENT}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment:${TRACE_COMMENT}"
    fi

    if [ -n "${TRACE_COMMENT_FILE}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/comment-file:${TRACE_COMMENT_FILE}"
    fi

    if [ -n "${TRACE_GROUPED}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/smpi/group:yes"
    fi

    if [ -n "${TRACE_RESOURCE}" ]; then
        TRACEOPTIONS="${TRACEOPTIONS} --cfg=tracing/categorized:yes --cfg=tracing/uncategorized:yes"
    fi
fi
##---------------------- end SMPI TRACING OPTIONS ---------------------------------

# Do not remove, this variable may be used by user code (e.g. StarPU)
export SMPI_GLOBAL_SIZE=${NUMPROCS}
if [ -n "${KEEP}" -a -z "${QUIET}" ] ; then
    echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
    if [ ${HOSTFILETMP} = 1 ] ; then
        echo "Generated hostfile ${HOSTFILE} kept."
    fi
    if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
        echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
    fi
fi

# Execute the process
#
# The shell still need to be alive for the duration in order to do some cleanup after the process.
#
# We are going through great lengths in order to both keep stdin and be able to handle signals:
#
# * The job is launched in the background in order to be able to handle signals.
#
# * The FD 3 is used to temporarily store FD 1. This is because the shell connects FD 1 to /dev/null when the command
#   is launched in the background: this can be overridden in bash but not in standard bourne shell.
exec 3<&0
${WRAPPER} "/usr/lib/simgrid/smpimain" ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP} <&3 3>&- &
pid=$!
exec 3>&-
wait $pid
status=$?
# With dash on Windows WSL/Ubuntu, "wait" sometimes returns early with an exit
# status of 128. Try again.
while test $status -eq 128 && kill -0 $pid 2>/dev/null; do
    wait $pid
    status=$?
done
pid=""

# Keep temporary files on failures to help debugging
#
if [ ${status} -ne 0 ] ; then
    if [ -z "${KEEP}" -a -z "${QUIET}" ]; then
        echo ${EXEC} ${PRIVATIZE} ${TRACEOPTIONS} ${SIMOPTS} ${PLATFORMTMP} ${APPLICATIONTMP}
        if [ ${HOSTFILETMP} = 1 ] ; then
            echo "Generated hostfile ${HOSTFILE} kept."
        fi
        if [ ${UNROLLEDHOSTFILETMP} = 1 ] ; then
            echo "Generated unrolled hostfile ${UNROLLEDHOSTFILE} kept."
        fi
        KEEP=true
    fi
    echo "Execution failed with code ${status}."
fi

smpirun_cleanup

exit $status
