#! /bin/sh

export LANG=C

date
echo
echo Hard limits:
ulimit -aH
echo

#ulimit -c unlimited
ulimit -s unlimited
ulimit -d unlimited
#ulimit -m unlimited
#ulimit -v unlimited
#ulimit -f unlimited
ulimit -t unlimited

echo
echo Soft limits:
ulimit -a
echo

#echo $LD_LIBRARY_PATH

# Intel MPI version 19 ff need some more environment massage
mpiexecver=`mpiexec --version | awk '/Intel.R. MPI Library for Linux/ {print $8;}'`
mpiexecupd=`mpiexec --version | awk '/Intel.R. MPI Library for Linux/ {print $10;}'`
#mpiexecbld=`mpiexec --version | awk '/Intel.R. MPI Library for Linux/ {print $12;}'`

case "${mpiexecver}-${mpiexecupd}" in
    2015-*|2016-*|2017-*|2018-*)
 	echo 'This MPI version does not support MPI_Comm_Spawn within slurm ??'
 	;;
   2019-*|2021-*|2022-*)
unset I_MPI_DAPL_UD
unset I_MPI_DAPL_UD_PROVIDER
export I_MPI_FABRICS=shm:ofi
export FI_MLX_ENABLE_SPAWN=yes
#export I_MPI_HYDRA_DEBUG=yes
;;
    2021.6-*)
unset I_MPI_DAPL_UD
unset I_MPI_DAPL_UD_PROVIDER
export I_MPI_FABRICS=shm:ofi
export I_MPI_SPAWN=on
;;
*)
	echo dont know how to handle this version of mpiexec
	mpiexec --version
	# do nothing special
;;
esac

unset I_MPI_PMI_LIBRARY

  for n in 16 24 28 30 32 36 40 48 64
  do
  time mpiexec.hydra -genvall -n $n \
     -outfile-pattern fms.out-$SLURM_JOBID-$SLURM_NNODES-$n-'%r' \
     -errfile-pattern fms.out-$SLURM_JOBID-$SLURM_NNODES-$n-'%r' \
   ./test_MPI_Spawn
  success=$?
  echo Running with $n CPUs exited with $success
  done