#!/bin/csh -f
# Minimal runscript: ICCMp1

if ( ! "$?1" || "" == "$1" || ! "$?2" || "" == "$2" ) then
  echo "usage: $0 number_of_CPUs variant_layout"
  exit 1
endif

#set platform      = ia64           # A unique identifier for your platform
if ( `hostname` =~ login0?a ) then
  set platform      = pik-clme-ifort
endif
if ( `hostname` =~ login0? ) then
  set platform      = pik-iplex-ifort11
endif
if ( `hostname` =~ viss0? ) then
  set platform      = pik-vis-ifort11
endif

set scratchdir    = /scratch/01/petri/mom4/

set name          = ICCMp1        # Name of the experiment you want to run
                                   # Currently one of: CM2.1p1
set variant       = $2             # Variant of the experiment you want to run (namelist to use)
set npes          = $1             # number of processor
                                   # Note: If you change npes you may need to change
                                   # the layout in the corresponding namelist
set type          = ICCMp1         # type of the experiment
set root          = $cwd:h         # The directory in which you checked out src
set code_dir      = $root/src      # source code directory
set workdir       = $root/work     # where the model is run and model output is produced
                                   # This is recommended to be a link to the $WORKDIR of the platform.
set expdir        = $workdir/$name
#set expdir       = $scratchdir/$name/workdir$npes-$variant # where the model is run and model output is produced
set inputDataDir  = $expdir/INPUT   # This is path to the directory that contains the input data for this experiment.
                                     # You should have downloaded and untared this directory from MOM4p1 FTP site.
set diagtable     = $inputDataDir/diag_table  # path to diagnositics table
set datatable     = $inputDataDir/data_table  # path to the data override table.
set fieldtable    = $inputDataDir/field_table # path to the field table
set namelist      = $inputDataDir/input.nml   # path to namelist file

set executable    = $root/exec_$platform/$type/fms_$type.x      # executable created after compilation
set static        = 0              # 1 if you want static memory allocation, 0 for dynamic
if($static)    set executable = $root/exec_$platform/${type}_static/fms_$type.x 

set archive       = /scratch/01/XXXXX/mom4p1_pubrel_dec2009/$name #Large directory to host the input and output data.

#===========================================================================
# The user need not change any of the following
#===========================================================================

#
# Users must ensure the correct environment file exists for their platform.
#
source $root/bin/environs.$platform  # environment variables and loadable modules

set mppnccombine  = $root/bin/mppnccombine.$platform  # path to executable mppnccombine
set time_stamp    = $root/bin/time_stamp.csh          # path to cshell to generate the date

# Check if the user has extracted the input data
  if ( ! -d $inputDataDir ) then

#    if( -e $archive/$name.input.tar.gz ) then
#      cd $workdir
#      tar zxvf $archive/$name.input.tar.gz
#    else  

    echo "ERROR: the experiment directory '$inputDataDir' does not exist or does not contain input and preprocessing data directories!"
    echo "Please make sure that the variable 'name' in this script is set to one of ICCMp1 ."
    echo "Then download and extract here the tar ball corresponding to this experiment from GFDL anonymous ftp site!"
   echo " cd  $workdir"
    echo " wget ftp.gfdl.noaa.gov:/perm/MOM4/mom4p1_pubrel_dec2009/exp/$name.input.tar.gz"
    echo " tar zxvf $name.input.tar.gz" 
    echo "Then rerun this script."
    exit 1

#    endif
  endif

set echo

# setup directory structure
  if ( ! -d $expdir )         mkdir -p $expdir
  if ( ! -d $expdir/RESTART ) mkdir -p $expdir/RESTART

# --- make sure executable is up to date ---
  set makeFile      = Make_$type
  cd $executable:h
  make -f $makeFile
  if ( $status != 0 ) then
    unset echo
    echo "ERROR: make failed"
    exit 1
  endif
#-------------------------------------------

#Change to expdir

  cd $expdir

# Create INPUT directory. Make a link instead of copy
# 
if ( ! -d $expdir/INPUT   ) mkdir -p $expdir/INPUT

  if ( ! -e $namelist ) then
    echo "ERROR: required input file does not exist $namelist "
    exit 1
  endif
  if ( ! -e $datatable ) then
    echo "ERROR: required input file does not exist $datatable "
    exit 1
  endif
  if ( ! -e $diagtable ) then
    echo "ERROR: required input file does not exist $diagtable "
    exit 1
  endif
  if ( ! -e $fieldtable ) then
    echo "ERROR: required input file does not exist $fieldtable "
    exit 1
  endif

  cp -p $namelist   input.nml
  cp -p $datatable  data_table
  cp -p $diagtable  diag_table
  cp -p $fieldtable field_table 


#   --- submit the model run job ---

  cd $expdir

  cp -p $executable $executable:t

# sigh. llsubmit 3.4.0.0 coredumps instead of read from stdin.
# thus workaround: use a temp file

# Note; this example case uses raw binary restart files, that were written
# in big endian order, but Intel native order is little endian.
# Thus we need to set the environment variable
# F_UFMTENDIAN=big . We do this via parameter -genv to the mpiexec invocation.

# In the hope to improve performance on PIKs IPLEX cluster,
# we allocate only entire nodes of 8 CPU cores each.
# Thus we need to round up the number of CPUs to the next multiple of 8.
@ npes_rounded = ( $npes / 8 ) * 8
@ frac = $npes % 8
if ( $frac ) @ npes_rounded += 8

( \
  echo '#\! /bin/sh' ; \
  echo '# @ group = climber3' ; \
  echo '# @ job_type = parallel' ; \
  echo "# @ total_tasks = $npes_rounded " ; \
  echo "# @ blocking = 8" ; \
  echo '# @ requirements = (Feature == "blue")' ; \
  echo '# @ class = short' ; \
  echo '# @ wall_clock_limit= 24:00:00 ' ; \
  echo '# @ as_limit = unlimited' ; \
  echo '# @ data_limit = unlimited' ; \
  echo '# @ stack_limit = unlimited' ; \
  echo '# @ file_limit = unlimited' ; \
  echo '# @ nofile_limit = unlimited' ; \
  echo '# @ core_limit = unlimited' ; \
  echo '# @ output = fms.'$npes'.$(Host).$(Cluster).$(Process).out' ; \
  echo '# @ error = fms.'$npes'.$(Host).$(Cluster).$(Process).err' ; \
  echo '# @ notification = always' ; \
  echo '# @ queue' ; \
 ; \
# we must prevent two jobs working in the same directory concurrently \
# thus we use a symbolic link as atomic lock, which should also work \
# well across network-shared directories \
  echo 'ln -s $LOADL_STEP_ID .workdir_locked_by_batchjob || { echo Error: Directory `pwd` is locked by another batch job: ; ls -l .workdir_locked_by_batchjob; exit 1 ; }' ; \
  ; \
  echo 'date' ; \
  echo 'date >&2' ; \
  echo '#ulimit -c unlimited' ; \
  echo '#ulimit -s unlimited' ; \
  echo '#ulimit -d unlimited' ; \
  echo '#ulimit -m unlimited' ; \
  echo '#ulimit -v unlimited' ; \
  echo '#ulimit -f unlimited' ; \
  echo 'ulimit -a' ; \
  ; \
  echo "if [ $npes -gt 1 ]" ; \
  echo 'then' ; \
  echo '  llgetmachinelist | sed "s/blade/bi/"  > cpulist.$LOADL_STEP_ID' ; \
  echo '  sort -u cpulist.$LOADL_STEP_ID > mpdhosts.$LOADL_STEP_ID' ; \
  echo '  machine_count=`cat mpdhosts.$LOADL_STEP_ID  | wc -l`' ; \
  echo '  cpu_count=`cat cpulist.$LOADL_STEP_ID  | wc -l`' ; \
  echo '  export MPD_CON_EXT=$LOADL_STEP_ID' ; \
  echo '  echo $machine_count machines $cpu_count cpus >&2' ; \
# echo "  echo MachineList:" ; \
# echo "  echo '------------------------------------------------------------'" ; \
  echo '  sort cpulist.$LOADL_STEP_ID | uniq -c >&2' ; \
# echo "  echo '------------------------------------------------------------'" ; \
  echo '  mpdboot -n $machine_count -r ssh -f mpdhosts.$LOADL_STEP_ID' ; \
# echo '  export I_MPI_DEBUG=3' ; \
# -env I_MPI_DEVICE rdma:OpenIB-cma-netdev \
# print out the selected MPI device, and the process mapping ; but 2 is too verbose \
# -env I_MPI_DEBUG 2 \
  echo '  time mpiexec -genv F_UFMTENDIAN big -genv I_MPI_DEBUG 2 -machinefile cpulist.$LOADL_STEP_ID  -n '$npes' '$executable:t ; \
  echo '  mpdallexit' ; \
  echo '  rm -f cpulist.$LOADL_STEP_ID' ; \
  echo '  rm -f mpdhosts.$LOADL_STEP_ID' ; \
  echo '  bases=`ls -1 *.nc.[0-9][0-9][0-9][0-9]* | sed -e "s/.nc.[0-9][0-9][0-9][0-9]\+/.nc/" | sort -u`' ; \
  echo '  for b in $bases ; do '$mppnccombine' -v -r $b $b.[0-9][0-9][0-9][0-9]* ; done' ; \
# possibly TODO: run $mppnccombine on files in RESTART/*.nc.???? \
  echo 'else' ; \
  echo '   time '$executable:t ; \
  echo 'fi' ; \
  echo 'rm -f .workdir_locked_by_batchjob' ; \
  ; \
# possibly TODO: rename output files with timestamp \
# possibly TODO: put results into archive \
) > submit-$$.jcf

llsubmit submit-$$.jcf
#rm -f submit-$$.jcf
