#!/bin/csh -f
# Minimal LoadLeveler submission script for MOM experiments

set type          = MOM_SIS       # type of the experiment
set name          = box1           
set platform      = gfortran     # A unique identifier for your platform
if ( `hostname` =~ login0? ) then
  set platform      = pik-iplex-ifort11
endif
if ( `hostname` =~ viss0?* ) then
  set platform      = pik-vis-ifort11
endif
if ( `hostname` =~ pc89 ) then
  set platform      = pik-pc89-gfortran
endif
if ( `hostname` =~ zuse ) then
  set platform      = ifi-zuse-ifort11
endif

set npes          = 8              # number of processor
                                   # Note: If you change npes you may need to change
                                   # the layout in the corresponding namelist
set valid_npes = 0
set help = 0
set download = 0
set argv = (`getopt -u -o h -l type: -l platform: -l npes: -l experiment: -l debug  -l help -l download_input_data --  $*`)
while ("$argv[1]" != "--")
    switch ($argv[1])
        case --type:
                set type = $argv[2]; shift argv; breaksw    
        case --platform:
                set platform = $argv[2]; shift argv; breaksw
        case --npes:
                set npes = $argv[2]; shift argv; breaksw
        case --experiment:
                set name = $argv[2]; shift argv; breaksw
        case --debug:
                set debug = 1;  breaksw
        case --help:
                set help = 1;  breaksw
        case -h:
                set help = 1;  breaksw
#        case --download_input_data:
#                set download = 1;  breaksw
    endsw
    shift argv
end
shift argv

if ( $help ) then
    echo "The optional arguments are:"
    echo "--type       followed by the type of the experiment, currently one of the following:"
    echo "             MOM_solo : solo ocean model"
    echo "             MOM_SIS  : ocean-seaice model"
    echo "             CM2M     : ocean-seaice-land-atmosphere coupled climate model"
    echo "             ESM2M    : ocean-seaice-land-atmosphere coupled climate model with biogeochemistry, EarthSystemModel"
    echo "             ICCM     : ocean-seaice-land-atmosphere coupled model"
    echo "             MOM_LAD_AEOLUS     : ocean-seaice with land_lad and Aeolus"
    echo 
    echo "--experiment followed by the name of the experiment of the specified type"
    echo "             To see the list of available experiments for each type use  -h --type type_name"
    if ( $type == MOM_solo ) then    
    echo "             Available expeiments for MOM_solo:"
    echo "             box1, box_channel1, bowl1, dome1, gyre1, iom1, mk3p51, symmetric_box1, torus1, dome_bates_blobs1"
    endif
    if ( $type == MOM_SIS ) then    
    echo "             Available expeiments for MOM_SIS:"
    echo "             om3_core1, om3_core3, MOM_SIS_TOPAZ, MOM_SIS_BLING, atlantic1, om3Xcoarse"
    endif
    if ( $type == CM2M ) then    
    echo "             Available expeiments for CM2M:"
    echo "             CM2.1p1, CM2M_coarse_BLING"
    endif
    if ( $type == ESM2M ) then    
    echo "             Available expeiments for ESM2M:"
    echo "             ESM2M_pi-control_C2"
    endif
    if ( $type == ICCM ) then    
    echo "             Available expeiments for ICCM:"
    echo "             ICCMp1"
    endif
    if ( $type == EBM ) then    
    echo "             Available expeiments for EBM:"
    echo "             mom4p1_ebm1"
    endif
    if ( $type == MOM_LAD_AEOLUS ) then    
    echo "             Available expeiments for MOM_LAD_AEOLUS:"
    echo "             mom4_Aeolus_lad_Xcoarse"
    endif
    echo 
    echo 
    echo "--platform   followed by the platform name that has a corresponfing environ file in the ../bin dir, default is ncrc.intel"
    echo 
    echo "--npes       followed by the number of pes to be used for this experiment"
    echo 
    echo Note that the executable for the run should have been built before calling this script. This could be done by calling the appropriate compile script for this experiment \"type\" beforehand.
    echo 
    echo 
    exit 0
endif

set root          = $cwd:h         # The directory in which you checked out src
set code_dir      = $root/src                         # source code directory
set workdir       = $root/work     # where the model is run and model output is produced
                                   # This is recommended to be a link to the $WORKDIR of the platform.
set expdir        = $workdir/$name
set inputDataDir  = $expdir/INPUT   # This is path to the directory that contains the input data for this experiment.
                                     # You should have downloaded and untared this directory from MOM4p1 FTP site.
set diagtable     = $inputDataDir/diag_table  # path to diagnositics table
set datatable     = $inputDataDir/data_table  # path to the data override table.
set fieldtable    = $inputDataDir/field_table # path to the field table
set namelist      = $inputDataDir/input.nml   # path to namelist file

set executable    = $root/exec/$platform/$type/fms_$type.x      # executable created after compilation

#set archive       = $ARCHIVE/$type #Large directory to host the input and output data.



#===========================================================================
# The user need not change any of the following
#===========================================================================

#
# Users must ensure the correct environment file exists for their platform.
#
source $root/bin/environs.$platform  # environment variables and loadable modules

set mppnccombine  = $root/bin/mppnccombine.$platform  # path to executable mppnccombine
set time_stamp    = $root/bin/time_stamp.csh          # path to cshell to generate the date

set echo

# Check if the user has extracted the input data
  if ( ! -d $root/exp/$name/INPUT ) then

#    if( $download ) then
#	cd $root/data/archives
#	git annex get $name.input.tar.gz
#	mkdir -p $workdir
#	cp $name.input.tar.gz $workdir
#	cd $workdir
#	tar zxvf $name.input.tar.gz
#    else  

    echo "ERROR: the experiment directory '$inputDataDir' does not exist or does not contain input and preprocessing data directories!"
    echo "Please copy the input data from the MOM data directory. This may required downloading data from a remote git annex if you do not already have the data locally."
    #echo "cd $root/data/archives"
    #echo "git annex get $name.input.tar.gz"
    #echo "mkdir -p $workdir"
    #echo "cp $name.input.tar.gz $workdir"
    #echo "cd $workdir"
    #echo "tar zxvf $name.input.tar.gz"
    #echo "Or use the --download option to do this automatically"
    exit 1

#    endif
  endif

  if ( -e $inputDataDir ) then
    echo "NOTE: the work directory $inputDataDir:h already exists - not linking INPUT/ directory contents"
  else
    mkdir -p $inputDataDir
    ( cd $inputDataDir ; ln -s $root/exp/$name/INPUT/* . )
    cd $expdir
    cp -p $root/exp/$name/INPUT/*_table .
    cp -p $root/exp/$name/INPUT/input.nml .
  endif

# setup directory structure
  if ( ! -e $expdir )         mkdir -p $expdir
  if ( ! -e $expdir/RESTART ) mkdir -p $expdir/RESTART
  if ( ! -e $expdir/history ) mkdir -p $expdir/history

#
#Check the existance of essential input files
#
#  if ( ! -e $inputDataDir/grid_spec.nc ) then
#    echo "ERROR: required input file does not exist $inputDataDir/grid_spec.nc "
#    exit 1
#  endif
#  if ( ! -e $inputDataDir/ocean_temp_salt.res.nc ) then
#    echo "ERROR: required input file does not exist $inputDataDir/ocean_temp_salt.res.nc "
#    exit 1
#  endif



# --- make sure executable is up to date ---
  set makeFile = Makefile
  cd $executable:h
  make -f $makeFile
  if ( $status != 0 ) then
    unset echo
    echo "ERROR: make failed"
    exit 1
  endif
#-------------------------------------------

#Change to expdir

  cd $expdir

# Create INPUT directory. Make a link instead of copy
# 
#if ( ! -d $expdir/INPUT   ) mkdir -p $expdir/INPUT

  if ( ! -e $namelist ) then
    echo "ERROR: required input file does not exist $namelist "
    exit 1
  endif
  if ( ! -e $datatable ) then
    echo "ERROR: required input file does not exist $datatable "
    exit 1
  endif
  if ( ! -e $diagtable ) then
    echo "ERROR: required input file does not exist $diagtable "
    exit 1
  endif
  if ( ! -e $fieldtable ) then
    echo "ERROR: required input file does not exist $fieldtable "
    exit 1
  endif

#  cp $namelist   input.nml
#  cp $datatable  data_table
#  cp $diagtable  diag_table
#  cp $fieldtable field_table 

#Preprocessings
  echo "using preprocessing.csh to convert input.nml from mom4p1 to mom5"
  $root/exp/preprocessing.csh
  
if ( $type == CM2M ) then
    if ( $name == CM2.1p1 & $npes != 45 ) set valid_npes = 45
    if ( $name == CM2M_coarse_BLING & $npes != 50 ) set valid_npes = 50
endif

if ( $type == ESM2M & $npes != 120 ) then
    set valid_npes = 120
endif
if ( $type == ICCM & $npes != 45 ) then
    set valid_npes = 45
endif
if ( $name  == atlantic1 & $npes != 24) then
    set valid_npes = 24
endif
if ( $name  == mom4p1_ebm1 & $npes != 17) then
    set valid_npes = 17
endif

if ( $type == MOM_LAD_AEOLUS & npes != 31 ) set valid_npes = 31

if ( $valid_npes ) then
    echo "ERROR: This experiment is designed to run on $valid_npes pes. Please specify --npes  $valid_npes "
    echo "Note:  In order to change the default npes for an expeiment the user may need to edit the values of layouts and atmos_npes and ocean_npes in the input.nml and run the mpi command manually in the working dir"
    exit 0 
endif

  
#   --- submit the model run job ---

  cd $expdir

  cp -p $executable $executable:t

# sigh. llsubmit 3.4.0.0 coredumps instead of read from stdin.
# thus workaround: use a temp file

# Note; this example case uses raw binary restart files, that were written
# in big endian order, but Intel native order is little endian.
# Thus we need to set the environment variable
# F_UFMTENDIAN=big . We do this via parameter -genv to the mpiexec invocation.

# In the hope to improve performance on PIKs IPLEX cluster,
# we allocate only entire nodes of 8 CPU cores each.
# Thus we need to round up the number of CPUs to the next multiple of 8.
@ npes_rounded = ( $npes / 8 ) * 8
@ frac = $npes % 8
if ( $frac ) @ npes_rounded += 8

# TODO: try to edit the npes and layout settings in the input.nml
# according to the specified npes

( \
  echo '#\! /bin/sh' ; \
  echo "npes=$npes" ; \
  echo "npres_rounded=$npes_rounded" ; \
  echo '# @ group = climber3' ; \
  echo '# @ job_type = parallel' ; \
  echo "# @ total_tasks = $npes_rounded " ; \
  echo "# @ blocking = 8" ; \
  echo '# @ requirements = (Feature == "blue")' ; \
  echo '# @ class = short' ; \
  echo '# @ wall_clock_limit= 24:00:00 ' ; \
  echo '# @ as_limit = unlimited' ; \
  echo '# @ data_limit = unlimited' ; \
  echo '# @ stack_limit = unlimited' ; \
  echo '# @ file_limit = unlimited' ; \
  echo '# @ nofile_limit = unlimited' ; \
  echo '# @ core_limit = unlimited' ; \
  echo '# @ output = fms.'$npes'.$(Host).$(Cluster).$(Process).out' ; \
  echo '# @ error = fms.'$npes'.$(Host).$(Cluster).$(Process).err' ; \
  echo '# @ notification = always' ; \
  echo '# @ queue' ; \
  echo '' ; \
  echo '# we must prevent two jobs working in the same directory concurrently' ; \
  echo '# thus we use a symbolic link as atomic lock, which should also work' ; \
  echo '# well across network-shared directories' ; \
  echo 'ln -s $LOADL_STEP_ID .workdir_locked_by_batchjob' ; \
  echo 'if [ 0 != $? ]' ; \
  echo 'then' ; \
  echo '  # sigh. it might be a restart attempt initated by loadleveler after a vacate order' ; \
  echo '  # in that case the job ID is still the same that created the existing lock file.' ; \
  echo '  # And if so, we just start anew silently.' ; \
  echo '  oldjobid="`ls -l .workdir_locked_by_batchjob | awk '\{print \$11\}'`"' ; \
  echo '  if [ "$oldjobid" != "$LOADL_STEP_ID" ]' ; \
  echo '  then' ; \
  echo '    echo Error: Directory `pwd` is locked by another batch job: ' ; \
  echo '    ls -l .workdir_locked_by_batchjob' ; \
  echo '    exit 1' ; \
  echo '  else' ; \
  echo '    echo own lockfile already there - looks like a Loadl restart attempt >> fms.$LOADL_STEP_ID.loadl-restart.log' ; \
  echo '    echo `date` `hostname` >> fms.$LOADL_STEP_ID.loadl-restart.log' ; \
  echo '  fi' ; \
  echo 'fi' ; \
  echo '' ; \
  echo 'date' ; \
  echo 'date >&2' ; \
  echo '#ulimit -c unlimited' ; \
  echo '#ulimit -s unlimited' ; \
  echo '#ulimit -d unlimited' ; \
  echo '#ulimit -m unlimited' ; \
  echo '#ulimit -v unlimited' ; \
  echo '#ulimit -f unlimited' ; \
  echo 'ulimit -a' ; \
  echo '' ; \
  echo "mkdir -p RESTART history" ; \
  echo 'if [ $npes -gt 1 ]' ; \
  echo 'then' ; \
  echo '  llgetmachinelist | sed "s/blade/bi/"  > cpulist.$LOADL_STEP_ID' ; \
  echo '  sort -u cpulist.$LOADL_STEP_ID > mpdhosts.$LOADL_STEP_ID' ; \
  echo '  machine_count=`cat mpdhosts.$LOADL_STEP_ID  | wc -l`' ; \
  echo '  cpu_count=`cat cpulist.$LOADL_STEP_ID  | wc -l`' ; \
  echo '  export MPD_CON_EXT=$LOADL_STEP_ID' ; \
  echo '  echo $machine_count machines $cpu_count cpus >&2' ; \
  echo "  #echo MachineList:" ; \
  echo "  #echo '------------------------------------------------------------'" ; \
  echo '  sort cpulist.$LOADL_STEP_ID | uniq -c >&2' ; \
  echo "  #echo '------------------------------------------------------------'" ; \
  echo '  mpdboot -n $machine_count -r ssh -f mpdhosts.$LOADL_STEP_ID' ; \
  echo '  #export I_MPI_DEBUG=3' ; \
  echo '  # -env I_MPI_DEVICE rdma:OpenIB-cma-netdev' ; \
  echo '  # print out the selected MPI device, and the process mapping ; but 2 is too verbose' ; \
  echo '  # -env I_MPI_DEBUG 2' ; \
  echo '  time mpiexec -genv F_UFMTENDIAN big -machinefile cpulist.$LOADL_STEP_ID  -n $npes './$executable:t' > fms.out-$LOADL_STEP_ID 2>&1' ; \
  echo '  success=$?' ; \
  echo '  mpdallexit' ; \
  echo '  rm -f cpulist.$LOADL_STEP_ID' ; \
  echo '  rm -f mpdhosts.$LOADL_STEP_ID' ; \
  echo '  # Do not combine blobs history files. They need special handling.' ; \
  echo '  for i in ocean_blobs.nc.???? ; do mv $i history/$enddate.$i; done' ; \
  echo '  bases=`ls -1 *.nc.[0-9][0-9][0-9][0-9]* | sed -e "s/.nc.[0-9][0-9][0-9][0-9]\+/.nc/" | sort -u`' ; \
  echo '  for b in $bases ; do '$mppnccombine' -r $b $b.[0-9][0-9][0-9][0-9]* ; done' ; \
  echo '  # possibly TODO: run $mppnccombine on files in RESTART/*.nc.????' ; \
  echo '  # Concatenate blobs restart files. mppnccombine would not work on them.' ; \
  echo '  (cd RESTART ; ncecat ocean_blobs.res.nc.???? ocean_blobs.res.nc ; rm ocean_blobs.res.nc.????)' ; \
  echo 'else' ; \
  echo '  time './$executable:t' > fms.out-$LOADL_STEP_ID 2>&1' ; \
  echo '  success=$?' ; \
  echo 'fi' ; \
  echo '# generate date for file names' ; \
  echo 'if [ 0 = "$success" ] ; then ' ; \
  echo '  begindate=`'$time_stamp' -bf digital`' ; \
  echo '  if [ "$begindate" == "" ] ; then begindate=tmp`date '+%Y%j%H%M%S'` ; fi' ; \
  echo '  enddate=`'$time_stamp' -ef digital`' ; \
  echo '  if [ "$enddate" == "" ] ; then enddate=tmp`date '+%Y%j%H%M%S'` ; fi' ; \
  echo '  rm -f time_stamp.out' ; \
  echo '  # rename nc files with the date and move to history' ; \
  echo '  for i in *.nc ; do mv $i history/$enddate.$i; done' ; \
  echo '  # archive text log out files' ; \
  echo '  tar cvjf history/$enddate.out.tar.bz2 fms.out-$LOADL_STEP_ID *.out' ; \
  echo '  # archive corresponding RESTART files' ; \
  echo '  mv RESTART $enddate.RESTART' ; \
  echo '  cp -p input.nml data_table diag_table field_table $enddate.RESTART/.' ; \
  echo '  tar cvjf history/$enddate.RESTART.tar.bz2 $enddate.RESTART/.' ; \
  echo '  mv $enddate.RESTART RESTART' ; \
  echo 'else' ; \
  echo '  echo FMS run failed with code "$success"' ; \
  echo 'fi' ; \
  echo 'rm -f .workdir_locked_by_batchjob' ; \
  echo 'exit $success' ; \
  ; \
# possibly TODO: put results into archive \
) > submit-$$.jcf

llsubmit submit-$$.jcf
#rm -f submit-$$.jcf
