#!/bin/tcsh 
#
#  last update = 8 June 2010
#
#  USE: Run GAMESS
#  AUDIENCE: System Admins
#
#  It is very unlikely that a typical user will call this script
#  on their own.  Typically, another script that will create a
#  PBS batch script will make a call to this script in the crated
#  batch script.
#
#  Example:
#  > qgms input.inp
#  [user answers some prompts]
#  qgms creates input.pbs which calls rungms
#  qgms submits input.pbs to the queue and deletes input.pbs
#
# ===========================================================================
#
#  CALLING ARGUMENTS: 
#  JOB    is the name of the 'JOB.inp' file to be executed,
#  VERNO  is the number of the executable you chose at 'lked' time,
#  NNODES is the number of cores to be used.
#
#
#  The installer needs to: 
#    a) choose the target for execution:  cray-xt (this is done for you)
#    b) choose a SCRDIR directory where large temporary files can reside.
#       this should be fastest possible disk access, and very spacious.
#       Users must be able to create a directory in $SCRDIR.  This script
#       will create a unique directory in $SCRDIR for each GAMESS calcuation
#       running on the machine.
#    c) choose the directories in which the GAMESS binary and ericfmt.dat
#       have been installed.  These may differ.
#
set TARGET  = cray-xt
set SCRDIR  = $SCRATCH
set ERICPTH = $GAMESS_HOME/common
set GMSPATH = $GAMESS_HOME/bin

if ($#argv == 3) then
  set APRUN = "aprun -n $3"
  echo $APRUN
else if ($#argv == 4) then
  set APRUN = "aprun -n $3 -N $4"
  echo $APRUN
else if ($#argv == 5) then
  set APRUN = "aprun -n $3 -N $4 -S $5"
  echo $APRUN
else
  set APRUN = "aprun -n $3 -N $4 -S $5 -d $6"
  echo $APRUN
endif

#
# Other options
#
set GMS_USE_OUTPUT=false
set GMS_USE_MINI_INPUT=false
#
# `get_scratch_dir` is a script which returns the name of the luster file
#  system that will be used for scratch storage.  You can hardcode a value
#  or use a script of your own creation.
#
set JOB=$1       # name of the input file xxx.inp, give only the xxx part
set VERNO=$2     # revision number of the executable created by 'lked' step
set NNODES=$3    # number of compute processes to be run

# provide defaults if last two arguments are not given to this script
if (null$VERNO == null) set VERNO=00
if (null$NNODES == null) set NNODES=1
#
set SCR=/scr/$USER
set SCR_MASTER=$SCR
#
# *************************** CRAY XT ************************************
#
#  the GAMESS scripts and bin directories should be added to the PATH
#  environmental variable when the module is loaded.
#
if($TARGET == cray-xt) then
#
#  In GAMESS there are a variety of different files in the scratch/working
#  directory.  Some of these files are only accessed by the master process
#  yet others are created and used by all processes.  Under the Lustre 
#  file system, different stripe settings will better match files used soley
#  by the master process versus those created by every process.
#
   set SCR = $SCRDIR/scratch/$PBS_JOBID
   set SCR_MASTER = $SCR/MASTER
   if(-e $SCR) then
     echo ************ WARNING ***************
     echo RARE OCCURENCE - $SCR exists!
     echo Moving $SCR to $SCR.old
     echo mv $SCR $SCR.old
     echo ************ WARNING ***************
   endif
endif
# *************************** CRAY XT ************************************


# create the scratch directory (-p will not output an error if $SCR exists)
mkdir -p $SCR
mkdir -p $SCR_MASTER

# Depending on the number of OSTs that you have configured in your system
# you can configure two key sections of the $SCR space differently.
# $SCR_MASTER is the files created and used by the master process only.
# $SCR is the space used for all other processes.  GAMESS appends the
# rank of the process to all files created, e.g. $JOB.F08.XXX where XXX is
# the DDI rank of the creating process.  Modifying these values will can
# and will have impact on I/O heavy calculations.  Trial and error is needed
# to see which options are optimal for different types of runs.  Ideally,
# the user will choose to run a "direct" calculation over a "conventional"
# calculation to avoid large amounts of I/O.
if($TARGET == cray-xt) then
# This module is not on the XE
#   module load xt-lustre-ss
   set LFS_STRIPE        = "-s 1048576 -c 1"
   set LFS_STRIPE_MASTER = "-s 2097152 -c 4"
   set LFS_STRIPE_MASTER = "-s 1048576 -c 4"
   echo " **** Setting the LUSTRE Striping for SCR to $LFS_STRIPE."
   echo " **** Setting the LUSTRE Striping for SCR_MASTER to $LFS_STRIPE_MASTER."
   echo lfs setstripe $SCR        $LFS_STRIPE
   echo lfs setstripe $SCR_MASTER $LFS_STRIPE_MASTER
   lfs setstripe $LFS_STRIPE        $SCR
   lfs setstripe $LFS_STRIPE_MASTER $SCR_MASTER
   lfs getstripe $SCR
   lfs getstripe $SCR_MASTER
endif

# if $SCR had existed previously, we need to clean up any files that have
# the same name as our current job.
rm -rf $SCR/$JOB.*


#  ---- the top section of the script is input and other file assignments ----
#
echo ----- GAMESS execution script -----
set master=`hostname`
echo This job is running on host $master
echo under operating system `uname` at `date`
echo "Available scratch disk space (Kbyte units) at beginning of the job is"
df -k $SCR

#        this added as experiment, February 2007
#        its intent is to detect large arrays allocated off the stack
#rmo limit stacksize 8192
#
#  Grab a copy of the input file.
#  In the case of EXAMnn jobs, this file might be in the "tests" subdirectory.
if ($JOB:r.inp == $JOB) set JOB=$JOB:r      # strip off possible .inp
if (-e $JOB.inp) then
   set echo
   cp  $JOB.inp  $SCR_MASTER/$JOB.F05
   unset echo
else
   if (-e tests/$JOB.inp) then
      set echo
      cp  tests/$JOB.inp  $SCR_MASTER/$JOB.F05
      unset echo
   else
      echo "Input file $JOB.inp does not exist.  Please fix and resubmit."
      exit 4
   endif
endif

#  Change to the $SCR directory
set WRKDIR = `pwd`
cd $SCR

#  file assignments.
#  all binary files should be put on a node's local disk ($SCR directory), 
#  both ASCII punch files might be written by NFS to user's permanent disk

# *************************** CRAY XT ************************************
#  The ericfmt.dat file is needed for GAMESS.
#  At start up a copy of the master file is made to the scratch directory
#  ensuring that file used at runtime is on the LUSTRE filesystem.
#  The MASTER copy of ericfmt.dat can be on any network mounted filesystem
#  from which aprun can be successfully executed to launch an application.
   set echo
   cp $ERICPTH/ericfmt.dat $SCR_MASTER/_ericfmt.dat
   setenv ERICFMT $SCR_MASTER/_ericfmt.dat
   unset echo
#
#  and we'll need the MCP data as well
#  set echo
#  cp -r $ERICPTH/mcpdata/ $SCR/_mpcdata
#  setenv MCPPATH $SCR/_mpcdata
#  unset echo
#
#  All binary files should be put on a node's local disk ($SCR directory), 
#  for the highest speed access possible.  These .Fxx files are typically
#  not saved for the next run, but they may be big and/or I/O intensive.
#  
#  It is convenient to write ASCII output files (PUNCH, RESTART, TRAJECT,
#  and MAKEFP) to the user's permanent disk, on your file server.  They
#  are small, written only by the master process, and are useful outputs
#  for further runs.
#
# *************************** CRAY XT ************************************

set echo
setenv  EXTBAS /dev/null
setenv  NUCBAS /dev/null

setenv IRCDATA $SCR_MASTER/$JOB.irc
setenv   INPUT $SCR_MASTER/$JOB.F05
setenv INPMINI $SCR_MASTER/$JOB.F03

# Creating Skeleton Input (Mini Input)
grep '^ \$' $INPUT > $INPMINI
if($GMS_USE_MINI_INPUT == true) then
   setenv USE_MINI_INPUT 1
endif

# Setting OUTPUT env if requested
if($GMS_USE_OUTPUT == true) then
   setenv OUTPUT $SCR_MASTER/$JOB.out
endif

setenv  MAKEFP $SCR_MASTER/$JOB.efp
setenv   GAMMA $SCR_MASTER/$JOB.gamma
setenv TRAJECT $SCR_MASTER/$JOB.trj
setenv RESTART $SCR_MASTER/$JOB.rst
setenv   PUNCH $SCR_MASTER/$JOB.dat
setenv  AOINTS $SCR/$JOB.F08
setenv  MOINTS $SCR/$JOB.F09
setenv DICTNRY $SCR_MASTER/$JOB.F10
setenv DRTFILE $SCR/$JOB.F11
setenv CIVECTR $SCR/$JOB.F12
setenv CASINTS $SCR/$JOB.F13
setenv  CIINTS $SCR/$JOB.F14
setenv  WORK15 $SCR/$JOB.F15
setenv  WORK16 $SCR/$JOB.F16
setenv CSFSAVE $SCR/$JOB.F17
setenv FOCKDER $SCR/$JOB.F18
setenv  WORK19 $SCR/$JOB.F19
setenv  DASORT $SCR/$JOB.F20
setenv DFTINTS $SCR/$JOB.F21
setenv DFTGRID $SCR/$JOB.F22
setenv  JKFILE $SCR/$JOB.F23
setenv  ORDINT $SCR/$JOB.F24
setenv  EFPIND $SCR/$JOB.F25
setenv PCMDATA $SCR/$JOB.F26                                     
setenv PCMINTS $SCR/$JOB.F27
setenv SVPWRK1 $SCR/$JOB.F26
setenv SVPWRK2 $SCR/$JOB.F27
setenv COSCAV  $SCR/$JOB.F26
setenv COSDATA ~$USER/scr/$JOB.cosmo
setenv COSPOT  ~$USER/scr/$JOB.pot
setenv  MLTPL  $SCR/$JOB.F28
setenv  MLTPLT $SCR/$JOB.F29
setenv  DAFL30 $SCR/$JOB.F30
setenv  SOINTX $SCR/$JOB.F31
setenv  SOINTY $SCR/$JOB.F32
setenv  SOINTZ $SCR/$JOB.F33
setenv  SORESC $SCR/$JOB.F34
#   35 is used by RESTART, see above
setenv GCILIST $SCR/$JOB.F37
setenv HESSIAN $SCR/$JOB.F38
setenv QMMMTEI $SCR/$JOB.F39
setenv SOCCDAT $SCR/$JOB.F40
setenv  AABB41 $SCR/$JOB.F41
setenv  BBAA42 $SCR/$JOB.F42
setenv  BBBB43 $SCR/$JOB.F43
setenv  MCQD50 $SCR/$JOB.F50
setenv  MCQD51 $SCR/$JOB.F51
setenv  MCQD52 $SCR/$JOB.F52
setenv  MCQD53 $SCR/$JOB.F53
setenv  MCQD54 $SCR/$JOB.F54
setenv  MCQD55 $SCR/$JOB.F55
setenv  MCQD56 $SCR/$JOB.F56
setenv  MCQD57 $SCR/$JOB.F57
setenv  MCQD58 $SCR/$JOB.F58
setenv  MCQD59 $SCR/$JOB.F59
setenv  MCQD60 $SCR/$JOB.F60
setenv  MCQD61 $SCR/$JOB.F61
setenv  MCQD62 $SCR/$JOB.F62
setenv  MCQD63 $SCR/$JOB.F63
setenv  MCQD64 $SCR/$JOB.F64
setenv NMRINT1 $SCR/$JOB.F61
setenv NMRINT2 $SCR/$JOB.F62
setenv NMRINT3 $SCR/$JOB.F63
setenv NMRINT4 $SCR/$JOB.F64
setenv NMRINT5 $SCR/$JOB.F65
setenv NMRINT6 $SCR/$JOB.F66
setenv DCPHFH2 $SCR/$JOB.F67
setenv DCPHF21 $SCR/$JOB.F68
setenv ELNUINT $SCR/$JOB.F67
setenv NUNUINT $SCR/$JOB.F68
setenv   GVVPT $SCR/$JOB.F69
setenv NUMOIN  $SCR/$JOB.F69
setenv NUMOCAS $SCR/$JOB.F70
setenv NUELMO  $SCR/$JOB.F71
setenv NUELCAS $SCR/$JOB.F72

#    next files are for RI-MP2
setenv RIVMAT  $SCR/$JOB.F51
setenv RIT2A   $SCR/$JOB.F52
setenv RIT3A   $SCR/$JOB.F53
setenv RIT2B   $SCR/$JOB.F54
setenv RIT3B   $SCR/$JOB.F55

#    Next files are for GMCQDPT
setenv GMCREF $SCR/$JOB.F70
setenv GMCO2R $SCR/$JOB.F71
setenv GMCROC $SCR/$JOB.F72
setenv GMCOOC $SCR/$JOB.F73
setenv GMCCC0 $SCR/$JOB.F74
setenv GMCHMA $SCR/$JOB.F75
setenv GMCEI1 $SCR/$JOB.F76
setenv GMCEI2 $SCR/$JOB.F77
setenv GMCEOB $SCR/$JOB.F78
setenv GMCEDT $SCR/$JOB.F79
setenv GMCERF $SCR/$JOB.F80
setenv GMCHCR $SCR/$JOB.F81
setenv GMCGJK $SCR/$JOB.F82
setenv GMCGAI $SCR/$JOB.F83
setenv GMCGEO $SCR/$JOB.F84
setenv GMCTE1 $SCR/$JOB.F85
setenv GMCTE2 $SCR/$JOB.F86
setenv GMCHEF $SCR/$JOB.F87
setenv GMCMOL $SCR/$JOB.F88
setenv GMCMOS $SCR/$JOB.F89
setenv GMCWGT $SCR/$JOB.F90
setenv GMCRM2 $SCR/$JOB.F91
setenv GMCRM1 $SCR/$JOB.F92
setenv GMCR00 $SCR/$JOB.F93
setenv GMCRP1 $SCR/$JOB.F94
setenv GMCRP2 $SCR/$JOB.F95
setenv GMCVEF $SCR/$JOB.F96
setenv GMCDIN $SCR/$JOB.F97
setenv GMC2SZ $SCR/$JOB.F98
setenv GMCCCS $SCR/$JOB.F99

#    Next files are used only during closed shell coupled cluster runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set cctyp=`grep -i 'CCTYP[(=]' $INPUT | wc -l`
if ($cctyp > 0) set echo
setenv  CCREST $SCR/$JOB.F70
setenv  CCDIIS $SCR/$JOB.F71
setenv  CCINTS $SCR/$JOB.F72
setenv CCT1AMP $SCR/$JOB.F73
setenv CCT2AMP $SCR/$JOB.F74
setenv CCT3AMP $SCR/$JOB.F75
setenv    CCVM $SCR/$JOB.F76
setenv    CCVE $SCR/$JOB.F77
setenv CCQUADS $SCR/$JOB.F78
setenv QUADSVO $SCR/$JOB.F79
setenv EOMSTAR $SCR/$JOB.F80
setenv EOMVEC1 $SCR/$JOB.F81
setenv EOMVEC2 $SCR/$JOB.F82
setenv  EOMHC1 $SCR/$JOB.F83
setenv  EOMHC2 $SCR/$JOB.F84
setenv EOMHHHH $SCR/$JOB.F85
setenv EOMPPPP $SCR/$JOB.F86
setenv EOMRAMP $SCR/$JOB.F87
setenv EOMRTMP $SCR/$JOB.F88
setenv EOMDG12 $SCR/$JOB.F89
setenv    MMPP $SCR/$JOB.F90
setenv   MMHPP $SCR/$JOB.F91
setenv MMCIVEC $SCR/$JOB.F92
setenv MMCIVC1 $SCR/$JOB.F93
setenv MMCIITR $SCR/$JOB.F94
setenv  EOMVL1 $SCR/$JOB.F95
setenv  EOMVL2 $SCR/$JOB.F96
setenv EOMLVEC $SCR/$JOB.F97
setenv  EOMHL1 $SCR/$JOB.F98
setenv  EOMHL2 $SCR/$JOB.F99
setenv  CCVVVV $SCR/$JOB.F80
#
#    Next files are used only during open shell coupled cluster runs.
#
setenv AMPROCC $SCR/$JOB.F70
setenv ITOPNCC $SCR/$JOB.F71
setenv FOCKMTX $SCR/$JOB.F72
setenv  LAMB23 $SCR/$JOB.F73
setenv   VHHAA $SCR/$JOB.F74
setenv   VHHBB $SCR/$JOB.F75
setenv   VHHAB $SCR/$JOB.F76
setenv    VMAA $SCR/$JOB.F77
setenv    VMBB $SCR/$JOB.F78
setenv    VMAB $SCR/$JOB.F79
setenv    VMBA $SCR/$JOB.F80
setenv  VHPRAA $SCR/$JOB.F81
setenv  VHPRBB $SCR/$JOB.F82
setenv  VHPRAB $SCR/$JOB.F83
setenv  VHPLAA $SCR/$JOB.F84
setenv  VHPLBB $SCR/$JOB.F85
setenv  VHPLAB $SCR/$JOB.F86
setenv  VHPLBA $SCR/$JOB.F87
setenv    VEAA $SCR/$JOB.F88
setenv    VEBB $SCR/$JOB.F89
setenv    VEAB $SCR/$JOB.F90
setenv    VEBA $SCR/$JOB.F91
setenv   VPPPP $SCR/$JOB.F92
setenv INTERM1 $SCR/$JOB.F93
setenv INTERM2 $SCR/$JOB.F94
setenv INTERM3 $SCR/$JOB.F95

setenv ITSPACE $SCR/$JOB.F96
setenv INSTART $SCR/$JOB.F97
setenv  ITSPC3 $SCR/$JOB.F98
#
#    Next files are used only during elongation method runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set elgtyp = `grep -i 'NELONG=' $INPUT | wc -l`
if ($elgtyp > 0) then
    set ELGNAME=$4
    if (null$4 == null) set ELGNAME=ELGFILE
    set echo
    setenv AOINTS   $SCR/$ELGNAME.F08
    setenv ELGDOS   ~$USER/scr/$JOB.ldos
    setenv ELGDAT   $SCR/$ELGNAME.F71
    setenv ELGPAR   $SCR/$ELGNAME.F72
    setenv ELGCUT   $SCR/$ELGNAME.F74
    setenv ELGVEC   $SCR/$ELGNAME.F75
    setenv EGINTA   $SCR/$ELGNAME.F77
    setenv EGINTB   $SCR/$ELGNAME.F78
    setenv EGTDHF   $SCR/$ELGNAME.F79
    setenv EGTEST   $SCR/$ELGNAME.F80
    unset echo
endif
#
#    Next files are used only during extended TDHF package runs.
#    Display the numerous definitions iff they are going to be used.
unset echo
set txtyp=`grep -i 'RUNTYP=TDHFX' $INPUT | wc -l`
if ($txtyp > 0) set echo
setenv  OLI201 $SCR/$JOB.F201
setenv  OLI202 $SCR/$JOB.F202
setenv  OLI203 $SCR/$JOB.F203
setenv  OLI204 $SCR/$JOB.F204
setenv  OLI205 $SCR/$JOB.F205
setenv  OLI206 $SCR/$JOB.F206
setenv  OLI207 $SCR/$JOB.F207
setenv  OLI208 $SCR/$JOB.F208
setenv  OLI209 $SCR/$JOB.F209
setenv  OLI210 $SCR/$JOB.F210
setenv  OLI211 $SCR/$JOB.F211
setenv  OLI212 $SCR/$JOB.F212
setenv  OLI213 $SCR/$JOB.F213
setenv  OLI214 $SCR/$JOB.F214
setenv  OLI215 $SCR/$JOB.F215
setenv  OLI216 $SCR/$JOB.F216
setenv  OLI217 $SCR/$JOB.F217
setenv  OLI218 $SCR/$JOB.F218
setenv  OLI219 $SCR/$JOB.F219
setenv  OLI220 $SCR/$JOB.F220
setenv  OLI221 $SCR/$JOB.F221
setenv  OLI222 $SCR/$JOB.F222
setenv  OLI223 $SCR/$JOB.F223
setenv  OLI224 $SCR/$JOB.F224
setenv  OLI225 $SCR/$JOB.F225
setenv  OLI226 $SCR/$JOB.F226
setenv  OLI227 $SCR/$JOB.F227
setenv  OLI228 $SCR/$JOB.F228
setenv  OLI229 $SCR/$JOB.F229
setenv  OLI230 $SCR/$JOB.F230
setenv  OLI231 $SCR/$JOB.F231
setenv  OLI232 $SCR/$JOB.F232
setenv  OLI233 $SCR/$JOB.F233
setenv  OLI234 $SCR/$JOB.F234
setenv  OLI235 $SCR/$JOB.F235
setenv  OLI236 $SCR/$JOB.F236
setenv  OLI237 $SCR/$JOB.F237
setenv  OLI238 $SCR/$JOB.F238
setenv  OLI239 $SCR/$JOB.F239
unset echo

#    Next files are used only during divide-and-conquer runs
setenv   DCSUB $SCR/$JOB.F250
setenv   DCVEC $SCR/$JOB.F251
setenv   DCEIG $SCR/$JOB.F252
setenv   DCDM  $SCR/$JOB.F253
setenv   DCDMO $SCR/$JOB.F254
setenv   DCQ   $SCR/$JOB.F255
setenv   DCW   $SCR/$JOB.F256
setenv   DCEDM $SCR/$JOB.F257

#    Next files are used only during LMO hyperpolarizability analysis
setenv LHYPWRK $SCR/$JOB.F297
setenv LHYPWK2 $SCR/$JOB.F298
setenv BONDDPF $SCR/$JOB.F299

#    Next value is used only within the VB2000 add-on code
setenv GMSJOBNAME $JOB

#    If a $GDDI input group is present, the calculation will be using
#    subgroups within DDI (the input NGROUP=0 means this isn't GDDI).
#
#    The master within each group must have a copy of INPUT, which is
#    dealt with below (prior to execution), once we know something about
#    the host names where INPUT is required.  The INPUT does not have
#    the global rank appended to its name, unlike all other files.
#
#    OUTPUT and PUNCH (and perhaps many other files) are opened on all
#    processes (not just the master in each subgroup), but unique names
#    will be generated by appending the global ranks.  Note that OUTPUT
#    is not opened by the master in the first group, but is used by all
#    other groups.  Typically, the OUTPUT from the first group's master
#    is the only one worth saving, unless perhaps if runs crash out.
#
#    The other files that GDDI runs might use are already defined above.
#
set ngddi=`grep -i '^ \$GDDI' $SCR/$JOB.F05 | grep -iv 'NGROUP=0 ' | wc -l`
if ($ngddi > 0) then
   set GDDIjob=true
   echo "This is a GDDI run, keeping various output files on local disks"
   set echo
   setenv  OUTPUT $SCR/$JOB.F06
   setenv   PUNCH $SCR/$JOB.F07
   unset echo
else
   set GDDIjob=false
endif

#    data left over from a previous run might be precious, stop if found.
if ((-e $PUNCH) || (-e $MAKEFP) || (-e $TRAJECT) || (-e $RESTART) ) then
   echo "Please save, rename, or erase these files from a previous run:"
   echo "     $PUNCH,"
   echo "     $TRAJECT,"
   echo "     $RESTART, and/or"
   echo "     $MAKEFP,"
   echo "and then resubmit this computation."
   exit 4
endif

#  ---- the middle section of the script is to execute GAMESS ----
#
#  see rungms.sockets the removed section of code.  you can restore
#  this original file by :r rungms.sockets after this line :)
#
#   Cray T3E wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-t3e) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   mpprun -n $NNODES $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif
#
#   Cray X1 wants you to
#      a) set the path to point to the GAMESS executable
if ($TARGET == cray-x1) then
   set GMSPATH=/u1/mike/gamess
   chdir $SCR
   set echo
   aprun -c core=0 -n $NNODES -p16m:16m $GMSPATH/gamess.$VERNO.x $JOB
   unset echo
endif
#
#   CRAY-XT3 with SHMEM
#
if ($TARGET == cray-xt) then
# location of GAMESS binary
   set echo
#  setenv MPICH_ENV_DISPLAY 1
#  setenv MPI_PROFILE_SUMMARY 1
#  setenv MPICH_FAST_MEMCPY 1
#  setenv SHMEM_SWAP_BACKOFF 100
#  setenv XT_LINUX_SHMEM_STACK_SIZE 64m
#  setenv MPICH_PTL_EAGER_LONG yes ! results in poor performance
#   setenv XT_SYMMETRIC_HEAP_SIZE 5m
#   setenv HUGETLB_MORECORE yes
#   setenv MPICH_PTL_MATCH_OFF yes 
   module add craype-hugepages2M
   unset echo
   set NMEMDDI = `grep -c -i memddi $INPUT`
   if($NMEMDDI > 0) then
     setenv DDI_DS_PER_NODE 1
     echo setenv DDI_DS_PER_NODE $DDI_DS_PER_NODE
   endif
   set echo
   logger -is -t GAMESS start
   $APRUN $GMSPATH/gamess.$VERNO.x $JOB
   # aprun -n $NNODES -N 31 -m400hs $GMSPATH/gamess.$VERNO.x $JOB
   logger -is -t GAMESS stop
   unset echo
endif
#
#  ---- the bottom section of the script is to clean up all disk files ----
#  It is quite useful to display to users how big the disk files got to be.
#
echo ----- accounting info -----
#
#   in the case of GDDI runs, we save the first PUNCH file only.
#   If something goes wrong, the .F06.00x, .F07.00x, ... from the
#   other groups are potentially interesting to look at.
if ($GDDIjob == true) cp $SCR/$JOB.F07 ~/scr/$JOB.dat
#
#   Clean up the master's scratch directory.
#
echo Files used by the master process on $master were:
ls -lF $SCR_MASTER
#
echo Files used by every process where:
ls -lF $SCR
#
echo " "
if(-e $PUNCH) then
  cp $PUNCH $WRKDIR
endif
if(-e $RESTART) then
  cp $RESTART $WRKDIR
endif

echo " "

if($GMS_USE_OUTPUT == true) then
  echo "An achive of the output:"
  echo " "
  cat $OUTPUT
endif

echo " "
echo "Removing the scratch files"

if(-e $SCR/core) then
  echo "Preserving core file"
  exit 1
endif

if($VERNO == fpmpi || $VERNO == opt-fpmpi) then
  echo "Not deleteing: $SCR"
else
  rm -rf $SCR
endif

#  and this is the end
echo " "
date
time
exit