#!/bin/bash -x
#SBATCH -J ASTER1
#SBATCH -o ASTER1.%j.out
#SBATCH -e ASTER1.%j.err
#SBATCH -t 48:00:00 
#SBATCH -N 11
#SBATCH -n 242 
#SBATCH --mail-user=
#SBATCH --mail-type=begin
#SBATCH --mail-type=end

#--- 0.load modules ------
ulimit -s hard
ulimit -u hard

#---- 1.set variables ------
#note for bash: can not have any space around =

#nprocs=36
#snx=90
#sny=90
nprocs=242
snx=30
sny=30
#nprocs=165
#snx=30
#sny=45
#nprocs=117
#snx=45
#sny=45
pickupts0="0000000006"
pickupts1="0000000007"
extpickup="_it55warm" 
extsmooth="_9x1000mod"

ext0="00"

iter=62
forwadj=
whichexp="_ASTE_R1"
jobfile=script${whichexp}${forwadj}.bash

  if [ ${iter} -lt 10 ]; then
      ext2=000${iter}
  elif [ ${iter} -lt 100 ]; then
      ext2=00${iter}
  elif [ ${iter} -lt 1000 ]; then
      ext2=0${iter}
  else
      ext2=${iter}
  fi

#--- 2.set dir ------------
  basedir=./
  builddir=$basedir/build${whichexp}${forwadj}
  codedir=$basedir/code${whichexp}
  inputdir=$basedir/NAMELISTS${whichexp}
  workdir=$scratchdir/run${whichexp}_pk${pickupts1}

  if [ ! -d $workdir ]; then
    mkdir -p $workdir;
    #mkdir -p $workdir/tapes;
    #mkdir -p $workdir/profiles;
    mkdir -p $workdir/diags;
  fi
  cd $workdir;

  #rm -f tapes/*
  #rm -f profiles/*

  cp -rf ${codedir}/ .
  #cp -rf ${basedir}/code_adv7_bypass_tamc .

#--- 3. link forcing -------------
  \rm -f jra55
  if [ ! -h jra55 ]; then
    ln -s $basedir/jra55 ./jra55
  fi

#--- 4. linking binary ---------
  ln -s ${scratchdir}/run_template/input_binaries/*.bin .
  #ln -s ${scratchdir}/run_template/input_insitu/+aste/+${snx}x${sny}/*.nc .
  ln -s ${scratchdir}/run_template/input_weight/* .
  ln -s ${scratchdir}/run_template/input_binaries/tile*.mitgrid .
  ln -s ${scratchdir}/run_template/input_obcs/input_obcs/*.bin .
  ###
  ln -s ${scratchdir}/run_template/input_smooth/smooth2Dscales001${extsmooth} ./smooth2Dscales001
  ln -s ${scratchdir}/run_template/input_smooth/smooth3DscalesH001${extsmooth} ./smooth3DscalesH001
  ln -s ${scratchdir}/run_template/input_smooth/smooth3DscalesZ001 ./
  ln -s ${scratchdir}/run_template/input_smooth/smooth2Dnorm001${extsmooth}.data ./smooth2Dnorm001.data
  ln -s ${scratchdir}/run_template/input_smooth/smooth3Dnorm001${extsmooth}.data ./smooth3Dnorm001.data
  ln -s ${scratchdir}/run_template/input_smooth/smooth2Dnorm001.meta ./
  ln -s ${scratchdir}/run_template/input_smooth/smooth3Dnorm001.meta ./

#=================================================================================
#need to copy namelists before xx
#--- 6. NAMELISTS ---------
  cp -f ${inputdir}/* .
  cp -f ${basedir}/${jobfile} .

  #cp -f data_profiles_${snx}x${sny}x${nprocs} data.profiles
  cp -f data_exch2_${snx}x${sny}x${nprocs} data.exch2

#=================================================================================
#--- 5. linking xx_ fields ------
  dirADXX=${basedir}/input_binaries/

#3D fields have already been mapped into pickups and input files
# so we're reading in a set of zeros:
  declare -a arr3d=("kapgm" "kapredi" "theta" "salt" "logdiffkr")
    for i in "${arr3d[@]}"
    do
      cp ${dirADXX}/xx_$i.000000${ext2}.data ./xx_$i.000000${ext2}.data
      echo xx_$i.000000${ext2}.data
    done

#2D time-varying fields were not mapped into jra55 because it cannot
#be done accurately in the Arctic.  As a result, need iter62 xx files:
  declare -a arr2d=("aqh" "atemp" "uwind" "vwind" "precip" "lwdown" "swdown")
    for i in "${arr2d[@]}"
    do
      cp ${dirADXX}/xx_$i.000000${ext2}.data    ./xx_$i.000000${ext2}.data
      echo xx_$i.000000${ext2}.data
    done

#setting data.ctrl flags in case they're not compatible
      sed -i -e 's/'"doinitXX = .TRUE."'/'"doinitXX = .FALSE."'/g' data.ctrl
      sed -i -e 's/'"doinitxx = .TRUE."'/'"doinitxx = .FALSE."'/g' data.ctrl
      sed -i -e 's/'"doMainUnpack = .TRUE."'/'"doMainUnpack = .FALSE."'/g' data.ctrl
      sed -i -e 's/'"doMainPack = .TRUE"'/'"doMainPack = .FALSE"'/g' data.ctrl

#
#=================================================================================

#--- 7. executable --------
  \rm -f mitgcmuv*
  cp -f ${builddir}/mitgcmuv${forwadj}_${snx}x${sny}x${nprocs} ./mitgcmuv${forwadj}
  cp -f ${builddir}/Makefile .

#--- 8. pickups -----------
#NOTE: for pickup: copy instead of link to prevent accidental over-write
  \rm -f pickup*
  pickupdir=${basedir}/input_binaries

  if [[ ${pickupts0} ]]; then
    cp -f ${pickupdir}/pickup${extpickup}.${pickupts1}.data ./pickup.${pickupts1}.data
    cp -f ${pickupdir}/pickup${extpickup}.${pickupts1}.meta ./pickup.${pickupts1}.meta
    cp -f ${pickupdir}/pickup_seaice${extpickup}.${pickupts1}.data ./pickup_seaice.${pickupts1}.data
    cp -f ${pickupdir}/pickup_seaice${extpickup}.${pickupts1}.meta ./pickup_seaice.${pickupts1}.meta
  fi

#--- 9. make a list of all linked files ------

  #\rm -f command_ln_input
  #ls -l input*/* >> command_ln_input
  
  \rm -f command_ln_binary
  ls -l *.nc > command_ln_binary
  ls -l *.bin >> command_ln_binary
  ls -l tile* >> command_ln_binary
  ls -l smooth* >> command_ln_binary
  ls -l xx* >> command_ln_binary

#--- 10. (re)set optimcycle --------------------

\rm data.optim
cat > data.optim <<EOF
 &OPTIM
 optimcycle=${iter},
 /
EOF

#--- 11. run ----------------------------------

  set -x
  date > run.MITGCM.timing
  mpiexec --mca btl ^tcp,openib --mca mtl psm2 ${workdir}/mitgcmuv${forwadj}
  date >> run.MITGCM.timing
