#!/bin/sh -l 
#set echo  doesn't work in bash  the equivalent in bash is $ bash -x shl 
#
#---------------------------------------------------------------------------
# This bash shell script is used to run FV3 deterministic forecast by Xiao-Ming Hu (xhu@ou.edu), Apr 1, 2022 
#---------------------------------------------------------------------------
#---------------------
# 1. Define variables
# 1.1 define path 
     declare -r HOME_DIR="/scratch/01178/tg804586"          #
#     declare -r WRF_FIX_DIR="/work/01178/tg804586/stampede2/code/V4.0/WRF_opt15_bothCPU_SH_sens"
     declare -r     WPS_DIR="/work/01178/tg804586/stampede2/code/WRFV4.3.3/WPS"
     declare -r WRF_FIX_DIR="/work/01178/tg804586/stampede2/code/WRFV4.3.3/WRF"
          n_test=1
     declare -r GEOG_DIR="/work/01178/tg804586/Data/WPS_GEOG"    # WPS GEOG Data Dir

# 1.2 define input data     
     output_interval=1
     declare -r grid_data_type="GFS"                              # 
     declare -r input_interval=6
     declare -r forecast_length=36                               # 

     declare -r run_times=0
     
# 1.3 define job running
     run_geogrid="yes"
     run_ungrib="yes"
     run_metgrid="yes"
     run_real="yes"
     run_wrf="yes"
     wps_way=16
     wps_cpus=16
     wrf_way=16
#     wrf_cpus=16   # for small domain, otherwise abort 
     wrf_cpus=36   # for small domain, otherwise abort , 36 permitted, but not 48

# 1.4 define domain
     max_dom=1
     center=(40 -97)

     wrf_dx=(12000 4000 4500 1500 500)                           # for domain 1, 2, 3, 4 ...; dx=dy in my experiments
     grids_we=(  443 250 289 349 241)
     grids_sn=(  266 349 274 379 286)
     i_parent_start=(1 186 75 88 88)
     j_parent_start=(1 20 64 68 231)
     parent_id=(1      1    2    3   4)
     parent_grid_ratio=(1 3  3   3   3)                           # 
     map_proj="lambert"                                       # map projection
     truelat1=33
     truelat2=45
     stand_lon=-97

#---------------------
#---------------------------------------------------------------------------
#---------------------
# 2. Get the time
# 2.1 get the time
 nt=0     # because of the resources limitation, we need to do our simulation one segment by one segment. 
 ntstart=3     # because of the resources limitation, we need to do our simulation one segment by one segment. 
 while [ ${nt} -le "${run_times}" ]; do
     if [ $# == 0 ]; then
          data_year=`date -u +%Y`                  # If no args, we'll get the computer time and run real-time case.
          data_month=`date -u +%m`                 #
          data_day=`date -u +%d`                   #
          data_hour=`date -u +%H`                  #
          data_minute=`date -u +%M`                #
          data_hour=`expr ${data_hour} - 4`        # The gfs forecast data will be later than about 4hr, 
          data_minute=`expr ${data_minute} - 00`   # and the ldm observation data will be later than 2:30Hr
          echo " We will run real-time case for raw date ${data_year}${data_month}${data_day}${data_hour}  "

     elif [ $# == 1 -a "${nt}" == 0 ]; then
          data_time=$1
          echo " We will run real-time case for ${data_time}  "
          data_year=`echo ${data_time} | cut -c1-4`
          data_month=`echo ${data_time} | cut -c5-6`
          data_day=`echo ${data_time} | cut -c7-8`
          data_hour=`echo ${data_time} | cut -c9-10`
          data_minute=`echo ${data_time} | cut -c11-12`
      else
          echo "gonna start ${data_time}" 
          data_year=`echo ${data_time} | cut -c1-4`
          data_month=`echo ${data_time} | cut -c5-6`
          data_day=`echo ${data_time} | cut -c7-8`
          data_hour=`echo ${data_time} | cut -c9-10`
          data_minute=`echo ${data_time} | cut -c11-12`
     fi

 
     data_minute=0
     if [ ${data_hour} -lt 0 ]; then
          data_hour=`expr ${data_hour} + 24`        
          data_day=`expr ${data_day} - 1`        
     fi
     if [ ${data_hour} -ge 0 -a ${data_hour} -lt 6 ]; then
        data_hour=0
     elif [ ${data_hour} -ge 6 -a ${data_hour} -lt 12 ]; then
        data_hour=6
     elif [ ${data_hour} -ge 12 -a ${data_hour} -lt 18 ]; then
        data_hour=12
     elif [ ${data_hour} -ge 18 -a ${data_hour} -lt 24 ]; then
        data_hour=18
     fi

#        data_hour=0  # for initial testing

     data_year=`echo "${data_year}"|awk '{if (length($1)==1) $1=200$1; printf "%s",$1}'`
     data_year=`echo "${data_year}"|awk '{if (length($1)==2) $1=20$1; printf "%s",$1}'`
     data_month=`echo "${data_month}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
     data_day=`echo "${data_day}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
     data_hour=`echo "${data_hour}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
     data_minute=`echo "${data_minute}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
          echo " for updated date ${data_year}${data_month}${data_day}${data_hour}  "
#---------------------
#---------------------------------------------------------------------------
#---------------------
# 3. Function
# 3.1 time calculation function
     function cal_time()
{
     S_YEAR=$1
     S_MONTH=$2
     S_DAY=$3
     S_HOUR=$4
     S_MINUTE=$5
     RUN_DAYS=$6
     RUN_HOURS=$7
     RUN_MINUTES=$8

     E_YEAR="$S_YEAR"
     E_MONTH="$S_MONTH"
     E_DAY=`expr $S_DAY + $RUN_DAYS`
     E_HOUR=`expr $S_HOUR + $RUN_HOURS`
     E_MINUTE=`expr $S_MINUTE + $RUN_MINUTES`

     until [ ${E_MINUTE} -lt 60 -a ${E_MINUTE} -ge 0 ]; do
        if [ ${E_MINUTE} -lt 0 ]; then
             let "E_MINUTE += 60"
             let "E_HOUR -= 1"
        elif [ ${E_MINUTE} -ge 60 ]; then
             let "E_MINUTE -= 60"
             let "E_HOUR += 1"
        fi
     done
     until [ ${E_HOUR} -lt 24 -a ${E_HOUR} -ge 0 ]; do
        if [ ${E_HOUR} -lt 0 ]; then
             let "E_HOUR += 24"
             let "E_DAY -= 1"
        elif [ ${E_HOUR} -ge 24 ]; then
             let "E_HOUR -= 24"
             let "E_DAY += 1"
        fi
     done 

     DAY_TAIL=`echo \`cal ${E_MONTH} ${E_YEAR}\` |tail -n1 |awk '{print $NF}'`
     until [ ${E_DAY} -le $DAY_TAIL -a ${E_MONTH} -le 12 ]; do
        let "E_DAY -= ${DAY_TAIL}"
        E_MONTH=`expr ${E_MONTH} + 1`
        if [ "${E_MONTH}" -gt "12" ]; then
           let "E_MONTH -= 12"
           let "E_YEAR += 1"
        fi
        DAY_TAIL=`echo \`cal ${E_MONTH} ${E_YEAR}\` |tail -n1 |awk '{print $NF}'`
     done

     E_YEAR=`echo "$E_YEAR"|awk '{if (length($1)==1) $1=200$1; printf "%s",$1}'`
     E_YEAR=`echo "$E_YEAR"|awk '{if (length($1)==2) $1=20$1; printf "%s",$1}'`
     E_MONTH=`echo "${E_MONTH}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
     E_DAY=`echo "${E_DAY}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
     E_HOUR=`echo "${E_HOUR}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
     E_MINUTE=`echo "${E_MINUTE}"|awk '{if (length($1)==1) $1=0$1; printf "%s",$1}'`
}

# 3.2 job run script
#     job_run ${way} ${ncpus} ${jobname} ${rundir} ${logdir} ${bin_file}
  function job_run()
{
rm -f ${5}/${3}.o ${5}/${3}.e
cd ${4}
cat > ${3}.sh << EOF
#!/bin/bash
#
#$ -V                     # Inherit the submission environment 
#$ -cwd                   # Start job in  submission directory
#$ -N ${3}                  # Job Name
#$ -e ${5}/${3}.e             # 
#$ -o ${5}/${3}.o             # 
#$ -pe ${1}way ${2}       # Requests cpus
#$ -q normal                # Queue name
#$ -A TG-ATM160014
#$ -l h_rt=24:00:00       # Run time (hh:mm:ss) - 1.5 hours

cd ${4}
time ibrun ${6} >& ${3}.print.out
EOF
chmod +x ${3}.sh
qsub ${3}.sh
}
 
#---------------------
#---------------------------------------------------------------------------
#---------------------
# 4. WPS run
cat > python3download_xhu.sh << EOF
#!/bin/bash -l

module load python3
/opt/apps/intel18/python3/3.7.0/bin/python3 /home1/01178/tg804586/code/Shell/KNL/download_aws_xhu.py --no-gefs --case ${data_year}${data_month}${data_day}${data_hour}
EOF
chmod +x python3download_xhu.sh   # module load python3 will mess up generate_FV3LAM_wflow.sh below, so has to put in a seperate script
./python3download_xhu.sh >& /scratch/01178/tg804586/code/python3download_xhu.sh_log.txt
              until [ -e /scratch/01178/tg804586/Run/Datagfs_0p25/${data_year}${data_month}${data_day}${data_hour}/gfs.t${data_hour}z.pgrb2.0p25.f048 ]; do
                 sleep 1m
                 echo "waiting for /scratch/01178/tg804586/Run/Datagfs_0p25/${data_year}${data_month}${data_day}${data_hour}/gfs.t${data_hour}z.pgrb2.0p25.f048"
              done 
###if one source fail, try another source using wget
bash -x /home1/01178/tg804586/code/Shell/KNL/wget_realtime_GFS_xhu.sh ${data_year}${data_month}${data_day}${data_hour} >& /home1/01178/tg804586/code/Shell/KNL/wget_realtime_GFS_xhu.sh_forFV3.log.txt

     cd /work2/01178/tg804586/stampede2/code/FV3/ufs-srweather-app/regional_workflow/ush
     cp config_gfs_xhu_template.sh config_gfs_xhu_daily.sh
     ex +g#sudir_xhu#s##${data_year}${data_month}${data_day}${data_hour}#g +wq config_gfs_xhu_daily.sh 
     ex +g#forecasttime#s##${data_year}${data_month}${data_day}#g +wq config_gfs_xhu_daily.sh 
     ex +g#initialhour_xhu#s##${data_hour}#g +wq config_gfs_xhu_daily.sh 
     ex +g#forecast_length#s##${forecast_length}#g +wq config_gfs_xhu_daily.sh 
     echo "doing config for ${data_year}${data_month}${data_day}${data_hour}"
     ln -sf config_gfs_xhu_daily.sh config.sh

     ./generate_FV3LAM_wflow.sh  >& ./generate_FV3LAM_wflow.sh_log.txt
  
     echo "finish generate_FV3LAM_wflow.sh"
    
     WORK_DIR="${HOME_DIR}/Run/SRWAPP_UFS_202201/FV3_${data_year}${data_month}${data_day}${data_hour}"
     mkdir ${WORK_DIR}   # since different job may interfere each other, permission deny issue if regenerate a file that is being using, so using different dir

     cd /work2/01178/tg804586/stampede2/code/FV3/SRW_v1.0/fv3lam-slurm
     rm ${WORK_DIR}/log/out.make_grid
     export   EXPTDIR=${WORK_DIR}
     run_fv3lam.sh  -m stampede   $EXPTDIR/var_defns.sh grid  # need to generate diag_table  
sleep 30s
        njobsabort=`grep -i "Grid files with various halo widths generated successfully" ${WORK_DIR}/log/out.make_grid | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 1m
          echo "since Grid job not finished, waiting and check !!!!!"
          njobsabort=`grep -i "Grid files with various halo widths generated successfully" ${WORK_DIR}/log/out.make_grid | wc -l`
        done
         echo "Grid successfully finished, will proceed to orog"

     rm ${WORK_DIR}/log/out.make_orog
     run_fv3lam.sh  -m stampede   $EXPTDIR/var_defns.sh orog  # 
sleep 30s
        njobsabort=`grep -i "Orography files with various halo widths generated successfully" ${WORK_DIR}/log/out.make_orog | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 1m
          echo "since orog job not finished, waiting and check !!!!!"
          njobsabort=`grep -i "Orography files with various halo widths generated successfully" ${WORK_DIR}/log/out.make_orog | wc -l`
        done
         echo "orog successfully finished, will proceed to sfc"

     rm ${WORK_DIR}/log/out.make_sfc_climo
     run_fv3lam.sh  -m stampede   $EXPTDIR/var_defns.sh sfc # 
sleep 30s
        njobsabort=`grep -i "All surface climatology files generated successfully" ${WORK_DIR}/log/out.make_sfc_climo | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 1m
          echo "since sfc_climo job not finished, waiting and check !!!!!"
          njobsabort=`grep -i "All surface climatology files generated successfully" ${WORK_DIR}/log/out.make_sfc_climo | wc -l`
        done
         echo "sfc_climo successfully finished, will proceed to get_files.sh"


     cd ${WORK_DIR} 
      ln -s /work2/01178/tg804586/stampede2/code/FV3/SRW_v1.0/fv3lam-slurm/get_files.sh .
      rm ${WORK_DIR}/log/out.get_files_lbcs
      ./get_files.sh >& ${WORK_DIR}/log/get_files.sh_log.txt
sleep 30s
        njobsabort=`grep -i "generating lateral boundary conditions for the FV3 forecast" ${WORK_DIR}/log/out.get_files_lbcs | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 1m
          echo "since get_files job not finished, waiting and check !!!!!"
          ./get_files.sh >& /scratch/01178/tg804586/Run/SRWAPP_UFS_202201/FV3/log/get_files.sh_log.txt
          njobsabort=`grep -i "generating lateral boundary conditions for the FV3 forecast" ${WORK_DIR}/log/out.get_files_lbcs | wc -l`
        done
         echo "get_files successfully finished, will proceed to make_ics"
 

     cd /work2/01178/tg804586/stampede2/code/FV3/SRW_v1.0/fv3lam-slurm

     rm ${WORK_DIR}/log/out.make_ics
     run_fv3lam.sh  -m stampede   $EXPTDIR/var_defns.sh ics

sleep 30s
        njobsabort=`grep -i "FV3 generated successfully" ${WORK_DIR}/log/out.make_ics | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 1m
          echo "since IC job not finished, waiting and check !!!!!"
          njobsabort=`grep -i "FV3 generated successfully" ${WORK_DIR}/log/out.make_ics | wc -l`
        done
         echo "IC successfully finished, will proceed to lbcs"

     rm ${WORK_DIR}/log/out.make_lbcs
     run_fv3lam.sh  -m stampede   $EXPTDIR/var_defns.sh lbcs

sleep 30s
        njobsabort=`grep -i "cessfully for all LBC update hours" ${WORK_DIR}/log/out.make_lbcs | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 1m
          echo "since LBC job not finished, waiting and check !!!!!"
          njobsabort=`grep -i "cessfully for all LBC update hours" ${WORK_DIR}/log/out.make_lbcs | wc -l`
        done
         echo "LBC successfully finished, will proceed to fcst"

     rm ${WORK_DIR}/log/out.run_fcst 
     run_fv3lam.sh  -m stampede   $EXPTDIR/var_defns.sh fcst

sleep 30s
        njobsabort=`grep -i "FV3 forecast completed successfully" ${WORK_DIR}/log/out.run_fcst | wc -l`
        until [ ${njobsabort} -gt 0 ]; do
          sleep 5m
          echo "since FV3 job not finished, waiting and check !!!!!"
          njobsabort=`grep -i "FV3 forecast completed successfully" ${WORK_DIR}/log/out.run_fcst | wc -l`
        done
         echo "FV3 successfully finished!!!!! proceed to plot"

        ndom=1
        while [ ${ndom} -le ${max_dom} ]; do
         if [ ${forecast_length} -lt 10 ]; then
          woutname="phyf00${forecast_length}.nc"
         elif [ ${forecast_length} -ge 10 ]; then
          woutname="phyf0${forecast_length}.nc"
         fi
         echo "waiting for ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/${woutname} "
              until [ -e ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/${woutname} ]; do
                 sleep 1m
              done
              let "ndom += 1"
        done
        echo "found ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/${woutname}, finish 1 round,plot now"

     simname="FV3_GFS025"

        ND=1
        bash -x /home1/01178/tg804586/code/Shell/KNL/FV3_ncl_1time_CONUS.sh ${data_year}${data_month}${data_day}${data_hour} ${nt} ${ND} ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour} ${simname} >& /home1/01178/tg804586/code/Shell/KNL/FV3_ncl_1time_CONUS.sh.log

         echo "waiting for ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/figures/wrfout_d0${max_dom}_T2_${forecast_length}.png"
              until [ -e ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/figures/wrfout_d0${max_dom}_T2_${forecast_length}.png ]; do
                 sleep 1m
              done
                 sleep 2m
         echo "found ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/figures/wrfout_d0${max_dom}_T2_${forecast_length}.png; transfer to CAPS now"
         chmod -R 755  ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/figures
             rsync -av ${WORK_DIR}/${data_year}${data_month}${data_day}${data_hour}/figures/ xhu@cumulus.caps.ou.edu:/nsftor/xhu/public_html/WRF-UCM/FocusON_OKC/WRFVFV3/YSU/wrf${simname}.${data_year}${data_month}${data_day}${data_hour}   >& /home1/01178/tg804586/code/Shell/KNL/rsync_transfer2CAPS.logFV3.txt
        
#fi 
     let "nt += 1"

 done
     echo "FV3 forecast and posting sucess!!!!!!!!!!!!!!!!! oh yeah!!!!!!!"