Valid time

app/get_data/bin/get_GM_data.sh

@@ -2,41 +2,53 @@
 
 fcst_date=$1
 datadir=$2
+max_lead=$3
+accum_period=$4
+
+# Remove T and Z if present to normalize date format
+fcst_date=${fcst_date//T/}
+fcst_date=${fcst_date//Z/}
 
 year=${fcst_date:0:4}
 date=${fcst_date:0:8}
-hour=${fcst_date:9:2}
+hour=${fcst_date:8:2}
+
+# Create formatted version with T and Z for output filenames
+save_current_date=${date}T${hour}00Z
+
+echo "year: $year"
+echo "date: $date"
+echo "hour: $hour"
+echo "save_current_date: $save_current_date"
 
 ### Determine lead times based on forecast hour ###
 
 if [[ "$hour" == "00" || "$hour" == "12" ]]; then
-    leads_6h_end='72'
-    leads_24h_end='144'
+    leads_end=$max_lead
 else
-    leads_6h_end='60'
-    leads_24h_end='48'
+    leads_end=$max_lead
 fi
 
 ## Define times for global update analysis - only 000 and 006 available ##
 analysis_runs=("000" "006")
 
-### Get 6hr leads ###
-leads_6h=""
-for ((i=0; i<=leads_6h_end; i+=6)); do
-    leads_6h="${leads_6h}${i} "
-done
-leads_6h=$(echo $leads_6h)  # Remove trailing space if needed
-
-### Get 24hr leads ###
-leads_24h=""
-for ((i=0; i<=leads_24h_end; i+=24)); do
-    leads_24h="${leads_24h}${i} "
-done
-leads_24h=$(echo $leads_24h)  # Remove trailing space if needed
-
+### Get leads ###
+# Special case: if hour is 12 and accum_period is 24, use 12, 36, 60... instead of 0, 24, 48...
+if [[ "$hour" == "12" && "$accum_period" == "24" ]]; then
+    leads_h=""
+    for ((i=12; i<=leads_end; i+=accum_period)); do
+        leads_h="${leads_h}${i} "
+    done
+else
+    leads_h=""
+    for ((i=0; i<=leads_end; i+=accum_period)); do
+        leads_h="${leads_h}${i} "
+    done
+fi
+leads_h=$(echo $leads_h)  # Remove trailing space if needed
 
 #################### UPDATE ANALYSES ##########################
-
+## 0 & 6 used here as update analyses are only available at 000 and 006 ##
 mass-pull () {
   local analysis=$1 
   touch query
@@ -47,15 +59,15 @@ begin
 end
 EOF
 
-  moo select -I query moose:/opfc/atm/global/prods/${year}.pp/ ${datadir}/6_hour/${fcst_date}_gl-up_${analysis}.pp
+  moo select -I query moose:/opfc/atm/global/prods/${year}.pp/ ${datadir}/6_hour/${save_current_date}_gl-up_${analysis}.pp
   rm query
 }
 
 for analysis in "${analysis_runs[@]}"; do
   mass-pull "$analysis"
 done
 
-#################### 6 hrly ###########################
+#################### PER ACCUM PERIOD ###########################
 
 mass-pull () {
 touch query
@@ -67,69 +79,33 @@ cat >query <<EOF
   end
 EOF
 
-moo select -I query moose:/opfc/atm/global/prods/${year}.pp/ ${datadir}/6_hour/${fcst_date}_gl-mn_T${this_lead}.pp
+moo select -I query moose:/opfc/atm/global/prods/${year}.pp/ ${datadir}/${accum_period}_hour/${save_current_date}_gl-mn_T${this_lead}.pp
 
 }
 
 
-# Pull 18 member mogreps ensemble from MASS archive
+# Pull from MASS archive
 list_of_files=''
 
-if [ ! -d "${datadir}/6_hour" ] ; then
-  echo "${datadir}/6_hour doesn't currently exist. Making..."
-  mkdir -p ${datadir}/6_hour
+if [ ! -d "${datadir}/${accum_period}_hour" ] ; then
+  echo "${datadir}/${accum_period}_hour doesn't currently exist. Making..."
+  mkdir -p ${datadir}/${accum_period}_hour
 fi
 
-for lead in $leads_6h
+for lead in $leads_h
 do
   this_lead=$(printf "%03d" ${lead})
-  echo ${this_lead}
-  file_to_cat="${datadir}/6_hour/${fcst_date}_gl-mn_T${this_lead}.pp"
+  echo "Processing lead time: ${this_lead}"
+  file_to_cat="${datadir}/${accum_period}_hour/${save_current_date}_gl-mn_T${this_lead}.pp"
   list_of_files=$(echo ${list_of_files} "${file_to_cat} ")
   mass-pull
   rm query
 done
 
 
-accum='006'
-echo "cat $list_of_files > ${datadir}/6_hour/${fcst_date}_gl-mn_${accum}.pp"
-cat $list_of_files > ${datadir}/6_hour/${fcst_date}_gl-mn_${accum}.pp
-
-##################### 24 hrly ##########################
-
-mass-pull1 () {
-touch query1
-cat >query1 <<EOF
-begin
- filename="prods_op_gl-mn_${date}_${hour}_*.pp"
- stash=(5201,5202,4201,4202)
- lbft=${lead}
-end
-EOF
-
-moo select -I query1 moose:/opfc/atm/global/prods/${year}.pp/ ${datadir}/24_hour/${fcst_date}_gl-mn_T${this_lead}.pp
-
-}
-list_of_files=''
-
-if [ ! -d "${datadir}/24_hour" ] ; then
-  echo "${datadir}/24_hour doesn't currently exist. Making..."
-  mkdir -p ${datadir}/24_hour
-fi
-
-for lead in $leads_24h
-do
-  this_lead=$(printf "%03d" ${lead})
-  echo ${this_lead}
-  file_to_cat="${datadir}/24_hour/${fcst_date}_gl-mn_T${this_lead}.pp"
-  list_of_files=$(echo ${list_of_files} "${file_to_cat} ")
-  mass-pull1
-  rm query1
-done
-
-accum='024'
-echo "cat $list_of_files > ${datadir}/24_hour/${fcst_date}_gl-mn_${accum}.pp"
-cat $list_of_files > ${datadir}/24_hour/${fcst_date}_gl-mn_${accum}.pp
+accum=$(printf "%03d" $accum_period)
+echo "cat $list_of_files > ${datadir}/${accum_period}_hour/${save_current_date}_gl-mn_${accum}.pp"
+cat $list_of_files > ${datadir}/${accum_period}_hour/${save_current_date}_gl-mn_${accum}.pp
 
 
 ############################## FOR TRIALS #########################################

app/get_data/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=set -x; get_GM_data.sh $CYLC_TASK_CYCLE_POINT $DATADIR
+default=set -x; get_GM_data.sh $CYLC_TASK_CYCLE_POINT $DATADIR $MAX_LEAD $ACCUM_PERIOD

app/get_gpm/bin/calc_gpm_accumulation.py

@@ -42,6 +42,8 @@ def parse_args():
                         help="Maximum lead time")
     parser.add_argument("-p", "--parallel", action="store_true",
                         help="Enable parallelism")
+    parser.add_argument("--cycling_on", type=str,
+                        help="Flag to indicate running on VT or DT within cylc")
 
     # Parse the command line.
     args = parser.parse_args()
@@ -190,6 +192,7 @@ def main():
     ## Create output directory if it doesn't exist
     period_outdir = os.path.join(out_dir, f"{acc_period}_hour_gpm")
     os.makedirs(period_outdir, exist_ok=True)
+    cycling_on = args.cycling_on # flag to indicate running on VT or DT within cycl
 
     if args.cutout:
         cutout = args.cutout
@@ -200,13 +203,27 @@ def main():
     cycle_point = datetime.datetime.strptime(args.cycle_point, '%Y%m%dT%H%MZ')
     lead = args.max_lead
 
-    START_ACCUM_DATE_DT = cycle_point
-    START_ACCUM_DATE_STR = cycle_point.strftime('%Y%m%d%H') 
-    print(f"START_ACCUM_DATE: {START_ACCUM_DATE_STR}")
-    END_ACCUM_DATE_DT = cycle_point + datetime.timedelta(hours=lead)
-    END_ACCUM_DATE_STR = END_ACCUM_DATE_DT.strftime('%Y%m%d%H')
-    print(f"END_ACCUM_DATE: {END_ACCUM_DATE_STR}")
-    i = START_ACCUM_DATE_DT
+    print(f"We are cycling on {cycling_on}")
+
+    if cycling_on == 'DT':
+        START_ACCUM_DATE_DT = cycle_point
+        START_ACCUM_DATE_STR = cycle_point.strftime('%Y%m%d%H') 
+        print(f"START_ACCUM_DATE: {START_ACCUM_DATE_STR}")
+        END_ACCUM_DATE_DT = cycle_point + datetime.timedelta(hours=lead)
+        END_ACCUM_DATE_STR = END_ACCUM_DATE_DT.strftime('%Y%m%d%H')
+        print(f"END_ACCUM_DATE: {END_ACCUM_DATE_STR}")
+        i = START_ACCUM_DATE_DT
+    elif cycling_on == 'VT':
+        END_ACCUM_DATE_DT = cycle_point
+        END_ACCUM_DATE_STR = cycle_point.strftime('%Y%m%d%H')
+        print(f"END_ACCUM_DATE: {END_ACCUM_DATE_STR}")
+        START_ACCUM_DATE_DT = cycle_point - datetime.timedelta(hours=acc_period)
+        START_ACCUM_DATE_STR = START_ACCUM_DATE_DT.strftime('%Y%m%d%H')
+        print(f"START_ACCUM_DATE: {START_ACCUM_DATE_STR}")
+        i = START_ACCUM_DATE_DT
+    else:
+        print("Error: cycling_on flag must be set to either VT or DT")
+        sys.exit(1)
 
     while i < END_ACCUM_DATE_DT:
 

app/get_gpm/bin/gpm_vt_wrapper.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -l
+
+set -x
+module load scitools/production-os47-2
+
+export PYTHONPATH=${PYTHONPATH}:${CYLC_SUITE_DEF_PATH}/app/${ROSE_TASK_APP}/bin
+echo $PYTHONPATH
+echo $ROSE_TASK_APP
+echo $CYLC_TASK_NAME
+
+# CYLC_TASK_CYCLE_POINT is the model run (initialisation date/time)
+echo $CYLC_TASK_CYCLE_POINT
+echo $OUTPUT_DATA
+GPM_DATA_DIR="/data/users/gpm_imerg"
+GPM_OBS_TYPE="GPM_NRTlate"
+
+# 6-hour accumulation: from (cycle point - 6h) to cycle point
+START_ACCUM_PERIOD=$(isodatetime -u $CYLC_TASK_CYCLE_POINT --offset -PT6H --print-format=%Y%m%d%H)
+echo $START_ACCUM_PERIOD
+END_ACCUM_PERIOD=$CYLC_TASK_CYCLE_POINT
+
+echo "6-hour accumulation: $START_ACCUM_PERIOD to $END_ACCUM_PERIOD"
+python ${CYLC_SUITE_DEF_PATH}/app/${ROSE_TASK_APP}/bin/og_calc_gpm_accumulation.py \
+    --outdir $OUTPUT_DATA \
+    --datadir $GPM_DATA_DIR \
+    --obs $GPM_OBS_TYPE \
+    --accum_period 6 \
+    --start_date $START_ACCUM_PERIOD \
+    --end_date $END_ACCUM_PERIOD \
+    --cycle_point $CYLC_TASK_CYCLE_POINT
+
+# 24-hour accumulation: from (cycle point - 24h) to cycle point
+START_ACCUM_PERIOD=$(isodatetime -u $CYLC_TASK_CYCLE_POINT --offset -PT24H --print-format=%Y%m%d%H)
+END_ACCUM_PERIOD=$CYLC_TASK_CYCLE_POINT
+
+echo "24-hour accumulation: $START_ACCUM_PERIOD to $END_ACCUM_PERIOD"
+python ${CYLC_SUITE_DEF_PATH}/app/${ROSE_TASK_APP}/bin/og_calc_gpm_accumulation.py \
+    --outdir $OUTPUT_DATA \
+    --datadir $GPM_DATA_DIR \
+    --obs $GPM_OBS_TYPE \
+    --accum_period 24 \
+    --start_date $START_ACCUM_PERIOD \
+    --end_date $END_ACCUM_PERIOD \
+    --cycle_point $CYLC_TASK_CYCLE_POINT
+
+

app/get_gpm/rose-app.conf

@@ -1,2 +1,2 @@
 [command]
-default=set -x; moxie-run calc_gpm_accumulation.py --datadir $GPM_DATA_DIR  --obs $GPM_OBS_TYPE --accum_period $ACCUM_PERIOD --cycle_point $CYLC_TASK_CYCLE_POINT --max_lead $MAX_LEAD --outdir $OUTPUT_DATA
+default=set -x; moxie-run calc_gpm_accumulation.py --datadir $GPM_DATA_DIR  --obs $GPM_OBS_TYPE --accum_period $ACCUM_PERIOD --cycle_point $CYLC_TASK_CYCLE_POINT --max_lead $MAX_LEAD --outdir $OUTPUT_DATA --cycling_on $CYCLING_ON