Added Anemone TIDY folder

scripts/TIDY/chkgood.script

@@ -0,0 +1,65 @@
+#!/bin/bash
+#########################################################################
+# Process and tidy the recombined output of a model simulation
+#
+# Usage: ./chkgood.script YY    .or.    ./chkgood.script YS YE
+#
+# Check rebuild_nemo completed successfully                  
+# -  If yes, move the rebuilt files to YDIR and the raw files to ARCHIVE
+# -  If no, leave files in place for investigation
+#
+#   A. Blaker - 12/05/2021
+#########################################################################
+
+DIR=${PWD%/*}
+
+# Check inputs
+if [ $# -eq 0 ]
+then
+  echo "No arguments supplied"
+  exit
+elif [ $# -eq 1 ]
+then
+  YS=$1
+  YE=$1
+  echo "Processing year" $YS
+elif [ $# -eq 2 ]
+then
+  YS=$1
+  YE=$2
+  echo "Processing years" $YS to $YE
+elif [ $# -gt 2 ]
+then
+  echo "Too many arguments supplied"
+  exit
+fi
+
+# Process outputs
+for ((YY=$YS;YY<=$YE;YY++))
+do
+
+ [ ! -d ${YY} ] && mkdir ${YY} 
+ [ ! -d rpts ] && mkdir rpts 
+ [ ! -d ARCHIVE ] && mkdir ARCHIVE
+
+ if [ -f $DIR/OUTPUT/*_scalar_*-${YY}.nc ]
+ then
+   mv $DIR/OUTPUT/*_scalar_*-${YY}.nc $DIR/TIDY/${YY}
+ fi
+
+ for ff in `ls *${YY}*rpt`
+ do
+   grep -q "NEMO rebuild completed successfully" $ff
+   if [ $? -eq 0 ] 
+   then 
+     mv $DIR/OUTPUT/${ff%.*}.nc $DIR/TIDY/${YY} &
+     mv $DIR/OUTPUT/${ff%.*}_????.nc $DIR/TIDY/ARCHIVE &
+     mv ${ff} rpts
+   fi
+ done
+
+done
+
+rm nam_rebuild_*
+
+exit

scripts/TIDY/chkit.script

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Generate md5sums for each file in a given year
+# Usage: ./chkit.script YYYY
+# Author: Adam Blaker 27/05/2021
+
+if [ $# -eq 0 ]
+then
+  echo "No arguments supplied"
+  exit
+fi
+
+export YY=$1
+
+ff=`ls ${YY}/*/*nc`
+
+xargs -n 1 -P 16 <<<$ff sh -c 'md5sum ${1} >> "${YY}"_files.txt' bash
+
+sort -k2 ${YY}_files.txt > ${YY}_md5s_files.txt
+
+rm ${YY}_files.txt

scripts/TIDY/combrest

@@ -0,0 +1,60 @@
+#!/bin/bash
+#SBATCH --job-name=combrest
+#SBATCH --time=16:20:0
+#SBATCH --ntasks=1
+#SBATCH --account=n01-CLASS
+#SBATCH --partition=serial
+#SBATCH --qos=serial
+#SBATCH --mem=40G
+
+export OMP_NUM_THREADS=1
+#============================
+TIME1=`date +%s`
+
+# Usage: sbatch {--export=YY="20070101"} combrest
+# Optionally supply restart subdiretory. If not provided script will attempt to recombine all available restarts.
+# Budget ~6 hours per restart.
+# Author: Adam Blaker (29/06/2022)
+
+
+echo "YY =" $YY
+
+# Array of restarts that need combining
+if [ ! -z "$YY" ]; then
+  A=(`find ../RESTARTS/${YY} -name "*_00000.nc"`)
+else
+  echo "No arguments supplied: try combining all restarts"
+  A=(`find ../RESTARTS -name "*_00000.nc"`)
+fi
+
+# Find length of the array
+Alen=${#A[@]}
+
+if [ $Alen = 0 ]; then
+  echo "No restarts found"
+  exit
+else
+  echo "Found" $Alen "restart to combine"
+fi
+
+# Find number of tiles  >> this will only work for 10^5 cores
+nf=`ls ${A[0]%_*}_?????.nc | wc -l`
+
+echo "Number of tiles =" $nf
+
+# Loop through restart files
+i="0"
+while [ $i -lt $Alen ]; do
+  echo /work/n01/n01/acc/NEMO/r4.0.5/dev_r4.0.5_NERC_latest/tools/REBUILD_NEMO/rebuild_nemo -d 1 ${A[$i]%_*} $nf
+  /work/n01/n01/acc/NEMO/r4.0.5/dev_r4.0.5_NERC_latest/tools/REBUILD_NEMO/rebuild_nemo -d 1 ${A[$i]%_*} $nf
+  i=$[$i+1]
+done
+
+#============================
+# Job timing
+
+TIME2=`date +%s`
+DIFFSEC=`expr ${TIME2} - ${TIME1}`
+echo Took ${DIFFSEC} seconds.
+echo Took `date +%H:%M:%S -ud @${DIFFSEC}`
+

scripts/TIDY/runcomb.slurm

@@ -0,0 +1,70 @@
+#!/bin/bash
+#SBATCH --job-name=rebuild
+#SBATCH --time=02:20:00
+#SBATCH --partition=compute
+#SBATCH --nodes=1
+#SBATCH --mem-per-cpu=4G
+#SBATCH --ntasks-per-core=1
+##SBATCH --ntasks-per-node=16
+##SBATCH --ntasks-per-socket=8
+#SBATCH --ntasks-per-node=32
+#SBATCH --ntasks-per-socket=16
+
+module load NEMO/prg-env
+
+#############################################
+# Run rebuild_nemo on all sets of output files in the OUTPUT directory
+# - Uses xargs to run N instances of rebuild_nemo in parallel "-P 16" 
+# - Note rebuild_nemo is also parallel "-p 4"
+#
+# A. Blaker 12/05/2021
+#############################################
+
+export I_MPI_SHM=icx
+echo "I_MPI_SHM = " $I_MPI_SHM
+export OMP_NUM_THREADS=4
+
+#============================
+TIME1=`date +%s`
+
+# This is dangerous - do not run whilst simulation is in progress
+ff=`ls ../OUTPUT/*scalar*000.nc`
+if [ ! -z "$ff" ]; then
+  xargs -n 1 -P 73 <<<$ff sh -c 'mv ${1} ${1%_*}.nc; rm ${1%_*}_????.nc' bash
+fi
+
+ff=`ls ../OUTPUT/*_1y*000.nc`
+##     ^^^^^^^^^^^^^^^^^^^^^
+##     Change this to point to your output folder/location
+if [ ! -z "$ff" ]; then
+  xargs -n 1 -P 8 <<<$ff sh -c '/dssgfs01/working/atb299/REBUILD_NEMO/rebuild_nemo -p 4 -d 1 ${1%_*} 24 &> `basename ${1%_*}`.rpt' bash
+#                                                                                                     ^^
+#                                                                                Set this to match the # XIOS servers you are using
+fi
+
+ff=`ls ../OUTPUT/*_1m*000.nc`
+##     ^^^^^^^^^^^^^^^^^^^^^
+##     Change this to point to your output folder/location
+if [ ! -z "$ff" ]; then
+  xargs -n 1 -P 8 <<<$ff sh -c '/dssgfs01/working/atb299/REBUILD_NEMO/rebuild_nemo -p 4 -d 1 ${1%_*} 24 &> `basename ${1%_*}`.rpt' bash
+#                                                                                                     ^^
+#                                                                                Set this to match the # XIOS servers you are using
+fi
+
+ff=`ls ../OUTPUT/*_1d*000.nc`
+##     ^^^^^^^^^^^^^^^^^^^^^
+##     Change this to point to your output folder/location
+if [ ! -z "$ff" ]; then
+  xargs -n 1 -P 8 <<<$ff sh -c '/dssgfs01/working/atb299/REBUILD_NEMO/rebuild_nemo -p 4 -d 1 ${1%_*} 24 &> `basename ${1%_*}`.rpt' bash
+#                                                                                                    ^^
+#                                                                                Set this to match the # XIOS servers you are using
+fi
+
+#============================
+# Job timing
+
+TIME2=`date +%s`
+DIFFSEC=`expr ${TIME2} - ${TIME1}`
+echo Took ${DIFFSEC} seconds.
+echo Took `date +%H:%M:%S -ud @${DIFFSEC}`
+

scripts/compile_nemo_Anemone

@@ -11,5 +11,5 @@ export OMP_NUM_THREADS=1
 # Compile tools
 while read -r LINE; do
   CFG=${LINE%% *}
-  ./makenemo -m Anemone-ifort -j 16 -r "$CFG" || exit
+  ./makenemo -m Anemone-ifort -j 32 -r "$CFG" || exit
 done <cfgs/work_cfgs.txt

scripts/compile_tools_Anemone

@@ -10,5 +10,5 @@ export OMP_NUM_THREADS=1
 
 # Compile tools
 for TOOL in DOMAINcfg REBUILD_NEMO WEIGHTS; do
-    ./maketools -m Anemone-ifort -j 16 -n $TOOL || exit
+    ./maketools -m Anemone-ifort -j 32 -n $TOOL || exit
 done

scripts/setup

@@ -4,8 +4,11 @@ WD=$PWD
 
 SYSTEM=Anemone
 
-BRANCH=4.2.0
-#COMMIT=
+#BRANCH=branch_4.2
+#COMMIT=405258c9896e0b1a3c430ea495fcda0fe55e2170
+
+BRANCH=main
+COMMIT=9a86d855a3ad9210b18ad8cf160dd5ecec4da466
 
 NEMODIR=${WD}/nemo
 STOP=0
@@ -82,6 +85,7 @@ ln -sfT INPUT_eORCA12_${SYSTEM} data/INPUT_eORCA12
 echo "Copying customized files into ${NEMODIR}"
 cp -r "${WD}"/{arch,cfgs,data,ext,scripts,tools} "$NEMODIR"/. || exit
 
+cp -r "${WD}"/scripts/TIDY "$NEMODIR"/cfgs/GLOBAL_QCO/EXP_MASTER/.
 
 ###########################################
 # TOOLS