Using MPAS-JEDI's own executable to calculate ensemble mean and variance

bin/MeanAnalysis.csh

@@ -10,77 +10,202 @@ date
 # Setup environment
 # =================
 source config/environmentJEDI.csh
+source config/mpas/variables.csh
 source config/tools.csh
 source config/auto/build.csh
+source config/auto/experiment.csh
 source config/auto/members.csh
+source config/auto/model.csh
+source config/auto/invariantstream.csh
+source config/auto/workflow.csh
 set yymmdd = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 1-8`
 set hh = `echo ${CYLC_TASK_CYCLE_POINT} | cut -c 10-11`
 set thisCycleDate = ${yymmdd}${hh}
 set thisValidDate = ${thisCycleDate}
 source ./bin/getCycleVars.csh
 
-# static work directory
-set self_WorkDir = $MeanAnalysisDirs[1]
+set self_WorkDir = $MeanAnalysisDirs[1] 
 echo "WorkDir = ${self_WorkDir}"
 mkdir -p ${self_WorkDir}
 cd ${self_WorkDir}
 
+# build, executable, yaml
+set myBuildDir = ${meanStateBuildDir}
+set myEXE = ${meanStateExe}
+set appyaml = ensMeanVariance.yaml
+set myYAML = ${self_WorkDir}/${appyaml}
+
 # other static variables
 set self_StateDirs = ($CyclingDAOutDirs)
 set self_StatePrefix = ${ANFilePrefix}
-set memberPrefix = ${self_StatePrefix}.${thisMPASFileDate}mem
+set memberPrefix = ${self_StatePrefix}.${thisMPASFileDate}.mem
 set meanName = ${self_StatePrefix}.$thisMPASFileDate.nc
 set varianceName = ${self_StatePrefix}.$thisMPASFileDate.variance.nc
 
 # ================================================================================================
-
-## link analysis members
+## Link background/analysis members
 set member = 1
 while ( $member <= ${nMembers} )
   set appMember = `${memberDir} 2 $member "{:03d}"`
-# set appMember = printf "%03d" $member`
   ln -sfv $self_StateDirs[$member]/${meanName} ./${memberPrefix}${appMember}
   @ member++
 end
 
 if (${nMembers} == 1) then
-  # TODO: just need to exit when follow-on applications receive states via python scripts
-
   ## pass-through for mean
   ln -sfv $self_StateDirs[1]/${meanName} ./
-
   echo "$0 (INFO): linked determinstic state for mean"
-
   exit 0
 endif
 
 ## make copy for mean
-cp $self_StateDirs[1]/${meanName} ./
+cp $self_StateDirs[1]/${meanName} ./${meanName}
 
 ## make copy for variance
 cp $self_StateDirs[1]/${meanName} ./${varianceName}
 
+# ===================================================
+## Prepare jedi-related files
+## copy invariant fields
+rm ${localInvariantFieldsPrefix}*.nc
+rm ${localInvariantFieldsPrefix}*.nc-lock
+set localInvariantFieldsFile = ${localInvariantFieldsFileEnsemble}
+rm ${localInvariantFieldsFile}
+set InvariantFieldsFile = ${InvariantFieldsDirEnsemble}/${InvariantFieldsFileEnsemble}
+ln -sfv ${InvariantFieldsFile} ${localInvariantFieldsFile}${OrigFileSuffix}
+cp -v ${InvariantFieldsFile} ${localInvariantFieldsFile}
+
+# ====================
+# Model-specific files
+# ====================
+## link MPAS mesh graph info
+ln -sfv $GraphInfoDir/x1.${nCellsEnsemble}.graph.info* .
+
+## link lookup tables
+foreach fileGlob ($MPASLookupFileGlobs)
+  ln -sfv ${MPASLookupDir}/*${fileGlob} .
+end
 
-# Run the executable
-# ==================
-set arg1 = ${self_WorkDir}
-set arg2 = ${meanName}
-set arg3 = ${varianceName}
-set arg4 = ${memberPrefix}
-set arg5 = ${nMembers}
+## link/copy stream_list/streams configs
+foreach staticfile ( \
+stream_list.${MPASCore}.background \
+stream_list.${MPASCore}.analysis \
+stream_list.${MPASCore}.ensemble \
+stream_list.${MPASCore}.control \
+)
+  ln -sfv $ModelConfigDir/ensMeanVariance/$staticfile .
+end
 
-ln -sfv ${meanStateBuildDir}/${meanStateExe} ./
-mpiexec ./${meanStateExe} "$arg1" "$arg2" "$arg3" "$arg4" "$arg5" >& log
+rm ${StreamsFile}
+cp -v $ModelConfigDir/ensMeanVariance/${StreamsFile} .
+sed -i 's@{{nCells}}@'${nCellsEnsemble}'@' ${StreamsFile}
+sed -i 's@{{TemplateFieldsPrefix}}@'${self_WorkDir}'/'${TemplateFieldsPrefix}'@' ${StreamsFile}
+sed -i 's@{{InvariantFieldsPrefix}}@'${self_WorkDir}'/'${localInvariantFieldsPrefix}'@' ${StreamsFile}
+sed -i 's@{{PRECISION}}@'${model__precision}'@' ${StreamsFile}
+
+# determine analysis output precision
+ncdump -h ${meanName} | grep uReconstruct | grep double
+if ($status == 0) then
+  set analysisPrecision=double
+else
+  ncdump -h ${meanName} | grep uReconstruct | grep float
+  if ($status == 0) then
+    set analysisPrecision=single
+  else
+    echo "ERROR in $0 : cannot determine analysis precision" > ./FAIL
+    exit 1
+  endif
+endif
+sed -i 's@{{analysisPRECISION}}@'${analysisPrecision}'@' ${StreamsFile}
+
+## copy/modify dynamic namelist
+## copy/modify dynamic namelist
+rm $NamelistFile
+cp -v $ModelConfigDir/ensMeanVariance/${NamelistFile} .
+sed -i 's@startTime@'${thisMPASNamelistDate}'@' $NamelistFile
+sed -i 's@blockDecompPrefix@'${self_WorkDir}'/x'${meshRatioEnsemble}'.'${nCellsEnsemble}'@' ${NamelistFile}
+sed -i 's@modelDT@'${TimeStepEnsemble}'@' $NamelistFile
+sed -i 's@diffusionLengthScale@'${DiffusionLengthScaleEnsemble}'@' $NamelistFile
+
+## modify namelist physics
+sed -i 's@radtlwInterval@'${RadiationLWIntervalEnsemble}'@' $NamelistFile
+sed -i 's@radtswInterval@'${RadiationSWIntervalEnsemble}'@' $NamelistFile
+sed -i 's@physicsSuite@'${PhysicsSuiteEnsemble}'@' $NamelistFile
+sed -i 's@micropScheme@'${MicrophysicsEnsemble}'@' $NamelistFile
+sed -i 's@convectionScheme@'${ConvectionEnsemble}'@' $NamelistFile
+sed -i 's@pblScheme@'${PBLEnsemble}'@' $NamelistFile
+sed -i 's@gwdoScheme@'${GwdoEnsemble}'@' $NamelistFile
+sed -i 's@radtCldScheme@'${RadiationCloudEnsemble}'@' $NamelistFile
+sed -i 's@radtLWScheme@'${RadiationLWEnsemble}'@' $NamelistFile
+sed -i 's@radtSWScheme@'${RadiationSWEnsemble}'@' $NamelistFile
+sed -i 's@sfcLayerScheme@'${SfcLayerEnsemble}'@' $NamelistFile
+sed -i 's@lsmScheme@'${LSMEnsemble}'@' $NamelistFile
+
+## MPASJEDI variable configs
+foreach file ($MPASJEDIVariablesFiles)
+  ln -sfv $ModelConfigDir/$file .
+end
+
+# =============
+# Generate yaml
+# =============
+## Copy jedi/applications yaml
+set thisYAML = orig.yaml
+cp -v ${ConfigDir}/jedi/applications/${appyaml} $thisYAML
+
+## streams
+sed -i 's@{{EnsembleStreamsFile}}@'${self_WorkDir}'/'${StreamsFile}'@' $thisYAML
+
+## namelist
+sed -i 's@{{EnsembleNamelistFile}}@'${self_WorkDir}'/'${NamelistFile}'@' $thisYAML
+
+## revise current date
+sed -i 's@{{thisISO8601Date}}@'${thisISO8601Date}'@g' $thisYAML
+
+# use one of the analyses as the TemplateFieldsFileOuter
+set meshFile = ${meanName}
+ln -sfv $meshFile ${TemplateFieldsFileOuter}
+
+# Variables necessary
+set StateVariables = ( \
+  $StandardStateVariables \
+  $MPASHydroStateVariables \
+)
+set VarSub = ""
+foreach var ($StateVariables)
+  set VarSub = "$VarSub$var,"
+end
+# remove trailing comma
+set VarSub = `echo "$VarSub" | sed 's/.$//'`
+sed -i 's@{{StateVariables}}@'$VarSub'@' $thisYAML
+
+## file naming
+sed -i 's@{{EnsembleMember}}@'${memberPrefix}'@g' $thisYAML
+sed -i 's@{{EnsembleNumber}}@'${nMembers}'@g' $thisYAML
+sed -i 's@{{EnsembleMeanFile}}@'${meanName}'@g' $thisYAML
+sed -i 's@{{EnsembleVarianceFile}}@'${varianceName}'@g' $thisYAML
+set prevYAML = $thisYAML
 
+## state and analysis variable configs
+mv $prevYAML $myYAML
+
+# Run the executable
+# ==================
+ln -sfv ${myBuildDir}/${myEXE} ./
+mpiexec ./${myEXE} $myYAML ./jedi.log >& jedi.log.all
 
 # Check status
 # ============
-grep 'All done' log
+grep 'Run: Finishing oops.* with status = 0' jedi.log
 if ( $status != 0 ) then
-  echo "$0 (ERROR): mean state application failed" > ./FAIL
+  echo "ERROR in $0 : jedi application failed" > ./FAIL
   exit 1
 endif
 
+## change static fields to a link, keeping for transparency
+rm ${localInvariantFieldsFile}
+mv ${localInvariantFieldsFile}${OrigFileSuffix} ${localInvariantFieldsFile}
+
 date
 
 exit 0