Skip to content

Updates to geotrace_cime for derecho #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rfiorella wants to merge 2 commits into
base: geotrace_cime
Choose a base branch
from
Open

Updates to geotrace_cime for derecho #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4587bd5
Updates to geotrace_cime for derecho
rfiorella Aug 19, 2024
36df70b
fix a derecho package issue
rfiorella Mar 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions config/cesm/machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,20 @@
</queues>
</batch_system>

<batch_system MACH="derecho" type="pbs" >
<batch_submit>qsub</batch_submit>
<submit_args>
<argument> -l job_priority=$JOB_PRIORITY </argument>
</submit_args>
<directives>
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
</directives>
<queues>
<queue walltimemax="4:00:00" nodemin="1" nodemax="2488" default="true">main</queue>
</queues>
</batch_system>

<batch_system MACH="eastwind" type="slurm" >
<batch_submit>sbatch</batch_submit>
<submit_args>
Expand Down
12 changes: 12 additions & 0 deletions config/cesm/machines/config_compilers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,18 @@ using a fortran linker.
</SLIBS>
</compiler>

<compiler MACH="derecho">
<CFLAGS>
<base> -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
<append MODEL="mpi-serial"> -std=gnu89 </append>
</CFLAGS>
<NETCDF_PATH>$ENV{NETCDF}</NETCDF_PATH>
<PNETCDF_PATH>$ENV{PNETCDF}</PNETCDF_PATH>
<SLIBS>
<append> -lnetcdff -lnetcdf </append>
</SLIBS>
</compiler>

<compiler MACH="eastwind" COMPILER="pgi">
<CFLAGS>
<append DEBUG="FALSE"> -O2 </append>
Expand Down
107 changes: 107 additions & 0 deletions config/cesm/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -906,6 +906,113 @@ This allows using a different mpirun command to launch unit tests
</environment_variables>
</machine>

<machine MACH="derecho">
<DESC>NCAR AMD EPYC </DESC>
<NODENAME_REGEX>de.*.hpc.ucar.edu</NODENAME_REGEX>
<OS>CNL</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>$ENV{CESMDATAROOT}/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>$ENV{CESMDATAROOT}/ccsm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>$ENV{CESMDATAROOT}/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>16</GMAKE_J>
<BATCH_SYSTEM>pbs</BATCH_SYSTEM>
<SUPPORTED_BY>cseg</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
<arguments>
<arg name="label"> --label</arg>
<arg name="buffer"> --line-buffer</arg>
<arg name="num_tasks" > -n {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="perl">$LMOD_ROOT/lmod/init/perl</init_path>
<init_path lang="python">$LMOD_ROOT/lmod/init/env_modules_python.py</init_path>
<init_path lang="sh">$LMOD_ROOT/lmod/init/sh</init_path>
<init_path lang="csh">$LMOD_ROOT/lmod/init/csh</init_path>
<cmd_path lang="perl">$LMOD_ROOT/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">$LMOD_ROOT/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="load">cesmdev/1.0</command>
<command name="load">ncarenv/23.09</command>
<command name="purge"/>
<command name="load">craype</command>
<command name="load">cmake</command>
</modules>
<modules compiler="intel">
<command name="load">intel/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="intel-oneapi">
<command name="load">intel-oneapi/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="intel-classic">
<command name="load">intel-classic/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="cray">
<command name="load">cce/15.0.1</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="gnu">
<command name="load">gcc/12.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="nvhpc">
<command name="load">nvhpc/23.7</command>
</modules>
<modules>
<command name="load">ncarcompilers/1.0.0</command>
<command name="load">cmake</command>
</modules>
<modules mpilib="mpich">
<command name="load">cray-mpich/8.1.27</command>
</modules>
<modules mpilib="mpi-serial">
<command name="load">mpi-serial/2.3.0</command>
</modules>

<modules mpilib="mpi-serial">
<command name="load">netcdf/4.9.2</command>
</modules>

<modules mpilib="!mpi-serial">
<command name="load">netcdf-mpi/4.9.2</command>
<command name="load">parallel-netcdf/1.12.3</command>
</modules>
<modules DEBUG="FALSE">
<command name="load">parallelio/2.6.2</command>
<command name="load">esmf/8.5.0</command>
</modules>
<modules DEBUG="TRUE">
<command name="load">parallelio/2.6.2-debug</command>
<command name="load">esmf/8.5.0</command>
</modules>

</module_system>

<environment_variables>
<env name="NETCDF_FORTRAN_PATH">/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/packages/netcdf-fortran/4.6.1/oneapi/2023.2.1/vfpj</env>
<env name="NETCDF_C_PATH">/glade/u/apps/derecho/23.09/spack/opt/spack/netcdf/4.9.2/packages/netcdf-c/4.9.2/oneapi/2023.2.1/cnx4</env>
<env name="OMP_STACKSIZE">64M</env>
<env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
<env name="FI_MR_CACHE_MONITOR">memhooks</env>
</environment_variables>
<environment_variables mpilib="mpich">
<env name="MPICH_MPIIO_HINTS">*:romio_cb_read=enable:romio_cb_write=enable:striping_factor=24</env>
</environment_variables>
</machine>

<machine MACH="eastwind">
<DESC>PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM</DESC>
<OS>LINUX</OS>
Expand Down
23 changes: 17 additions & 6 deletions config/xml_schemas/config_batch.xsd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,19 @@

<xs:element name="submit_args">
<xs:complexType>
<xs:sequence>
<xs:element maxOccurs="unbounded" ref="arg"/>
</xs:sequence>
<xs:choice>
<xs:element name="arg" maxOccurs="unbounded">
<xs:complexType>
<xs:attribute name="flag" use="required"/>
<xs:attribute name="name"/>
</xs:complexType>
</xs:element>
<xs:element name="argument" maxOccurs="unbounded">
<xs:complexType mixed="true">
<xs:attribute name="job_queue"/>
</xs:complexType>
</xs:element>
</xs:choice>
</xs:complexType>
</xs:element>

Expand All @@ -147,16 +157,17 @@
<xs:element maxOccurs="unbounded" ref="directive"/>
</xs:sequence>
<xs:attribute ref="queue"/>
<xs:attribute name="compiler"/>
<xs:attribute name="mpilib"/>
<xs:attribute name="threaded" type="xs:boolean"/>
</xs:complexType>
</xs:element>

<xs:element name="directive">
<xs:complexType mixed="true">
<xs:attribute name="default"/>
<xs:attribute name="prefix"/>
<xs:attribute name="compiler"/>
<xs:attribute name="mpilib"/>
<xs:attribute name="threaded" type="xs:boolean"/>
<xs:attribute name="queue"/>
</xs:complexType>
</xs:element>

Expand Down
2 changes: 1 addition & 1 deletion src/build_scripts/buildlib.pio
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def buildlib(bldroot, installpath, case):
if not os.path.isdir(pio_dir):
os.makedirs(pio_dir)
casetools = case.get_value("CASETOOLS")
cmake_opts = "\"-D GENF90_PATH=$CIMEROOT/src/externals/genf90 \""
cmake_opts = '"-D GENF90_PATH=$CIMEROOT/src/externals/genf90 "'
stdargs = get_standard_makefile_args(case, shared_lib=True)

gmake_vars = "CASEROOT={caseroot} MODEL={pio_model} "\
Expand Down
9 changes: 9 additions & 0 deletions src/drivers/mct/cime_config/config_component.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,15 @@
<desc>List of job ids for most recent case.submit</desc>
</entry>

<entry id="JOB_PRIORITY">
<type>char</type>
<default_value>regular</default_value>
<valid_values>regular,premium,economy</valid_values>
<group>run_begin_stop_restart</group>
<file>env_run.xml</file>
<desc>job priority for systems supporting this option</desc>
</entry>

<!-- ===================================================================== -->
<!-- definitions archive -->
<!-- ===================================================================== -->
Expand Down
12 changes: 9 additions & 3 deletions src/externals/genf90/genf90.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,39 +34,45 @@
# foo(1, bar), foo(2, bar), foo(3, bar), ...

# defaults
my @types = qw(text real double int);
my @types = qw(text real double int short);
my $vtype = {'text' => 'character(len=*)',
'real' => 'real(r4)',
'double' => 'real(r8)',
'int' => 'integer(i4)',
'short' => 'integer(i2)',
'long' => 'integer(i8)',
'logical' => 'logical' };
my $itype = {'text' => 100,
'real' => 101,
'double' => 102,
'int' => 103,
'long' => 104,
'logical' => 105};
'logical' => 105,
'short' => 106};
my $itypename = {'text' => 'TYPETEXT',
'real' => 'TYPEREAL',
'double' => 'TYPEDOUBLE',
'int' => 'TYPEINT',
'short' => 'TYPESHORT',
'long' => 'TYPELONG',
'logical' => 'TYPELOGICAL'};
my $mpitype = {'text' => 'MPI_CHARACTER',
'real' => 'MPI_REAL4',
'short' => 'MPI_SHORT',
'double' => 'MPI_REAL8',
'int' => 'MPI_INTEGER'};
# Netcdf C datatypes
my $nctype = {'text' => 'text',
'real' => 'float',
'short' => 'short',
'double' => 'double',
'int' => 'int'};
# C interoperability types
my $ctype = {'text' => 'character(C_CHAR)',
'real' => 'real(C_FLOAT)',
'double' => 'real(C_DOUBLE)',
'int' => 'integer(C_INT)'};
'int' => 'integer(C_INT)',
'short' => 'integer(C_SHORT)'};



Expand Down