Skip to content

Adjust store directives, so that it compiles with MITgcm PR862 without any intervention #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mjlosch wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

Adjust store directives, so that it compiles with MITgcm PR862 without any intervention #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fd7d31c
this seems to work with MITgcm PR862 without any intervention
mjlosch Sep 30, 2024
53c9e27
add feeble attempts to get rid of final recomputation warning
mjlosch Sep 30, 2024
d540da9
remove one store directive for njj
mjlosch Sep 30, 2024
73dbfef
more global tapes for gcmSSP, more comments and pointers
mjlosch Oct 7, 2024
26883ff
simplify logic and remove all complexity and recomputations
mjlosch Oct 7, 2024
5110481
fine tuning
mjlosch Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mitgcm_code/forward_step.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1145,6 +1145,8 @@ SUBROUTINE FORWARD_STEP( iloop, myTime, myIter, myThid )
C-- Acoustical physics
CALL IHOP_SOUND_SPEED( myThid )
CALL IHOP_DRIVER( myTime,myIter,myThid )
CML I think that this could go here instead of being called twice in the_main_loop
CML CALL IHOP_COST_INLOOP( myTime, myThid )
#endif /* ALLOW_IHOP */

#ifdef ALLOW_DIAGNOSTICS
Expand Down
64 changes: 61 additions & 3 deletions mitgcm_code/the_main_loop.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,16 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
C \ev

C !USES:
#ifdef ALLOW_AUTODIFF_TAMC
# ifdef ALLOW_IHOP
use arr_mod, only: arr, narr
use bdry_mod, only: bdry
use ssp_mod, only: ssp
use srpos_mod, only: pos
use angle_mod, only: angles
use ihop_mod, only: beam, ray2d
# endif
#endif
IMPLICIT NONE
C == Global variables ==
#include "SIZE.h"
Expand Down Expand Up @@ -428,6 +438,11 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
max_lev2=nTimeSteps/nchklev_1+1

C Declaring variables as passive does not work properly
CML# ifdef ALLOW_IHOP
CML!$TAF PASSIVE angles
CML# endif

c**************************************
# ifdef ALLOW_DIVIDED_ADJOINT
CADJ loop = divided
Expand All @@ -443,6 +458,10 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
CALL AUTODIFF_STORE( myThid )
#include "checkpoint_lev4_directives.h"
# ifdef ALLOW_IHOP
!$TAF STORE arr, bdry, narr, pos, ssp = tapelev4, key = ilev_4
!$TAF STORE angles, beam, ray2d = tapelev4, key = ilev_4
# endif
CALL AUTODIFF_RESTORE( myThid )
# ifdef ALLOW_AUTODIFF_WHTAPEIO
CALL AUTODIFF_WHTAPEIO_SYNC( 4 , 1, myThid )
Expand All @@ -462,6 +481,10 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
CALL AUTODIFF_STORE( myThid )
#include "checkpoint_lev3_directives.h"
# ifdef ALLOW_IHOP
!$TAF STORE arr, bdry, narr, pos, ssp = tapelev3, key = ilev_3
!$TAF STORE angles, beam, ray2d = tapelev3, key = ilev_3
# endif
CALL AUTODIFF_RESTORE( myThid )
# ifdef ALLOW_AUTODIFF_WHTAPEIO
CALL AUTODIFF_WHTAPEIO_SYNC( 3 , 1, myThid )
Expand All @@ -480,6 +503,19 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
# endif
CALL AUTODIFF_STORE( myThid )
#include "checkpoint_lev2_directives.h"
# ifdef ALLOW_IHOP
CML!$TAF PASSIVE angles
CML!$TAF STORE arr, bdry, narr, pos, ssp = tapelev2, key = ilev_2
CML!$TAF STORE angles, beam, ray2d = tapelev2, key = ilev_2
!$TAF STORE arr, narr, ray2d = tapelev2, key = ilev_2
!$TAF STORE angles%dalpha = tapelev2, key = ilev_2
!$TAF STORE bdry%bot, bdry%top = tapelev2, key = ilev_2
!$TAF STORE beam%nsteps = tapelev2, key = ilev_2
!$TAF STORE pos%rz, pos%sz = tapelev2, key = ilev_2
!$TAF STORE ssp%c, ssp%cmat = tapelev2, key = ilev_2
!$TAF STORE ssp%cz, ssp%czmat = tapelev2, key = ilev_2
!$TAF STORE ssp%rho = tapelev2, key = ilev_2
Comment on lines +510 to +517
Copy link
Author

@mjlosch mjlosch Sep 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be done in a similar way for tapelevel3 and tapelevel4.

Copy link
Author

@mjlosch mjlosch Oct 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use the PASSIVE statement does not work properly

# endif
CALL AUTODIFF_RESTORE( myThid )
# ifdef ALLOW_AUTODIFF_WHTAPEIO
CALL AUTODIFF_WHTAPEIO_SYNC( 2 , 1, myThid )
Expand Down Expand Up @@ -520,7 +556,11 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
c--
# ifdef ALLOW_IHOP
! for ssp_mod>gcmSSP; make local?
!$TAF INIT comlev1_bibj_ij_ihop = COMMON, nchklev_1*nSx*nSy*(2*OLy+sNy)*(2*OLx+sNx)
!$TAF INIT comlev1_ihop = STATIC, nchklev_1
!$TAF INIT comlev1_ihop_nts = STATIC, nchklev_1*nts
C global tapes for gcmSSP
!$TAF INIT comlev1_ihop_iijj = STATIC, nchklev_1*nSx*nSy*sNx*sNy*IHOP_MAX_RANGE*IHOP_MAX_NC_SIZE
!$TAF INIT comlev1_ihop_iijj_k = STATIC, nchklev_1*nSx*nSy*sNx*sNy*IHOP_MAX_RANGE*IHOP_MAX_NC_SIZE*(Nr + 2)
# endif /* ALLOW_IHOP */
c--
# ifdef ALLOW_MOM_COMMON
Expand Down Expand Up @@ -698,16 +738,30 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
ENDIF
#endif
#ifdef ALLOW_IHOP
# ifdef ALLOW_AUTODIFF_TAMC
CML!$TAF PASSIVE angles
CML!$TAF STORE arr,narr,bdry,pos,ssp = comlev1_ihop, key = ikey_dynamics
!$TAF STORE arr, narr, ray2d = comlev1_ihop, key = ikey_dynamics
!$TAF STORE angles%dalpha = comlev1_ihop, key = ikey_dynamics
!$TAF STORE bdry%bot, bdry%top = comlev1_ihop, key = ikey_dynamics
!$TAF STORE beam%nsteps = comlev1_ihop, key = ikey_dynamics
!$TAF STORE pos%rz, pos%sz = comlev1_ihop, key = ikey_dynamics
CML!$TAF STORE ssp%c, ssp%cmat = comlev1_ihop, key = ikey_dynamics
CML!$TAF STORE ssp%cz, ssp%czmat = comlev1_ihop, key = ikey_dynamics
C for some reason TAF does not do this right, so we store ssp instead
CML!$TAF STORE ssp%rho = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp = comlev1_ihop, key = ikey_dynamics
#endif
IF (useIHOP) THEN
#ifdef ALLOW_DEBUG
IF (debugMode) CALL DEBUG_CALL('ihop_cost_inloop',myThid)
#endif
c-- Accumulate in-situ acoutsic travel times.
#ifdef ALLOW_AUTODIFF
# ifdef ALLOW_AUTODIFF
C-- Reset the model iteration counter and the model time.
myIter = nIter0 + (iloop-1)
myTime = startTime + deltaTClock*(iloop-1)
#endif
# endif
CALL TIMER_START('IHOP_COST_INLOOP [MAIN_DO_LOOP]', myThid)
CALL IHOP_COST_INLOOP( myTime, myThid )
CALL TIMER_STOP ('IHOP_COST_INLOOP [MAIN_DO_LOOP]', myThid)
Expand Down Expand Up @@ -771,6 +825,10 @@ SUBROUTINE THE_MAIN_LOOP( myTime, myIter, myThid )
#endif
#ifdef ALLOW_IHOP
IF (useIHOP) THEN
# ifdef ALLOW_AUTODIFF
!$TAF STORE arr, narr = comlev1_ihop, key = ikey_dynamics
!$TAF STORE pos%rz, pos%sz = comlev1_ihop, key = ikey_dynamics
# endif
c-- Accumulate travel times
CALL TIMER_START('IHOP_COST_INLOOP [THE_MAIN_LOOP]', myThid)
CALL IHOP_COST_INLOOP( endtime, myThid )
Expand Down
18 changes: 15 additions & 3 deletions src/ihop.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
#include "IHOP_OPTIONS.h"
#ifdef ALLOW_AUTODIFF
# include "AUTODIFF_OPTIONS.h"
#endif
!BOP
! !ROUTINE: IHOP
! !INTERFACE:
Expand Down Expand Up @@ -59,10 +62,16 @@ SUBROUTINE IHOP_MAIN ( myTime, myIter, myThid )
USE ihop_init_diag, only: initPRTFile, openOutputFiles, resetMemory
USE bdry_mod, only: Bdry, writeBdry
USE ssp_mod, only: setSSP
USE refCoef, only: writeRefCoef
#ifdef ALLOW_AUTODIFF_TAMC
USE ssp_mod, only: SSP
#endif
USE refCoef, only: writeRefCoef
USE beampat, only: writePat
USE ihop_mod, only: Beam

#ifdef ALLOW_AUTODIFF
# include "tamc.h"
#endif
! == Routine Arguments ==
! myThid :: Thread number. Unused by IESCO
! msgBuf :: Used to build messages for printing.
Expand Down Expand Up @@ -91,16 +100,19 @@ SUBROUTINE IHOP_MAIN ( myTime, myIter, myThid )
CALL writeBdry ( myThid )

! write refCoef: OPTIONAL
CALL writeRefCoef( myThid )
CALL writeRefCoef( myThid )

! Source Beam Pattern: OPTIONAL, default is omni source pattern
CALL writePat( myThid )


! set SSP%cmat from gcm SSP: REQUIRED
#ifdef ALLOW_AUTODIFF_TAMC
!$TAF STORE ssp%c, ssp%cz = comlev1_ihop, key = ikey_dynamics
!$TAF STORE ssp%czmat, ssp%rho = comlev1_ihop, key = ikey_dynamics
#endif
CALL setSSP( myThid )


! open all output files
IF ( IHOP_dumpfreq .GE. 0 ) &
CALL OpenOutputFiles( IHOP_fileroot, myTime, myIter, myThid )
Expand Down
4 changes: 0 additions & 4 deletions src/ihop_cost_inloop.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,6 @@ SUBROUTINE IHOP_COST_INLOOP( myTime, myThid )
& .AND.(ihopObs_time(num_file,m,bi,bj).LT.
& (myTime+deltaTclock))) THEN

#ifdef ALLOW_AUTODIFF_TAMC
!$TAF STORE ihop_modval = comlev1_bibj, key=itdkey, byte=isbyte
#endif

ihop_modval=0

CALL ihop_cost_modval(ihop_modval, num_file,
Expand Down
10 changes: 6 additions & 4 deletions src/ihop_driver.F
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,18 @@ SUBROUTINE IHOP_DRIVER( myTime, myIter, myThid )

C !LOCAL VARIABLES: ====================================================
INTEGER t
LOGICAL do_ihop_main

CEOP

#ifdef ALLOW_IHOP
do_ihop_main=.FALSE.
DO t=1,nts
IF ( IHOP_iter(t).GE.0 ) THEN
IF ( IHOP_iter(t).EQ.myIter ) THEN
CALL IHOP_MAIN( myTime, myIter, myThid )
ENDIF
IF ( IHOP_iter(t).GE.0 .AND. IHOP_iter(t).EQ.myIter ) THEN
do_ihop_main=.TRUE.
ENDIF
ENDDO
IF ( do_ihop_main ) CALL IHOP_MAIN( myTime, myIter, myThid )
#endif /* ALLOW_IHOP */

RETURN
Expand Down
20 changes: 13 additions & 7 deletions src/splinec_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,9 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

L = N - 1

!$TAF init cspline1 = static, 4*ndim*L
!$TAF init cspline1 = static, 1
!$TAF init cspline1_ndim = static, NDIM
!ML!$TAF init cspline1 = static, 4*NDIM*L ! not sure if this correct

DO M = 2,N
C(3,M) = TAU(M) - TAU(M-1)
Expand All @@ -88,7 +90,7 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

! * BEGINNING BOUNDARY CONDITION SECTION *

!$TAF store C = cspline1
!$TAF INCOMPLETE C
IF (IBCBEG==0) THEN ! IBCBEG = 0
!$TAF store C = cspline1
IF (N.GT.2) THEN ! N > 2
Expand Down Expand Up @@ -121,17 +123,20 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)
! * RUNNING CALCULATIONS TO N-1 - LOOP IS NOT EXECUTED IF N = 2 *

DO M = 2,L
!$TAF store C = cspline1
!$TAF store C = cspline1_ndim, key = M
G = -C(3,M+1) / C(4,M-1)
C(2,M) = G*C(2,M-1) + 3.0*(C(3,M)*C(4,M+1) + C(3,M+1)*C(4,M))
C(4,M) = G*C(3,M-1) + 2.0*(C(3,M) + C(3,M+1))
END DO
!$TAF store G = cspline1

! * ENDING BOUNDARY CONDITION SECTION *

!$TAF INCOMPLETE C
IF (IBCEND /= 1) THEN
!$TAF store C = cspline1
IF (IBCEND==0) THEN
!$TAF INCOMPLETE C
IF (N==2 .AND. IBCBEG==0) THEN
C(2,N) = C(4,N)
ELSE IF ((N==3 .AND. IBCBEG==0) .OR. N==2) THEN
Expand All @@ -154,17 +159,18 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)
G = -1.0 / C(4,N-1)
ELSE
! do nothing
C(2,N) = C(2,N)
C(4,N) = C(4,N)
C(2,N) = C(2,N)
C(4,N) = C(4,N)
G = G
END IF
ENDIF

IF ( IBCBEG.GT.0 .OR. N.GT.2) THEN
!$TAF store C = cspline1
C(4,N) = G*C(3,N-1) + C(4,N)
C(2,N) = (G*C(2,N-1) + C(2,N)) / C(4,N)
END IF
END IF
!$TAF store C = cspline1

! * RUN THE ENDING BOUNDARY EFFECT BACK THROUGH THE EQUATIONS *

Expand Down Expand Up @@ -194,7 +200,7 @@ SUBROUTINE CSPLINE (TAU, C, N, IBCBEG, IBCEND, NDIM)

C(4,N) = (0.0,0.0)
DO I = 1,L ! INTEGRATE OVER THE INTERVAL
!$TAF store C = cspline1
!$TAF store C(4,:) = cspline1_ndim, key = I
DTAU = TAU(I+1) - TAU(I)
C(4,N) = C(4,N) + DTAU*(C(1,I) + DTAU*(C(2,I)/2.0 + &
DTAU*(C(3,I)/6.0 + DTAU*C(4,I)/24.0)))
Expand Down
Loading