New URL for NEMO forge! http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.

mpp_nc_generic.h90 in NEMO/branches/2019/dev_r11470_HPC_12_mpi3/src/OCE/LBC – NEMO

Context Navigation

source: NEMO/branches/2019/dev_r11470_HPC_12_mpi3/src/OCE/LBC/mpp_nc_generic.h90 @ 11940

Last change on this file since 11940 was 11940, checked in by mocavero, 5 years ago
Add MPI3 neighbourhood collectives halo exchange in LBC and call it in tracer advection FCT scheme #2011
File size: 17.7 KB

Line
1	# define NAT_IN(k) cd_nat(k)
2	# define SGN_IN(k) psgn(k)
3	# define F_SIZE(ptab) kfld
4	# define OPT_K(k) ,ipf
5	# if defined DIM_2d
6	# define ARRAY_TYPE(i,j,k,l,f) TYPE(PTR_2D) , INTENT(inout) :: ptab(f)
7	# define ARRAY_IN(i,j,k,l,f) ptab(f)%pt2d(i,j)
8	# define K_SIZE(ptab) 1
9	# define L_SIZE(ptab) 1
10	# endif
11	# if defined DIM_3d
12	# define ARRAY_TYPE(i,j,k,l,f) TYPE(PTR_3D) , INTENT(inout) :: ptab(f)
13	# define ARRAY_IN(i,j,k,l,f) ptab(f)%pt3d(i,j,k)
14	# define K_SIZE(ptab) SIZE(ptab(1)%pt3d,3)
15	# define L_SIZE(ptab) 1
16	# endif
17	# if defined DIM_4d
18	# define ARRAY_TYPE(i,j,k,l,f) TYPE(PTR_4D) , INTENT(inout) :: ptab(f)
19	# define ARRAY_IN(i,j,k,l,f) ptab(f)%pt4d(i,j,k,l)
20	# define K_SIZE(ptab) SIZE(ptab(1)%pt4d,3)
21	# define L_SIZE(ptab) SIZE(ptab(1)%pt4d,4)
22	# endif
23
24	SUBROUTINE ROUTINE_NC( cdname, ptab, cd_nat, psgn, kfld, kfillmode, pfillval, lsend, lrecv, ihlcom )
25	INTEGER , INTENT(in ) :: kfld ! number of pt3d arrays
26	ARRAY_TYPE(:,:,:,:,:) ! array or pointer of arrays on which the boundary condition is applied
27	CHARACTER(len=*) , INTENT(in ) :: cdname ! name of the calling subroutine
28	CHARACTER(len=1) , INTENT(in ) :: NAT_IN(:) ! nature of array grid-points
29	REAL(wp) , INTENT(in ) :: SGN_IN(:) ! sign used across the north fold boundary
30	INTEGER , OPTIONAL, INTENT(in ) :: kfillmode ! filling method for halo over land (default = constant)
31	REAL(wp), OPTIONAL, INTENT(in ) :: pfillval ! background value (used at closed boundaries)
32	LOGICAL, DIMENSION(4),OPTIONAL, INTENT(in ) :: lsend, lrecv ! communication with other 4 proc
33	INTEGER ,OPTIONAL, INTENT(in ) :: ihlcom ! number of ranks and rows to be communicated
34	!
35	INTEGER :: ji, jj, jk, jl, jf ! dummy loop indices
36	INTEGER :: ipi, ipj, ipk, ipl, ipf ! dimension of the input array
37	INTEGER :: ishift, ishift2, idx ! local integers
38	INTEGER :: ierr
39	INTEGER :: ifill_we, ifill_ea, ifill_so, ifill_no
40	INTEGER :: ihl ! number of ranks and rows to be communicated
41	REAL(wp) :: zland
42	INTEGER , DIMENSION(MPI_STATUS_SIZE) :: istate ! for mpi_isend
43	REAL(wp), DIMENSION(:), ALLOCATABLE :: zsnd, zrcv ! halos arrays
44	INTEGER , DIMENSION(4) :: isizes ! number of elements to be sent/received
45	INTEGER , DIMENSION(4) :: idataty ! datatype of halos arrays
46	INTEGER (KIND=MPI_ADDRESS_KIND) idispls(4) ! displacement in halos arrays
47	LOGICAL :: llsend_we, llsend_ea, llsend_no, llsend_so ! communication send
48	LOGICAL :: llrecv_we, llrecv_ea, llrecv_no, llrecv_so ! communication receive
49	LOGICAL :: lldo_nfd ! do north pole folding
50
51	!!----------------------------------------------------------------------
52	!
53	! ----------------------------------------- !
54	! 0. local variables initialization !
55	! ----------------------------------------- !
56	!
57	ipk = K_SIZE(ptab) ! 3rd dimension
58	ipl = L_SIZE(ptab) ! 4th -
59	ipf = F_SIZE(ptab) ! 5th - use in "multi" case (array of pointers)
60	!
61	IF( PRESENT(ihlcom) ) THEN ; ihl = ihlcom
62	ELSE ; ihl = 1
63	END IF
64	!
65	IF( narea == 1 .AND. numcom == -1 ) CALL mpp_report( cdname, ipk, ipl, ipf, ld_lbc = .TRUE. )
66	!
67	IF ( PRESENT(lsend) .AND. PRESENT(lrecv) ) THEN
68	llsend_we = lsend(1) ; llsend_ea = lsend(2) ; llsend_so = lsend(3) ; llsend_no = lsend(4)
69	llrecv_we = lrecv(1) ; llrecv_ea = lrecv(2) ; llrecv_so = lrecv(3) ; llrecv_no = lrecv(4)
70	ELSE IF( PRESENT(lsend) .OR. PRESENT(lrecv) ) THEN
71	WRITE(ctmp1,*) ' E R R O R : Routine ', cdname, ' is calling lbc_lnk with only one of the two arguments lsend or lrecv'
72	WRITE(ctmp2,*) ' ========== '
73	CALL ctl_stop( ' ', ctmp1, ctmp2, ' ' )
74	ELSE ! send and receive with every neighbour
75	llsend_we = nbondi == 1 .OR. nbondi == 0 ! keep for compatibility, should be defined in mppini
76	llsend_ea = nbondi == -1 .OR. nbondi == 0 ! keep for compatibility, should be defined in mppini
77	llsend_so = nbondj == 1 .OR. nbondj == 0 ! keep for compatibility, should be defined in mppini
78	llsend_no = nbondj == -1 .OR. nbondj == 0 ! keep for compatibility, should be defined in mppini
79	llrecv_we = llsend_we ; llrecv_ea = llsend_ea ; llrecv_so = llsend_so ; llrecv_no = llsend_no
80	END IF
81
82
83	lldo_nfd = npolj /= 0 ! keep for compatibility, should be defined in mppini
84
85	zland = 0._wp ! land filling value: zero by default
86	IF( PRESENT( pfillval ) ) zland = pfillval ! set land value
87
88	! define the method we will use to fill the halos in each direction
89	IF( llrecv_we ) THEN ; ifill_we = jpfillmpi
90	ELSEIF( l_Iperio ) THEN ; ifill_we = jpfillperio
91	ELSEIF( PRESENT(kfillmode) ) THEN ; ifill_we = kfillmode
92	ELSE ; ifill_we = jpfillcst
93	END IF
94	!
95	IF( llrecv_ea ) THEN ; ifill_ea = jpfillmpi
96	ELSEIF( l_Iperio ) THEN ; ifill_ea = jpfillperio
97	ELSEIF( PRESENT(kfillmode) ) THEN ; ifill_ea = kfillmode
98	ELSE ; ifill_ea = jpfillcst
99	END IF
100	!
101	IF( llrecv_so ) THEN ; ifill_so = jpfillmpi
102	ELSEIF( l_Jperio ) THEN ; ifill_so = jpfillperio
103	ELSEIF( PRESENT(kfillmode) ) THEN ; ifill_so = kfillmode
104	ELSE ; ifill_so = jpfillcst
105	END IF
106	!
107	IF( llrecv_no ) THEN ; ifill_no = jpfillmpi
108	ELSEIF( l_Jperio ) THEN ; ifill_no = jpfillperio
109	ELSEIF( PRESENT(kfillmode) ) THEN ; ifill_no = kfillmode
110	ELSE ; ifill_no = jpfillcst
111	END IF
112	!
113	#if defined PRINT_CAUTION
114	!
115	! ================================================================================== !
116	! CAUTION: semi-column notation is often impossible because of the cpp preprocessing !
117	! ================================================================================== !
118	!
119	#endif
120	!
121	! -------------------------------------------------- !
122	! 1. Do west, east, south and north MPI exchange !
123	! -------------------------------------------------- !
124	!
125	! Allocate local temporary arrays to be sent/received. Fill arrays to be sent
126
127	ALLOCATE( zsnd (2 * ipk * ipl * ipf * ihl * ((jpi + jpj - 4*ihl) ) ) )
128	ALLOCATE( zrcv (2 * ipk * ipl * ipf * ihl * ((jpi + jpj - 4*ihl) ) ) )
129
130	zrcv(:)=-1
131	zsnd(:)=-1
132
133	idx = 1
134	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
135	zsnd(idx) = ARRAY_IN(ihl+ji,jj,jk,jl,jf)
136	idx = idx + 1
137	END DO ; END DO ; END DO ; END DO ; END DO
138
139	isizes(1) = ihl * (jpj - 2ihl) ipk * ipl * ipf
140	idispls(1) = 0
141
142	ishift = jpi-2*ihl
143
144	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
145	zsnd(idx) = ARRAY_IN(ishift+ji,jj,jk,jl,jf)
146	idx = idx + 1
147	END DO ; END DO ; END DO ; END DO ; END DO
148
149	isizes(2) = ihl * (jpj - 2ihl) ipk * ipl * ipf
150	idispls(2) = jpbyt*isizes(1)
151
152	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
153	zsnd(idx) = ARRAY_IN(ji,ihl+jj,jk,jl,jf)
154	idx = idx + 1
155	END DO ; END DO ; END DO ; END DO ; END DO
156
157	isizes(3) = (jpi - 2ihl) ihl * ipk * ipl * ipf
158	idispls(3) = jpbyt*(isizes(1)+isizes(2))
159
160	ishift = jpj-2*ihl
161
162	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
163	zsnd(idx) = ARRAY_IN(ji,ishift+jj,jk,jl,jf)
164	idx = idx + 1
165	END DO ; END DO ; END DO ; END DO ; END DO
166
167	isizes(4) = (jpi - 2ihl) ihl * ipk * ipl * ipf
168	idispls(4) = jpbyt*(isizes(1)+isizes(2)+isizes(3))
169
170	idataty(1) = MPI_DOUBLE_PRECISION
171	idataty(2) = MPI_DOUBLE_PRECISION
172	idataty(3) = MPI_DOUBLE_PRECISION
173	idataty(4) = MPI_DOUBLE_PRECISION
174
175	IF( ln_timing ) CALL tic_tac(.TRUE.)
176
177	CALL mpi_neighbor_alltoallw (zsnd, isizes, idispls, idataty, zrcv, isizes, idispls, idataty, mpi_nc_com, ierr)
178
179	IF( ln_timing ) CALL tic_tac(.FALSE.)
180
181	! --------------------------------------------------- !
182	! 2. Fill east and west north and south halos !
183	! --------------------------------------------------- !
184	!
185	! 2.1 fill weastern halo
186	! ----------------------
187	idx = 1
188	SELECT CASE ( ifill_we )
189	CASE ( jpfillnothing ) ! no filling
190	CASE ( jpfillmpi ) ! use data received by MPI
191	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
192	ARRAY_IN(ji,jj,jk,jl,jf) = zrcv(idx) ! 1 -> ihl
193	idx = idx + 1
194	END DO; END DO ; END DO ; END DO ; END DO
195	CASE ( jpfillperio ) ! use east-weast periodicity
196	ishift2 = jpi - 2 * ihl
197	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
198	ARRAY_IN(ji,jj,jk,jl,jf) = ARRAY_IN(ishift2+ji,jj,jk,jl,jf)
199	END DO; END DO ; END DO ; END DO ; END DO
200	idx = idx + (jpj-2ihl)ihlipkipl*ipf
201	CASE ( jpfillcopy ) ! filling with inner domain values
202	DO jf = 1, ipf ! number of arrays to be treated
203	IF( .NOT. NAT_IN(jf) == 'F' ) THEN ! do nothing for F point
204	DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
205	ARRAY_IN(ji,jj,jk,jl,jf) = ARRAY_IN(ihl+1,jj,jk,jl,jf)
206	END DO ; END DO ; END DO ; END DO
207	ENDIF
208	END DO
209	idx = idx + (jpj-2ihl)ihlipkipl*ipf
210	CASE ( jpfillcst ) ! filling with constant value
211	DO jf = 1, ipf ! number of arrays to be treated
212	IF( .NOT. NAT_IN(jf) == 'F' ) THEN ! do nothing for F point
213	DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
214	ARRAY_IN(ji,jj,jk,jl,jf) = zland
215	END DO; END DO ; END DO ; END DO
216	ENDIF
217	END DO
218	idx = idx + (jpj-2ihl)ihlipkipl*ipf
219	END SELECT
220	!
221	! 2.2 fill eastern halo
222	! ---------------------
223	ishift = jpi - ihl ! fill halo from ji = jpi-ihl+1 to jpi
224	SELECT CASE ( ifill_ea )
225	CASE ( jpfillnothing ) ! no filling
226	CASE ( jpfillmpi ) ! use data received by MPI
227	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
228	ARRAY_IN(ishift+ji,jj,jk,jl,jf) = zrcv(idx) ! jpi - ihl + 1 -> jpi
229	idx = idx + 1
230	END DO ; END DO ; END DO ; END DO ; END DO
231	CASE ( jpfillperio ) ! use east-weast periodicity
232	ishift2 = ihl
233	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
234	ARRAY_IN(ishift+ji,jj,jk,jl,jf) = ARRAY_IN(ishift2+ji,jj,jk,jl,jf)
235	END DO ; END DO ; END DO ; END DO ; END DO
236	idx = idx + (jpj-2ihl)ihlipkipl*ipf
237	CASE ( jpfillcopy ) ! filling with inner domain values
238	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
239	ARRAY_IN(ishift+ji,jj,jk,jl,jf) = ARRAY_IN(ishift,jj,jk,jl,jf)
240	END DO ; END DO ; END DO ; END DO ; END DO
241	idx = idx + (jpj-2ihl)ihlipkipl*ipf
242	CASE ( jpfillcst ) ! filling with constant value
243	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = ihl + 1, jpj - ihl ; DO ji = 1, ihl
244	ARRAY_IN(ishift+ji,jj,jk,jl,jf) = zland
245	END DO; END DO ; END DO ; END DO ; END DO
246	idx = idx + (jpj-2ihl)ihlipkipl*ipf
247	END SELECT
248	!
249	! 2.3 fill southern halo
250	! ----------------------
251	SELECT CASE ( ifill_so )
252	CASE ( jpfillnothing ) ! no filling
253	CASE ( jpfillmpi ) ! use data received by MPI
254	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
255	ARRAY_IN(ji,jj,jk,jl,jf) = zrcv(idx) ! 1 -> ihl
256	idx = idx + 1
257	END DO; END DO ; END DO ; END DO ; END DO
258	CASE ( jpfillperio ) ! use north-south periodicity
259	ishift2 = jpj - 2 * ihl
260	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
261	ARRAY_IN(ji,jj,jk,jl,jf) = ARRAY_IN(ji,ishift2+jj,jk,jl,jf)
262	END DO; END DO ; END DO ; END DO ; END DO
263	idx = idx + (jpi-2ihl)ihlipkipl*ipf
264	CASE ( jpfillcopy ) ! filling with inner domain values
265	DO jf = 1, ipf ! number of arrays to be treated
266	IF( .NOT. NAT_IN(jf) == 'F' ) THEN ! do nothing for F point
267	DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
268	ARRAY_IN(ji,jj,jk,jl,jf) = ARRAY_IN(ji,ihl+1,jk,jl,jf)
269	END DO ; END DO ; END DO ; END DO
270	ENDIF
271	END DO
272	idx = idx + (jpi-2ihl)ihlipkipl*ipf
273	CASE ( jpfillcst ) ! filling with constant value
274	DO jf = 1, ipf ! number of arrays to be treated
275	IF( .NOT. NAT_IN(jf) == 'F' ) THEN ! do nothing for F point
276	DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
277	ARRAY_IN(ji,jj,jk,jl,jf) = zland
278	END DO; END DO ; END DO ; END DO
279	ENDIF
280	END DO
281	idx = idx + (jpi-2ihl)ihlipkipl*ipf
282	END SELECT
283	!
284	! 2.4 fill northern halo
285	! ----------------------
286	ishift = jpj - ihl ! fill halo from jj = jpj-ihl+1 to jpj
287	SELECT CASE ( ifill_no )
288	CASE ( jpfillnothing ) ! no filling
289	CASE ( jpfillmpi ) ! use data received by MPI
290	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
291	ARRAY_IN(ji,ishift+jj,jk,jl,jf) = zrcv(idx) ! jpj-ihl+1 -> jpj
292	idx = idx + 1
293	END DO ; END DO ; END DO ; END DO ; END DO
294	CASE ( jpfillperio ) ! use north-south periodicity
295	ishift2 = ihl
296	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
297	ARRAY_IN(ji,ishift+jj,jk,jl,jf) = ARRAY_IN(ji,ishift2+jj,jk,jl,jf)
298	END DO; END DO ; END DO ; END DO ; END DO
299	idx = idx + (jpi-2ihl)ihlipkipl*ipf
300	CASE ( jpfillcopy ) ! filling with inner domain values
301	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
302	ARRAY_IN(ji,ishift+jj,jk,jl,jf) = ARRAY_IN(ji,ishift,jk,jl,jf)
303	END DO; END DO ; END DO ; END DO ; END DO
304	idx = idx + (jpi-2ihl)ihlipkipl*ipf
305	CASE ( jpfillcst ) ! filling with constant value
306	DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1, ihl ; DO ji = ihl + 1, jpi - ihl
307	ARRAY_IN(ji,ishift+jj,jk,jl,jf) = zland
308	END DO; END DO ; END DO ; END DO ; END DO
309	idx = idx + (jpi-2ihl)ihlipkipl*ipf
310	END SELECT
311	!
312	! -------------------------------------------- !
313	! 3. deallocate local temporary arrays !
314	! -------------------------------------------- !
315	!
316	DEALLOCATE( zsnd )
317	DEALLOCATE( zrcv )
318	!
319	! ------------------------------- !
320	! 4. north fold treatment !
321	! ------------------------------- !
322	!
323	IF( lldo_nfd .AND. ifill_no /= jpfillnothing ) THEN
324	!
325	SELECT CASE ( jpni )
326	CASE ( 1 ) ; CALL lbc_nfd( ptab, NAT_IN(:), SGN_IN(:) OPT_K(:) ) ! only 1 northern proc, no mpp
327	CASE DEFAULT ; CALL mpp_nfd( ptab, NAT_IN(:), SGN_IN(:) OPT_K(:) ) ! for all northern procs.
328	END SELECT
329	!
330	ifill_no = jpfillnothing ! force to do nothing for the northern halo as we just done the north pole folding
331	!
332	ENDIF
333
334	END SUBROUTINE ROUTINE_NC
335
336	#undef ARRAY_TYPE
337	#undef NAT_IN
338	#undef SGN_IN
339	#undef ARRAY_IN
340	#undef K_SIZE
341	#undef L_SIZE
342	#undef F_SIZE
343	#undef OPT_K

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format