1 | #LyX file created by tex2lyx 2.3 |
---|
2 | \lyxformat 544 |
---|
3 | \begin_document |
---|
4 | \begin_header |
---|
5 | \save_transient_properties true |
---|
6 | \origin /homel/ywang/Documents/MPI_Endpoints/Note/ |
---|
7 | \textclass article |
---|
8 | \begin_preamble |
---|
9 | \usepackage{listings} |
---|
10 | \usepackage[usenames,dvipsnames,svgnames,table]{xcolor} |
---|
11 | |
---|
12 | |
---|
13 | % Title Page |
---|
14 | \title{Note for MPI Endpoints} |
---|
15 | \author{} |
---|
16 | |
---|
17 | |
---|
18 | |
---|
19 | \end_preamble |
---|
20 | \use_default_options false |
---|
21 | \maintain_unincluded_children false |
---|
22 | \language english |
---|
23 | \language_package none |
---|
24 | \inputencoding utf8 |
---|
25 | \fontencoding default |
---|
26 | \font_roman "default" "default" |
---|
27 | \font_sans "default" "default" |
---|
28 | \font_typewriter "default" "default" |
---|
29 | \font_math "auto" "auto" |
---|
30 | \font_default_family default |
---|
31 | \use_non_tex_fonts false |
---|
32 | \font_sc false |
---|
33 | \font_osf false |
---|
34 | \font_sf_scale 100 100 |
---|
35 | \font_tt_scale 100 100 |
---|
36 | \use_microtype false |
---|
37 | \use_dash_ligatures true |
---|
38 | \graphics default |
---|
39 | \default_output_format default |
---|
40 | \output_sync 0 |
---|
41 | \bibtex_command default |
---|
42 | \index_command default |
---|
43 | \paperfontsize 10 |
---|
44 | \spacing single |
---|
45 | \use_hyperref false |
---|
46 | \papersize a4paper |
---|
47 | \use_geometry false |
---|
48 | \use_package amsmath 2 |
---|
49 | \use_package amssymb 0 |
---|
50 | \use_package cancel 0 |
---|
51 | \use_package esint 1 |
---|
52 | \use_package mathdots 0 |
---|
53 | \use_package mathtools 0 |
---|
54 | \use_package mhchem 0 |
---|
55 | \use_package stackrel 0 |
---|
56 | \use_package stmaryrd 0 |
---|
57 | \use_package undertilde 0 |
---|
58 | \cite_engine basic |
---|
59 | \cite_engine_type default |
---|
60 | \biblio_style plain |
---|
61 | \use_bibtopic false |
---|
62 | \use_indices false |
---|
63 | \paperorientation portrait |
---|
64 | \suppress_date false |
---|
65 | \justification true |
---|
66 | \use_refstyle 0 |
---|
67 | \use_minted 0 |
---|
68 | \index Index |
---|
69 | \shortcut idx |
---|
70 | \color #008000 |
---|
71 | \end_index |
---|
72 | \secnumdepth 3 |
---|
73 | \tocdepth 3 |
---|
74 | \paragraph_separation indent |
---|
75 | \paragraph_indentation default |
---|
76 | \is_math_indent 0 |
---|
77 | \math_numbering_side default |
---|
78 | \quotes_style english |
---|
79 | \dynamic_quotes 0 |
---|
80 | \papercolumns 1 |
---|
81 | \papersides 1 |
---|
82 | \paperpagestyle default |
---|
83 | \tracking_changes false |
---|
84 | \output_changes false |
---|
85 | \html_math_output 0 |
---|
86 | \html_css_as_file 0 |
---|
87 | \html_be_strict false |
---|
88 | \end_header |
---|
89 | |
---|
90 | \begin_body |
---|
91 | |
---|
92 | \begin_layout Standard |
---|
93 | |
---|
94 | \begin_inset ERT |
---|
95 | status collapsed |
---|
96 | |
---|
97 | \begin_layout Plain Layout |
---|
98 | |
---|
99 | \backslash |
---|
100 | maketitle |
---|
101 | \end_layout |
---|
102 | |
---|
103 | \end_inset |
---|
104 | |
---|
105 | |
---|
106 | \end_layout |
---|
107 | |
---|
108 | \begin_layout Standard |
---|
109 | |
---|
110 | \begin_inset ERT |
---|
111 | status collapsed |
---|
112 | |
---|
113 | \begin_layout Plain Layout |
---|
114 | % |
---|
115 | \backslash |
---|
116 | begin{abstract} |
---|
117 | \end_layout |
---|
118 | |
---|
119 | \begin_layout Plain Layout |
---|
120 | |
---|
121 | \end_layout |
---|
122 | |
---|
123 | \end_inset |
---|
124 | |
---|
125 | |
---|
126 | \begin_inset ERT |
---|
127 | status collapsed |
---|
128 | |
---|
129 | \begin_layout Plain Layout |
---|
130 | % |
---|
131 | \backslash |
---|
132 | end{abstract} |
---|
133 | \end_layout |
---|
134 | |
---|
135 | \end_inset |
---|
136 | |
---|
137 | |
---|
138 | \end_layout |
---|
139 | |
---|
140 | \begin_layout Section |
---|
141 | Purpose |
---|
142 | \end_layout |
---|
143 | |
---|
144 | \begin_layout Standard |
---|
145 | Use threads as if they are MPI processes. Each thread will be assigned a rank and be associated with a endpoints communicator (EP_Comm). Convention: one OpenMP thread corresponds to one endpoint. |
---|
146 | \end_layout |
---|
147 | |
---|
148 | \begin_layout Section |
---|
149 | MPI Endpoints Semantics |
---|
150 | \end_layout |
---|
151 | |
---|
152 | \begin_layout Standard |
---|
153 | \align center |
---|
154 | |
---|
155 | \begin_inset Graphics |
---|
156 | filename scheme.png |
---|
157 | scale 40 |
---|
158 | |
---|
159 | \end_inset |
---|
160 | |
---|
161 | |
---|
162 | \end_layout |
---|
163 | |
---|
164 | \begin_layout Standard |
---|
165 | Endpoints are created from one MPI communicator and the number of available threads: |
---|
166 | \end_layout |
---|
167 | |
---|
168 | \begin_layout Verbatim |
---|
169 | int MPI_Comm_create_endpoints(MPI_Comm parent_comm, int num_ep, |
---|
170 | \end_layout |
---|
171 | |
---|
172 | \begin_layout Verbatim |
---|
173 | MPI_Info info, MPI_Comm out_comm_hdls[]) |
---|
174 | \end_layout |
---|
175 | |
---|
176 | \begin_layout Verbatim |
---|
177 | |
---|
178 | \end_layout |
---|
179 | |
---|
180 | \begin_layout Standard |
---|
181 | |
---|
182 | \begin_inset Quotes eld |
---|
183 | \end_inset |
---|
184 | |
---|
185 | In this collective call, a single output communicator is created, and an array of |
---|
186 | \begin_inset ERT |
---|
187 | status collapsed |
---|
188 | |
---|
189 | \begin_layout Plain Layout |
---|
190 | |
---|
191 | \backslash |
---|
192 | verb|num_ep| |
---|
193 | \end_layout |
---|
194 | |
---|
195 | \end_inset |
---|
196 | |
---|
197 | handles to this new communicator are returned, where the |
---|
198 | \begin_inset Formula $i^{th}$ |
---|
199 | \end_inset |
---|
200 | |
---|
201 | handle corresponds to the |
---|
202 | \begin_inset Formula $i^{th}$ |
---|
203 | \end_inset |
---|
204 | |
---|
205 | rank requested by the caller of |
---|
206 | \begin_inset ERT |
---|
207 | status collapsed |
---|
208 | |
---|
209 | \begin_layout Plain Layout |
---|
210 | |
---|
211 | \backslash |
---|
212 | verb|MPI_Comm_create_endpoints| |
---|
213 | \end_layout |
---|
214 | |
---|
215 | \end_inset |
---|
216 | |
---|
217 | . Ranks in the output communicator are ordered sequentially and in the same order as the parent communicator. After it has been created, the output communicator behaves as a normal communicator, and MPI calls on each endpoint (i.e., communicator handle) behave as though they originated from a separate MPI process. In particular, collective calls must be made once per endpoint. |
---|
218 | \begin_inset Quotes erd |
---|
219 | \end_inset |
---|
220 | |
---|
221 | |
---|
222 | \begin_inset CommandInset citation |
---|
223 | LatexCommand cite |
---|
224 | after "" |
---|
225 | key "Dinan:2013" |
---|
226 | literal "true" |
---|
227 | |
---|
228 | \end_inset |
---|
229 | |
---|
230 | |
---|
231 | \end_layout |
---|
232 | |
---|
233 | \begin_layout Standard |
---|
234 | |
---|
235 | \begin_inset Quotes eld |
---|
236 | \end_inset |
---|
237 | |
---|
238 | Once created, endpoints behave as MPI processes. For example, all ranks in an endpoints communicator must participate in collective operations. A consequence of this semantic is that endpoints also have MPI process progress requirements; that operations on that endpoint are required to make progress only when an MPI operation (e.g. |
---|
239 | \begin_inset ERT |
---|
240 | status collapsed |
---|
241 | |
---|
242 | \begin_layout Plain Layout |
---|
243 | |
---|
244 | \backslash |
---|
245 | verb|MPI_Test| |
---|
246 | \end_layout |
---|
247 | |
---|
248 | \end_inset |
---|
249 | |
---|
250 | ) is performed on that endpoint. This semantic enables an MPI implementation to logically separate endpoints, treat them independently within the progress engine, and eliminate synchronization in updating their state. |
---|
251 | \begin_inset Quotes erd |
---|
252 | \end_inset |
---|
253 | |
---|
254 | |
---|
255 | \begin_inset CommandInset citation |
---|
256 | LatexCommand cite |
---|
257 | after "" |
---|
258 | key "Sridharan:2014" |
---|
259 | literal "true" |
---|
260 | |
---|
261 | \end_inset |
---|
262 | |
---|
263 | |
---|
264 | \end_layout |
---|
265 | |
---|
266 | \begin_layout Section |
---|
267 | EP types |
---|
268 | \end_layout |
---|
269 | |
---|
270 | \begin_layout Subsection* |
---|
271 | MPI_Comm |
---|
272 | \end_layout |
---|
273 | |
---|
274 | \begin_layout Standard |
---|
275 | |
---|
276 | \begin_inset ERT |
---|
277 | status collapsed |
---|
278 | |
---|
279 | \begin_layout Plain Layout |
---|
280 | |
---|
281 | \backslash |
---|
282 | verb|MPI_Comm| |
---|
283 | \end_layout |
---|
284 | |
---|
285 | \end_inset |
---|
286 | |
---|
287 | is composed by: |
---|
288 | \end_layout |
---|
289 | |
---|
290 | \begin_layout Itemize |
---|
291 | |
---|
292 | \begin_inset Argument item:1 |
---|
293 | status open |
---|
294 | |
---|
295 | \begin_layout Plain Layout |
---|
296 | |
---|
297 | \begin_inset Formula $\bullet$ |
---|
298 | \end_inset |
---|
299 | |
---|
300 | |
---|
301 | \end_layout |
---|
302 | |
---|
303 | \end_inset |
---|
304 | |
---|
305 | |
---|
306 | \begin_inset ERT |
---|
307 | status collapsed |
---|
308 | |
---|
309 | \begin_layout Plain Layout |
---|
310 | |
---|
311 | \backslash |
---|
312 | verb|bool is_ep| |
---|
313 | \end_layout |
---|
314 | |
---|
315 | \end_inset |
---|
316 | |
---|
317 | : true |
---|
318 | \begin_inset Formula $\implies$ |
---|
319 | \end_inset |
---|
320 | |
---|
321 | EP, false |
---|
322 | \begin_inset Formula $\implies$ |
---|
323 | \end_inset |
---|
324 | |
---|
325 | MPI classic; |
---|
326 | \end_layout |
---|
327 | |
---|
328 | \begin_layout Itemize |
---|
329 | |
---|
330 | \begin_inset Argument item:1 |
---|
331 | status open |
---|
332 | |
---|
333 | \begin_layout Plain Layout |
---|
334 | |
---|
335 | \begin_inset Formula $\bullet$ |
---|
336 | \end_inset |
---|
337 | |
---|
338 | |
---|
339 | \end_layout |
---|
340 | |
---|
341 | \end_inset |
---|
342 | |
---|
343 | |
---|
344 | \begin_inset ERT |
---|
345 | status collapsed |
---|
346 | |
---|
347 | \begin_layout Plain Layout |
---|
348 | |
---|
349 | \backslash |
---|
350 | verb|int mpi_comm| |
---|
351 | \end_layout |
---|
352 | |
---|
353 | \end_inset |
---|
354 | |
---|
355 | : handle to the parent MPI communicator; |
---|
356 | \end_layout |
---|
357 | |
---|
358 | \begin_layout Itemize |
---|
359 | |
---|
360 | \begin_inset Argument item:1 |
---|
361 | status open |
---|
362 | |
---|
363 | \begin_layout Plain Layout |
---|
364 | |
---|
365 | \begin_inset Formula $\bullet$ |
---|
366 | \end_inset |
---|
367 | |
---|
368 | |
---|
369 | \end_layout |
---|
370 | |
---|
371 | \end_inset |
---|
372 | |
---|
373 | |
---|
374 | \begin_inset ERT |
---|
375 | status collapsed |
---|
376 | |
---|
377 | \begin_layout Plain Layout |
---|
378 | |
---|
379 | \backslash |
---|
380 | verb|OMPbarrier *ep_barrier| |
---|
381 | \end_layout |
---|
382 | |
---|
383 | \end_inset |
---|
384 | |
---|
385 | : openMP barrier, used for in-process synchronization and is different from |
---|
386 | \begin_inset ERT |
---|
387 | status collapsed |
---|
388 | |
---|
389 | \begin_layout Plain Layout |
---|
390 | |
---|
391 | \backslash |
---|
392 | verb|omp barrier| |
---|
393 | \end_layout |
---|
394 | |
---|
395 | \end_inset |
---|
396 | |
---|
397 | ; |
---|
398 | \end_layout |
---|
399 | |
---|
400 | \begin_layout Itemize |
---|
401 | |
---|
402 | \begin_inset Argument item:1 |
---|
403 | status open |
---|
404 | |
---|
405 | \begin_layout Plain Layout |
---|
406 | |
---|
407 | \begin_inset Formula $\bullet$ |
---|
408 | \end_inset |
---|
409 | |
---|
410 | |
---|
411 | \end_layout |
---|
412 | |
---|
413 | \end_inset |
---|
414 | |
---|
415 | |
---|
416 | \begin_inset ERT |
---|
417 | status collapsed |
---|
418 | |
---|
419 | \begin_layout Plain Layout |
---|
420 | |
---|
421 | \backslash |
---|
422 | verb|int[2] size_rank_info[3]| |
---|
423 | \end_layout |
---|
424 | |
---|
425 | \end_inset |
---|
426 | |
---|
427 | : topology information of the current endpoint: |
---|
428 | \end_layout |
---|
429 | |
---|
430 | \begin_deeper |
---|
431 | \begin_layout Itemize |
---|
432 | rank of parent MPI process; |
---|
433 | \end_layout |
---|
434 | |
---|
435 | \begin_layout Itemize |
---|
436 | size of parent MPI communicator; |
---|
437 | \end_layout |
---|
438 | |
---|
439 | \begin_layout Itemize |
---|
440 | rank of endpoint, returned by |
---|
441 | \begin_inset ERT |
---|
442 | status collapsed |
---|
443 | |
---|
444 | \begin_layout Plain Layout |
---|
445 | |
---|
446 | \backslash |
---|
447 | verb|MPI_Comm_rank| |
---|
448 | \end_layout |
---|
449 | |
---|
450 | \end_inset |
---|
451 | |
---|
452 | ; |
---|
453 | \end_layout |
---|
454 | |
---|
455 | \begin_layout Itemize |
---|
456 | size of EP communicator, returned by |
---|
457 | \begin_inset ERT |
---|
458 | status collapsed |
---|
459 | |
---|
460 | \begin_layout Plain Layout |
---|
461 | |
---|
462 | \backslash |
---|
463 | verb|MPI_Comm_size| |
---|
464 | \end_layout |
---|
465 | |
---|
466 | \end_inset |
---|
467 | |
---|
468 | ; |
---|
469 | \end_layout |
---|
470 | |
---|
471 | \begin_layout Itemize |
---|
472 | in-process rank of endpoint; |
---|
473 | \end_layout |
---|
474 | |
---|
475 | \begin_layout Itemize |
---|
476 | in-process size of EP communicator, also noted as the number of endpoints in one MPI process. |
---|
477 | \end_layout |
---|
478 | |
---|
479 | \end_deeper |
---|
480 | \begin_layout Itemize |
---|
481 | |
---|
482 | \begin_inset Argument item:1 |
---|
483 | status open |
---|
484 | |
---|
485 | \begin_layout Plain Layout |
---|
486 | |
---|
487 | \begin_inset Formula $\bullet$ |
---|
488 | \end_inset |
---|
489 | |
---|
490 | |
---|
491 | \end_layout |
---|
492 | |
---|
493 | \end_inset |
---|
494 | |
---|
495 | |
---|
496 | \begin_inset ERT |
---|
497 | status collapsed |
---|
498 | |
---|
499 | \begin_layout Plain Layout |
---|
500 | |
---|
501 | \backslash |
---|
502 | verb|MPI_Comm *comm_list| |
---|
503 | \end_layout |
---|
504 | |
---|
505 | \end_inset |
---|
506 | |
---|
507 | : pointer of the first endpoint communicator of one process; |
---|
508 | \end_layout |
---|
509 | |
---|
510 | \begin_layout Itemize |
---|
511 | |
---|
512 | \begin_inset Argument item:1 |
---|
513 | status open |
---|
514 | |
---|
515 | \begin_layout Plain Layout |
---|
516 | |
---|
517 | \begin_inset Formula $\bullet$ |
---|
518 | \end_inset |
---|
519 | |
---|
520 | |
---|
521 | \end_layout |
---|
522 | |
---|
523 | \end_inset |
---|
524 | |
---|
525 | |
---|
526 | \begin_inset ERT |
---|
527 | status collapsed |
---|
528 | |
---|
529 | \begin_layout Plain Layout |
---|
530 | |
---|
531 | \backslash |
---|
532 | verb|Message_list *message_queue| |
---|
533 | \end_layout |
---|
534 | |
---|
535 | \end_inset |
---|
536 | |
---|
537 | : location of in-coming messages for each endpoint; |
---|
538 | \end_layout |
---|
539 | |
---|
540 | \begin_layout Itemize |
---|
541 | |
---|
542 | \begin_inset Argument item:1 |
---|
543 | status open |
---|
544 | |
---|
545 | \begin_layout Plain Layout |
---|
546 | |
---|
547 | \begin_inset Formula $\bullet$ |
---|
548 | \end_inset |
---|
549 | |
---|
550 | |
---|
551 | \end_layout |
---|
552 | |
---|
553 | \end_inset |
---|
554 | |
---|
555 | |
---|
556 | \begin_inset ERT |
---|
557 | status collapsed |
---|
558 | |
---|
559 | \begin_layout Plain Layout |
---|
560 | |
---|
561 | \backslash |
---|
562 | verb|RANK_MAP *rank_map| |
---|
563 | \end_layout |
---|
564 | |
---|
565 | \end_inset |
---|
566 | |
---|
567 | : a map composed by an integer and a pair of integers. The integer key represents the rank of an endpoint. The mapped type (pair of integers) gives the in-process rank of the endpoint and the rank of its parent MPI process: |
---|
568 | \end_layout |
---|
569 | |
---|
570 | \begin_layout Verbatim |
---|
571 | rank_map->at(ep_rank)=(ep_rank_local, mpi_rank) |
---|
572 | \end_layout |
---|
573 | |
---|
574 | \begin_layout Verbatim |
---|
575 | |
---|
576 | \end_layout |
---|
577 | |
---|
578 | \begin_layout Itemize |
---|
579 | |
---|
580 | \begin_inset Argument item:1 |
---|
581 | status open |
---|
582 | |
---|
583 | \begin_layout Plain Layout |
---|
584 | |
---|
585 | \begin_inset Formula $\bullet$ |
---|
586 | \end_inset |
---|
587 | |
---|
588 | |
---|
589 | \end_layout |
---|
590 | |
---|
591 | \end_inset |
---|
592 | |
---|
593 | |
---|
594 | \begin_inset ERT |
---|
595 | status collapsed |
---|
596 | |
---|
597 | \begin_layout Plain Layout |
---|
598 | |
---|
599 | \backslash |
---|
600 | verb|BUFFER *ep_buffer| |
---|
601 | \end_layout |
---|
602 | |
---|
603 | \end_inset |
---|
604 | |
---|
605 | : buffer (of type |
---|
606 | \begin_inset ERT |
---|
607 | status collapsed |
---|
608 | |
---|
609 | \begin_layout Plain Layout |
---|
610 | |
---|
611 | \backslash |
---|
612 | verb|int| |
---|
613 | \end_layout |
---|
614 | |
---|
615 | \end_inset |
---|
616 | |
---|
617 | , |
---|
618 | \begin_inset ERT |
---|
619 | status collapsed |
---|
620 | |
---|
621 | \begin_layout Plain Layout |
---|
622 | |
---|
623 | \backslash |
---|
624 | verb|float| |
---|
625 | \end_layout |
---|
626 | |
---|
627 | \end_inset |
---|
628 | |
---|
629 | , |
---|
630 | \begin_inset ERT |
---|
631 | status collapsed |
---|
632 | |
---|
633 | \begin_layout Plain Layout |
---|
634 | |
---|
635 | \backslash |
---|
636 | verb|double| |
---|
637 | \end_layout |
---|
638 | |
---|
639 | \end_inset |
---|
640 | |
---|
641 | , |
---|
642 | \begin_inset ERT |
---|
643 | status collapsed |
---|
644 | |
---|
645 | \begin_layout Plain Layout |
---|
646 | |
---|
647 | \backslash |
---|
648 | verb|char| |
---|
649 | \end_layout |
---|
650 | |
---|
651 | \end_inset |
---|
652 | |
---|
653 | , |
---|
654 | \begin_inset ERT |
---|
655 | status collapsed |
---|
656 | |
---|
657 | \begin_layout Plain Layout |
---|
658 | |
---|
659 | \backslash |
---|
660 | verb|long| |
---|
661 | \end_layout |
---|
662 | |
---|
663 | \end_inset |
---|
664 | |
---|
665 | , and |
---|
666 | \begin_inset ERT |
---|
667 | status collapsed |
---|
668 | |
---|
669 | \begin_layout Plain Layout |
---|
670 | |
---|
671 | \backslash |
---|
672 | verb|unsigned long| |
---|
673 | \end_layout |
---|
674 | |
---|
675 | \end_inset |
---|
676 | |
---|
677 | ) used for in-process communication. |
---|
678 | \end_layout |
---|
679 | |
---|
680 | \begin_layout Subsection |
---|
681 | MPI_Request |
---|
682 | \end_layout |
---|
683 | |
---|
684 | \begin_layout Standard |
---|
685 | |
---|
686 | \begin_inset ERT |
---|
687 | status collapsed |
---|
688 | |
---|
689 | \begin_layout Plain Layout |
---|
690 | |
---|
691 | \backslash |
---|
692 | verb|MPI_Request| |
---|
693 | \end_layout |
---|
694 | |
---|
695 | \end_inset |
---|
696 | |
---|
697 | is composed by: |
---|
698 | \end_layout |
---|
699 | |
---|
700 | \begin_layout Itemize |
---|
701 | |
---|
702 | \begin_inset Argument item:1 |
---|
703 | status open |
---|
704 | |
---|
705 | \begin_layout Plain Layout |
---|
706 | |
---|
707 | \begin_inset Formula $\bullet$ |
---|
708 | \end_inset |
---|
709 | |
---|
710 | |
---|
711 | \end_layout |
---|
712 | |
---|
713 | \end_inset |
---|
714 | |
---|
715 | |
---|
716 | \begin_inset ERT |
---|
717 | status collapsed |
---|
718 | |
---|
719 | \begin_layout Plain Layout |
---|
720 | |
---|
721 | \backslash |
---|
722 | verb|int mpi_request| |
---|
723 | \end_layout |
---|
724 | |
---|
725 | \end_inset |
---|
726 | |
---|
727 | : handle to the MPI request; |
---|
728 | \end_layout |
---|
729 | |
---|
730 | \begin_layout Itemize |
---|
731 | |
---|
732 | \begin_inset Argument item:1 |
---|
733 | status open |
---|
734 | |
---|
735 | \begin_layout Plain Layout |
---|
736 | |
---|
737 | \begin_inset Formula $\bullet$ |
---|
738 | \end_inset |
---|
739 | |
---|
740 | |
---|
741 | \end_layout |
---|
742 | |
---|
743 | \end_inset |
---|
744 | |
---|
745 | |
---|
746 | \begin_inset ERT |
---|
747 | status collapsed |
---|
748 | |
---|
749 | \begin_layout Plain Layout |
---|
750 | |
---|
751 | \backslash |
---|
752 | verb|int ep_datatype| |
---|
753 | \end_layout |
---|
754 | |
---|
755 | \end_inset |
---|
756 | |
---|
757 | : data type of the communication; |
---|
758 | \end_layout |
---|
759 | |
---|
760 | \begin_layout Itemize |
---|
761 | |
---|
762 | \begin_inset Argument item:1 |
---|
763 | status open |
---|
764 | |
---|
765 | \begin_layout Plain Layout |
---|
766 | |
---|
767 | \begin_inset Formula $\bullet$ |
---|
768 | \end_inset |
---|
769 | |
---|
770 | |
---|
771 | \end_layout |
---|
772 | |
---|
773 | \end_inset |
---|
774 | |
---|
775 | |
---|
776 | \begin_inset ERT |
---|
777 | status collapsed |
---|
778 | |
---|
779 | \begin_layout Plain Layout |
---|
780 | |
---|
781 | \backslash |
---|
782 | verb|MPI_Comm comm| |
---|
783 | \end_layout |
---|
784 | |
---|
785 | \end_inset |
---|
786 | |
---|
787 | : handle to the EP communicator; |
---|
788 | \end_layout |
---|
789 | |
---|
790 | \begin_layout Itemize |
---|
791 | |
---|
792 | \begin_inset Argument item:1 |
---|
793 | status open |
---|
794 | |
---|
795 | \begin_layout Plain Layout |
---|
796 | |
---|
797 | \begin_inset Formula $\bullet$ |
---|
798 | \end_inset |
---|
799 | |
---|
800 | |
---|
801 | \end_layout |
---|
802 | |
---|
803 | \end_inset |
---|
804 | |
---|
805 | |
---|
806 | \begin_inset ERT |
---|
807 | status collapsed |
---|
808 | |
---|
809 | \begin_layout Plain Layout |
---|
810 | |
---|
811 | \backslash |
---|
812 | verb|int ep_src| |
---|
813 | \end_layout |
---|
814 | |
---|
815 | \end_inset |
---|
816 | |
---|
817 | : rank of the source endpoint; |
---|
818 | \end_layout |
---|
819 | |
---|
820 | \begin_layout Itemize |
---|
821 | |
---|
822 | \begin_inset Argument item:1 |
---|
823 | status open |
---|
824 | |
---|
825 | \begin_layout Plain Layout |
---|
826 | |
---|
827 | \begin_inset Formula $\bullet$ |
---|
828 | \end_inset |
---|
829 | |
---|
830 | |
---|
831 | \end_layout |
---|
832 | |
---|
833 | \end_inset |
---|
834 | |
---|
835 | |
---|
836 | \begin_inset ERT |
---|
837 | status collapsed |
---|
838 | |
---|
839 | \begin_layout Plain Layout |
---|
840 | |
---|
841 | \backslash |
---|
842 | verb|int ep_tag| |
---|
843 | \end_layout |
---|
844 | |
---|
845 | \end_inset |
---|
846 | |
---|
847 | : tag of the communication. |
---|
848 | \end_layout |
---|
849 | |
---|
850 | \begin_layout Itemize |
---|
851 | |
---|
852 | \begin_inset Argument item:1 |
---|
853 | status open |
---|
854 | |
---|
855 | \begin_layout Plain Layout |
---|
856 | |
---|
857 | \begin_inset Formula $\bullet$ |
---|
858 | \end_inset |
---|
859 | |
---|
860 | |
---|
861 | \end_layout |
---|
862 | |
---|
863 | \end_inset |
---|
864 | |
---|
865 | |
---|
866 | \begin_inset ERT |
---|
867 | status collapsed |
---|
868 | |
---|
869 | \begin_layout Plain Layout |
---|
870 | |
---|
871 | \backslash |
---|
872 | verb|int type| |
---|
873 | \end_layout |
---|
874 | |
---|
875 | \end_inset |
---|
876 | |
---|
877 | : type of the communication: |
---|
878 | \end_layout |
---|
879 | |
---|
880 | \begin_deeper |
---|
881 | \begin_layout Itemize |
---|
882 | 1 |
---|
883 | \begin_inset Formula $\implies$ |
---|
884 | \end_inset |
---|
885 | |
---|
886 | non-blocking send; |
---|
887 | \end_layout |
---|
888 | |
---|
889 | \begin_layout Itemize |
---|
890 | 2 |
---|
891 | \begin_inset Formula $\implies$ |
---|
892 | \end_inset |
---|
893 | |
---|
894 | pending non-blocking receive; |
---|
895 | \end_layout |
---|
896 | |
---|
897 | \begin_layout Itemize |
---|
898 | 3 |
---|
899 | \begin_inset Formula $\implies$ |
---|
900 | \end_inset |
---|
901 | |
---|
902 | non-blocking matching receive. |
---|
903 | \end_layout |
---|
904 | |
---|
905 | \end_deeper |
---|
906 | \begin_layout Subsection |
---|
907 | MPI_Status |
---|
908 | \end_layout |
---|
909 | |
---|
910 | \begin_layout Standard |
---|
911 | |
---|
912 | \begin_inset ERT |
---|
913 | status collapsed |
---|
914 | |
---|
915 | \begin_layout Plain Layout |
---|
916 | |
---|
917 | \backslash |
---|
918 | verb|MPI_Status| |
---|
919 | \end_layout |
---|
920 | |
---|
921 | \end_inset |
---|
922 | |
---|
923 | consists of: |
---|
924 | \end_layout |
---|
925 | |
---|
926 | \begin_layout Itemize |
---|
927 | |
---|
928 | \begin_inset Argument item:1 |
---|
929 | status open |
---|
930 | |
---|
931 | \begin_layout Plain Layout |
---|
932 | |
---|
933 | \begin_inset Formula $\bullet$ |
---|
934 | \end_inset |
---|
935 | |
---|
936 | |
---|
937 | \end_layout |
---|
938 | |
---|
939 | \end_inset |
---|
940 | |
---|
941 | |
---|
942 | \begin_inset ERT |
---|
943 | status collapsed |
---|
944 | |
---|
945 | \begin_layout Plain Layout |
---|
946 | |
---|
947 | \backslash |
---|
948 | verb|int mpi_status| |
---|
949 | \end_layout |
---|
950 | |
---|
951 | \end_inset |
---|
952 | |
---|
953 | : handle to the MPI status; |
---|
954 | \end_layout |
---|
955 | |
---|
956 | \begin_layout Itemize |
---|
957 | |
---|
958 | \begin_inset Argument item:1 |
---|
959 | status open |
---|
960 | |
---|
961 | \begin_layout Plain Layout |
---|
962 | |
---|
963 | \begin_inset Formula $\bullet$ |
---|
964 | \end_inset |
---|
965 | |
---|
966 | |
---|
967 | \end_layout |
---|
968 | |
---|
969 | \end_inset |
---|
970 | |
---|
971 | |
---|
972 | \begin_inset ERT |
---|
973 | status collapsed |
---|
974 | |
---|
975 | \begin_layout Plain Layout |
---|
976 | |
---|
977 | \backslash |
---|
978 | verb|int ep_datatype| |
---|
979 | \end_layout |
---|
980 | |
---|
981 | \end_inset |
---|
982 | |
---|
983 | : data type of the communication; |
---|
984 | \end_layout |
---|
985 | |
---|
986 | \begin_layout Itemize |
---|
987 | |
---|
988 | \begin_inset Argument item:1 |
---|
989 | status open |
---|
990 | |
---|
991 | \begin_layout Plain Layout |
---|
992 | |
---|
993 | \begin_inset Formula $\bullet$ |
---|
994 | \end_inset |
---|
995 | |
---|
996 | |
---|
997 | \end_layout |
---|
998 | |
---|
999 | \end_inset |
---|
1000 | |
---|
1001 | |
---|
1002 | \begin_inset ERT |
---|
1003 | status collapsed |
---|
1004 | |
---|
1005 | \begin_layout Plain Layout |
---|
1006 | |
---|
1007 | \backslash |
---|
1008 | verb|int ep_src| |
---|
1009 | \end_layout |
---|
1010 | |
---|
1011 | \end_inset |
---|
1012 | |
---|
1013 | : rank of the source endpoint; |
---|
1014 | \end_layout |
---|
1015 | |
---|
1016 | \begin_layout Itemize |
---|
1017 | |
---|
1018 | \begin_inset Argument item:1 |
---|
1019 | status open |
---|
1020 | |
---|
1021 | \begin_layout Plain Layout |
---|
1022 | |
---|
1023 | \begin_inset Formula $\bullet$ |
---|
1024 | \end_inset |
---|
1025 | |
---|
1026 | |
---|
1027 | \end_layout |
---|
1028 | |
---|
1029 | \end_inset |
---|
1030 | |
---|
1031 | |
---|
1032 | \begin_inset ERT |
---|
1033 | status collapsed |
---|
1034 | |
---|
1035 | \begin_layout Plain Layout |
---|
1036 | |
---|
1037 | \backslash |
---|
1038 | verb|int ep_tag| |
---|
1039 | \end_layout |
---|
1040 | |
---|
1041 | \end_inset |
---|
1042 | |
---|
1043 | : tag of the communication. |
---|
1044 | \end_layout |
---|
1045 | |
---|
1046 | \begin_layout Subsection |
---|
1047 | MPI_Message |
---|
1048 | \end_layout |
---|
1049 | |
---|
1050 | \begin_layout Standard |
---|
1051 | |
---|
1052 | \begin_inset ERT |
---|
1053 | status collapsed |
---|
1054 | |
---|
1055 | \begin_layout Plain Layout |
---|
1056 | |
---|
1057 | \backslash |
---|
1058 | verb|MPI_Message| |
---|
1059 | \end_layout |
---|
1060 | |
---|
1061 | \end_inset |
---|
1062 | |
---|
1063 | includes: |
---|
1064 | \end_layout |
---|
1065 | |
---|
1066 | \begin_layout Itemize |
---|
1067 | |
---|
1068 | \begin_inset Argument item:1 |
---|
1069 | status open |
---|
1070 | |
---|
1071 | \begin_layout Plain Layout |
---|
1072 | |
---|
1073 | \begin_inset Formula $\bullet$ |
---|
1074 | \end_inset |
---|
1075 | |
---|
1076 | |
---|
1077 | \end_layout |
---|
1078 | |
---|
1079 | \end_inset |
---|
1080 | |
---|
1081 | |
---|
1082 | \begin_inset ERT |
---|
1083 | status collapsed |
---|
1084 | |
---|
1085 | \begin_layout Plain Layout |
---|
1086 | |
---|
1087 | \backslash |
---|
1088 | verb|int mpi_message| |
---|
1089 | \end_layout |
---|
1090 | |
---|
1091 | \end_inset |
---|
1092 | |
---|
1093 | : handle to the MPI message; |
---|
1094 | \end_layout |
---|
1095 | |
---|
1096 | \begin_layout Itemize |
---|
1097 | |
---|
1098 | \begin_inset Argument item:1 |
---|
1099 | status open |
---|
1100 | |
---|
1101 | \begin_layout Plain Layout |
---|
1102 | |
---|
1103 | \begin_inset Formula $\bullet$ |
---|
1104 | \end_inset |
---|
1105 | |
---|
1106 | |
---|
1107 | \end_layout |
---|
1108 | |
---|
1109 | \end_inset |
---|
1110 | |
---|
1111 | |
---|
1112 | \begin_inset ERT |
---|
1113 | status collapsed |
---|
1114 | |
---|
1115 | \begin_layout Plain Layout |
---|
1116 | |
---|
1117 | \backslash |
---|
1118 | verb|int ep_src| |
---|
1119 | \end_layout |
---|
1120 | |
---|
1121 | \end_inset |
---|
1122 | |
---|
1123 | : rank of the source endpoint; |
---|
1124 | \end_layout |
---|
1125 | |
---|
1126 | \begin_layout Itemize |
---|
1127 | |
---|
1128 | \begin_inset Argument item:1 |
---|
1129 | status open |
---|
1130 | |
---|
1131 | \begin_layout Plain Layout |
---|
1132 | |
---|
1133 | \begin_inset Formula $\bullet$ |
---|
1134 | \end_inset |
---|
1135 | |
---|
1136 | |
---|
1137 | \end_layout |
---|
1138 | |
---|
1139 | \end_inset |
---|
1140 | |
---|
1141 | |
---|
1142 | \begin_inset ERT |
---|
1143 | status collapsed |
---|
1144 | |
---|
1145 | \begin_layout Plain Layout |
---|
1146 | |
---|
1147 | \backslash |
---|
1148 | verb|int ep_tag| |
---|
1149 | \end_layout |
---|
1150 | |
---|
1151 | \end_inset |
---|
1152 | |
---|
1153 | : tag of the communication. |
---|
1154 | \end_layout |
---|
1155 | |
---|
1156 | \begin_layout Standard |
---|
1157 | Other types, such as |
---|
1158 | \begin_inset ERT |
---|
1159 | status collapsed |
---|
1160 | |
---|
1161 | \begin_layout Plain Layout |
---|
1162 | |
---|
1163 | \backslash |
---|
1164 | verb|MPI_Info| |
---|
1165 | \end_layout |
---|
1166 | |
---|
1167 | \end_inset |
---|
1168 | |
---|
1169 | , |
---|
1170 | \begin_inset ERT |
---|
1171 | status collapsed |
---|
1172 | |
---|
1173 | \begin_layout Plain Layout |
---|
1174 | |
---|
1175 | \backslash |
---|
1176 | verb|MPI_Aint| |
---|
1177 | \end_layout |
---|
1178 | |
---|
1179 | \end_inset |
---|
1180 | |
---|
1181 | , and |
---|
1182 | \begin_inset ERT |
---|
1183 | status collapsed |
---|
1184 | |
---|
1185 | \begin_layout Plain Layout |
---|
1186 | |
---|
1187 | \backslash |
---|
1188 | verb|MPI_Fint| |
---|
1189 | \end_layout |
---|
1190 | |
---|
1191 | \end_inset |
---|
1192 | |
---|
1193 | are defined in the same way. |
---|
1194 | \end_layout |
---|
1195 | |
---|
1196 | \begin_layout Section |
---|
1197 | P2P communication |
---|
1198 | \end_layout |
---|
1199 | |
---|
1200 | \begin_layout Standard |
---|
1201 | All EP point-to-point communication use tag to distinguish the source and destination endpoint. To be able to add these extra information to tag, we require that the tag value is represented using 31 bits in the underlying MPI inmplemention. |
---|
1202 | \end_layout |
---|
1203 | |
---|
1204 | \begin_layout Standard |
---|
1205 | |
---|
1206 | \begin_inset Graphics |
---|
1207 | filename tag.png |
---|
1208 | scale 40 |
---|
1209 | |
---|
1210 | \end_inset |
---|
1211 | |
---|
1212 | |
---|
1213 | \end_layout |
---|
1214 | |
---|
1215 | \begin_layout Standard |
---|
1216 | EP_tag is user defined. MPI_tag is internally computed and used inside MPI calls. Because of the extension of tag, wild-cards as |
---|
1217 | \begin_inset ERT |
---|
1218 | status collapsed |
---|
1219 | |
---|
1220 | \begin_layout Plain Layout |
---|
1221 | |
---|
1222 | \backslash |
---|
1223 | verb|MPI_ANY_SOURCE| |
---|
1224 | \end_layout |
---|
1225 | |
---|
1226 | \end_inset |
---|
1227 | |
---|
1228 | and |
---|
1229 | \begin_inset ERT |
---|
1230 | status collapsed |
---|
1231 | |
---|
1232 | \begin_layout Plain Layout |
---|
1233 | |
---|
1234 | \backslash |
---|
1235 | verb|MPI_ANY_TAG| |
---|
1236 | \end_layout |
---|
1237 | |
---|
1238 | \end_inset |
---|
1239 | |
---|
1240 | will not be usable directly. An extra step of tag analysis is needed which leads to the message dequeuing mechanism. |
---|
1241 | \end_layout |
---|
1242 | |
---|
1243 | \begin_layout Standard |
---|
1244 | \align center |
---|
1245 | |
---|
1246 | \begin_inset Graphics |
---|
1247 | filename sendrecv.png |
---|
1248 | scale 50 |
---|
1249 | |
---|
1250 | \end_inset |
---|
1251 | |
---|
1252 | |
---|
1253 | \end_layout |
---|
1254 | |
---|
1255 | \begin_layout Standard |
---|
1256 | In MPI environment, each MPI process has an incoming message queue. In EP case, messages for all threads inside one MPI process are stored in this MPI queue. With the MPI 3 standard, we use the |
---|
1257 | \begin_inset ERT |
---|
1258 | status collapsed |
---|
1259 | |
---|
1260 | \begin_layout Plain Layout |
---|
1261 | |
---|
1262 | \backslash |
---|
1263 | verb|MPI_Improbe| |
---|
1264 | \end_layout |
---|
1265 | |
---|
1266 | \end_inset |
---|
1267 | |
---|
1268 | routine to inquire the message queue and relocate the incoming message in the local message queue for the corresponding thread/endpoint. |
---|
1269 | \end_layout |
---|
1270 | |
---|
1271 | \begin_layout Standard |
---|
1272 | |
---|
1273 | \begin_inset Graphics |
---|
1274 | filename dequeue.png |
---|
1275 | scale 30 |
---|
1276 | |
---|
1277 | \end_inset |
---|
1278 | |
---|
1279 | |
---|
1280 | \end_layout |
---|
1281 | |
---|
1282 | \begin_layout Standard |
---|
1283 | |
---|
1284 | \begin_inset ERT |
---|
1285 | status collapsed |
---|
1286 | |
---|
1287 | \begin_layout Plain Layout |
---|
1288 | % Any EP calls will trigger the message dequeuing and the probing, (matched-)receiving operations are performed upon the local message queue. |
---|
1289 | \end_layout |
---|
1290 | |
---|
1291 | \begin_layout Plain Layout |
---|
1292 | |
---|
1293 | \end_layout |
---|
1294 | |
---|
1295 | \end_inset |
---|
1296 | |
---|
1297 | |
---|
1298 | \begin_inset ERT |
---|
1299 | status collapsed |
---|
1300 | |
---|
1301 | \begin_layout Plain Layout |
---|
1302 | % |
---|
1303 | \end_layout |
---|
1304 | |
---|
1305 | \begin_layout Plain Layout |
---|
1306 | |
---|
1307 | \end_layout |
---|
1308 | |
---|
1309 | \end_inset |
---|
1310 | |
---|
1311 | |
---|
1312 | \begin_inset ERT |
---|
1313 | status collapsed |
---|
1314 | |
---|
1315 | \begin_layout Plain Layout |
---|
1316 | % Example: |
---|
1317 | \backslash |
---|
1318 | verb|EP_Recv(src=2, tag=10, comm1)|: |
---|
1319 | \end_layout |
---|
1320 | |
---|
1321 | \begin_layout Plain Layout |
---|
1322 | |
---|
1323 | \end_layout |
---|
1324 | |
---|
1325 | \end_inset |
---|
1326 | |
---|
1327 | |
---|
1328 | \begin_inset ERT |
---|
1329 | status collapsed |
---|
1330 | |
---|
1331 | \begin_layout Plain Layout |
---|
1332 | % |
---|
1333 | \backslash |
---|
1334 | begin{itemize} |
---|
1335 | \end_layout |
---|
1336 | |
---|
1337 | \begin_layout Plain Layout |
---|
1338 | |
---|
1339 | \end_layout |
---|
1340 | |
---|
1341 | \end_inset |
---|
1342 | |
---|
1343 | |
---|
1344 | \begin_inset ERT |
---|
1345 | status collapsed |
---|
1346 | |
---|
1347 | \begin_layout Plain Layout |
---|
1348 | % |
---|
1349 | \backslash |
---|
1350 | item[1.] Dequeue MPI message queue; |
---|
1351 | \end_layout |
---|
1352 | |
---|
1353 | \begin_layout Plain Layout |
---|
1354 | |
---|
1355 | \end_layout |
---|
1356 | |
---|
1357 | \end_inset |
---|
1358 | |
---|
1359 | |
---|
1360 | \begin_inset ERT |
---|
1361 | status collapsed |
---|
1362 | |
---|
1363 | \begin_layout Plain Layout |
---|
1364 | % |
---|
1365 | \backslash |
---|
1366 | item[2.] call |
---|
1367 | \backslash |
---|
1368 | verb|EP_Improb(src=2, tag=10, comm1, message)|; |
---|
1369 | \end_layout |
---|
1370 | |
---|
1371 | \begin_layout Plain Layout |
---|
1372 | |
---|
1373 | \end_layout |
---|
1374 | |
---|
1375 | \end_inset |
---|
1376 | |
---|
1377 | |
---|
1378 | \begin_inset ERT |
---|
1379 | status collapsed |
---|
1380 | |
---|
1381 | \begin_layout Plain Layout |
---|
1382 | % |
---|
1383 | \backslash |
---|
1384 | item[3.] if find corresponding triple (src, tag, comm1), call |
---|
1385 | \backslash |
---|
1386 | verb|EP_Mrecv(src=2, tag=10, comm1, message)|; |
---|
1387 | \end_layout |
---|
1388 | |
---|
1389 | \begin_layout Plain Layout |
---|
1390 | |
---|
1391 | \end_layout |
---|
1392 | |
---|
1393 | \end_inset |
---|
1394 | |
---|
1395 | |
---|
1396 | \begin_inset ERT |
---|
1397 | status collapsed |
---|
1398 | |
---|
1399 | \begin_layout Plain Layout |
---|
1400 | % |
---|
1401 | \backslash |
---|
1402 | item[4.] else, repeat from step 2. |
---|
1403 | \end_layout |
---|
1404 | |
---|
1405 | \begin_layout Plain Layout |
---|
1406 | |
---|
1407 | \end_layout |
---|
1408 | |
---|
1409 | \end_inset |
---|
1410 | |
---|
1411 | |
---|
1412 | \begin_inset ERT |
---|
1413 | status collapsed |
---|
1414 | |
---|
1415 | \begin_layout Plain Layout |
---|
1416 | % |
---|
1417 | \backslash |
---|
1418 | end{itemize} |
---|
1419 | \end_layout |
---|
1420 | |
---|
1421 | \end_inset |
---|
1422 | |
---|
1423 | |
---|
1424 | \end_layout |
---|
1425 | |
---|
1426 | \begin_layout Paragraph |
---|
1427 | Messages are |
---|
1428 | \shape italic |
---|
1429 | non-overtaking |
---|
1430 | \shape default |
---|
1431 | |
---|
1432 | \end_layout |
---|
1433 | |
---|
1434 | \begin_layout Standard |
---|
1435 | Incoming messages' order is important! If one thread is receiving multiple messages from the same source with the same tag. The receive order should be the same order in which the messages are sent. That is to say, the n-th sent message should be the n-th received message. |
---|
1436 | \end_layout |
---|
1437 | |
---|
1438 | \begin_layout Paragraph |
---|
1439 | Progress |
---|
1440 | \end_layout |
---|
1441 | |
---|
1442 | \begin_layout Standard |
---|
1443 | |
---|
1444 | \begin_inset Quotes eld |
---|
1445 | \end_inset |
---|
1446 | |
---|
1447 | If a pair of matching send and receives have been initiated on two processes, then at least one of these two operations will complete, independently of other actions in the system: the send operation will complete, unless the receive is satisfied by another message, and completes; the receive operation will complete, unless the message sent is consumed by another matching receive that was posted at the same destination process. |
---|
1448 | \begin_inset Quotes erd |
---|
1449 | \end_inset |
---|
1450 | |
---|
1451 | |
---|
1452 | \begin_inset CommandInset citation |
---|
1453 | LatexCommand cite |
---|
1454 | after "" |
---|
1455 | key "MPI" |
---|
1456 | literal "true" |
---|
1457 | |
---|
1458 | \end_inset |
---|
1459 | |
---|
1460 | |
---|
1461 | \end_layout |
---|
1462 | |
---|
1463 | \begin_layout Standard |
---|
1464 | When one |
---|
1465 | \begin_inset ERT |
---|
1466 | status collapsed |
---|
1467 | |
---|
1468 | \begin_layout Plain Layout |
---|
1469 | |
---|
1470 | \backslash |
---|
1471 | verb|EP_Irecv| |
---|
1472 | \end_layout |
---|
1473 | |
---|
1474 | \end_inset |
---|
1475 | |
---|
1476 | is issued, we first dequeue the MPI incoming message queue and distribute all incoming messages to the local queues according to the destination identifier. Next, the nonblocking receive request is added at the end of the request pending list. Third, the pending list is checked and requests with matching source, tag, and communicator will be accomplished. |
---|
1477 | \end_layout |
---|
1478 | |
---|
1479 | \begin_layout Standard |
---|
1480 | Because of the importance of message order, some communication completion functions must be discussed here such as |
---|
1481 | \begin_inset ERT |
---|
1482 | status collapsed |
---|
1483 | |
---|
1484 | \begin_layout Plain Layout |
---|
1485 | |
---|
1486 | \backslash |
---|
1487 | verb|MPI_Test| |
---|
1488 | \end_layout |
---|
1489 | |
---|
1490 | \end_inset |
---|
1491 | |
---|
1492 | and |
---|
1493 | \begin_inset ERT |
---|
1494 | status collapsed |
---|
1495 | |
---|
1496 | \begin_layout Plain Layout |
---|
1497 | |
---|
1498 | \backslash |
---|
1499 | verb|MPI_Wait| |
---|
1500 | \end_layout |
---|
1501 | |
---|
1502 | \end_inset |
---|
1503 | |
---|
1504 | . |
---|
1505 | \begin_inset Quotes eld |
---|
1506 | \end_inset |
---|
1507 | |
---|
1508 | The functions |
---|
1509 | \begin_inset ERT |
---|
1510 | status collapsed |
---|
1511 | |
---|
1512 | \begin_layout Plain Layout |
---|
1513 | |
---|
1514 | \backslash |
---|
1515 | verb|MPI_Wait| |
---|
1516 | \end_layout |
---|
1517 | |
---|
1518 | \end_inset |
---|
1519 | |
---|
1520 | and |
---|
1521 | \begin_inset ERT |
---|
1522 | status collapsed |
---|
1523 | |
---|
1524 | \begin_layout Plain Layout |
---|
1525 | |
---|
1526 | \backslash |
---|
1527 | verb|MPI_Test| |
---|
1528 | \end_layout |
---|
1529 | |
---|
1530 | \end_inset |
---|
1531 | |
---|
1532 | are used to complete a nonblocking communication. The completion of a send operation indicates that the sender is now free to update the locations in the send buffer (the send operation itself leaves the content of the send buffer unchanged). It does not indicate that the message has been received, rather, it may have been buffered by the communication subsystem. However, if a synchronous mode send was used, the completion of the send operation indicates that a matching receive was initiated, and that the message will eventually be received by this matching receive. The completion of a receive operation indicates that the receive buffer contains the received message, the receiver is now free to access it, and that the status object is set. It does not indicate that the matching send operation has completed (but indicates, of course, that the send was initiated). |
---|
1533 | \begin_inset Quotes erd |
---|
1534 | \end_inset |
---|
1535 | |
---|
1536 | |
---|
1537 | \begin_inset CommandInset citation |
---|
1538 | LatexCommand cite |
---|
1539 | after "" |
---|
1540 | key "MPI" |
---|
1541 | literal "true" |
---|
1542 | |
---|
1543 | \end_inset |
---|
1544 | |
---|
1545 | |
---|
1546 | \end_layout |
---|
1547 | |
---|
1548 | \begin_layout Paragraph |
---|
1549 | Example 1 |
---|
1550 | \end_layout |
---|
1551 | |
---|
1552 | \begin_layout Standard |
---|
1553 | |
---|
1554 | \begin_inset ERT |
---|
1555 | status collapsed |
---|
1556 | |
---|
1557 | \begin_layout Plain Layout |
---|
1558 | |
---|
1559 | \backslash |
---|
1560 | verb|MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)| |
---|
1561 | \end_layout |
---|
1562 | |
---|
1563 | \end_inset |
---|
1564 | |
---|
1565 | |
---|
1566 | \end_layout |
---|
1567 | |
---|
1568 | \begin_layout Itemize |
---|
1569 | |
---|
1570 | \begin_inset Argument item:1 |
---|
1571 | status open |
---|
1572 | |
---|
1573 | \begin_layout Plain Layout |
---|
1574 | 1. |
---|
1575 | \end_layout |
---|
1576 | |
---|
1577 | \end_inset |
---|
1578 | |
---|
1579 | If |
---|
1580 | \begin_inset ERT |
---|
1581 | status collapsed |
---|
1582 | |
---|
1583 | \begin_layout Plain Layout |
---|
1584 | |
---|
1585 | \backslash |
---|
1586 | verb|request->type == 1| |
---|
1587 | \end_layout |
---|
1588 | |
---|
1589 | \end_inset |
---|
1590 | |
---|
1591 | , communication to be tested is indeed issued from a non-blocking send. The completion status is returned by: |
---|
1592 | \end_layout |
---|
1593 | |
---|
1594 | \begin_layout Verbatim |
---|
1595 | MPI_Test(& request->mpi_request, flag, & status->mpi_status) |
---|
1596 | \end_layout |
---|
1597 | |
---|
1598 | \begin_layout Verbatim |
---|
1599 | |
---|
1600 | \end_layout |
---|
1601 | |
---|
1602 | \begin_layout Itemize |
---|
1603 | |
---|
1604 | \begin_inset Argument item:1 |
---|
1605 | status open |
---|
1606 | |
---|
1607 | \begin_layout Plain Layout |
---|
1608 | 2. |
---|
1609 | \end_layout |
---|
1610 | |
---|
1611 | \end_inset |
---|
1612 | |
---|
1613 | If |
---|
1614 | \begin_inset ERT |
---|
1615 | status collapsed |
---|
1616 | |
---|
1617 | \begin_layout Plain Layout |
---|
1618 | |
---|
1619 | \backslash |
---|
1620 | verb|request->type == 2| |
---|
1621 | \end_layout |
---|
1622 | |
---|
1623 | \end_inset |
---|
1624 | |
---|
1625 | , it means that a non-blocking receive is called but the corresponding message is not yet probed. The request is in the pending list thus not yet completed. All incoming message is once again probed and all pending requests are checked. If after the second check, the matching message is found, thus a |
---|
1626 | \begin_inset ERT |
---|
1627 | status collapsed |
---|
1628 | |
---|
1629 | \begin_layout Plain Layout |
---|
1630 | |
---|
1631 | \backslash |
---|
1632 | verb|MPI_Imrecv| |
---|
1633 | \end_layout |
---|
1634 | |
---|
1635 | \end_inset |
---|
1636 | |
---|
1637 | is called and the type is set to 3. Otherwise, the type is still 2, then |
---|
1638 | \begin_inset ERT |
---|
1639 | status collapsed |
---|
1640 | |
---|
1641 | \begin_layout Plain Layout |
---|
1642 | |
---|
1643 | \backslash |
---|
1644 | verb|flag = false| |
---|
1645 | \end_layout |
---|
1646 | |
---|
1647 | \end_inset |
---|
1648 | |
---|
1649 | is returned. |
---|
1650 | \end_layout |
---|
1651 | |
---|
1652 | \begin_layout Itemize |
---|
1653 | |
---|
1654 | \begin_inset Argument item:1 |
---|
1655 | status open |
---|
1656 | |
---|
1657 | \begin_layout Plain Layout |
---|
1658 | 3. |
---|
1659 | \end_layout |
---|
1660 | |
---|
1661 | \end_inset |
---|
1662 | |
---|
1663 | If |
---|
1664 | \begin_inset ERT |
---|
1665 | status collapsed |
---|
1666 | |
---|
1667 | \begin_layout Plain Layout |
---|
1668 | |
---|
1669 | \backslash |
---|
1670 | verb|request->type == 3| |
---|
1671 | \end_layout |
---|
1672 | |
---|
1673 | \end_inset |
---|
1674 | |
---|
1675 | , this indcates that the request is issued from a non-blocking receive call and the matching message is probed thus the status of the communication lies in the status of the |
---|
1676 | \begin_inset ERT |
---|
1677 | status collapsed |
---|
1678 | |
---|
1679 | \begin_layout Plain Layout |
---|
1680 | |
---|
1681 | \backslash |
---|
1682 | verb|MPI_Imrecv| |
---|
1683 | \end_layout |
---|
1684 | |
---|
1685 | \end_inset |
---|
1686 | |
---|
1687 | function. The completion result is returned by: |
---|
1688 | \end_layout |
---|
1689 | |
---|
1690 | \begin_layout Verbatim |
---|
1691 | MPI_Test(& request->mpi_request, flag, & status->mpi_status) |
---|
1692 | \end_layout |
---|
1693 | |
---|
1694 | \begin_layout Verbatim |
---|
1695 | |
---|
1696 | \end_layout |
---|
1697 | |
---|
1698 | \begin_layout Paragraph |
---|
1699 | Example 2 |
---|
1700 | \end_layout |
---|
1701 | |
---|
1702 | \begin_layout Standard |
---|
1703 | |
---|
1704 | \begin_inset ERT |
---|
1705 | status collapsed |
---|
1706 | |
---|
1707 | \begin_layout Plain Layout |
---|
1708 | |
---|
1709 | \backslash |
---|
1710 | verb|MPI_Wait(MPI_Request *request, MPI_Status *status)| |
---|
1711 | \end_layout |
---|
1712 | |
---|
1713 | \end_inset |
---|
1714 | |
---|
1715 | |
---|
1716 | \end_layout |
---|
1717 | |
---|
1718 | \begin_layout Itemize |
---|
1719 | |
---|
1720 | \begin_inset Argument item:1 |
---|
1721 | status open |
---|
1722 | |
---|
1723 | \begin_layout Plain Layout |
---|
1724 | 1. |
---|
1725 | \end_layout |
---|
1726 | |
---|
1727 | \end_inset |
---|
1728 | |
---|
1729 | If |
---|
1730 | \begin_inset ERT |
---|
1731 | status collapsed |
---|
1732 | |
---|
1733 | \begin_layout Plain Layout |
---|
1734 | |
---|
1735 | \backslash |
---|
1736 | verb|request->type == 1| |
---|
1737 | \end_layout |
---|
1738 | |
---|
1739 | \end_inset |
---|
1740 | |
---|
1741 | , communication to be tested is indeed issued from a non-blocking send. Jump to step 4. |
---|
1742 | \end_layout |
---|
1743 | |
---|
1744 | \begin_layout Itemize |
---|
1745 | |
---|
1746 | \begin_inset Argument item:1 |
---|
1747 | status open |
---|
1748 | |
---|
1749 | \begin_layout Plain Layout |
---|
1750 | 2. |
---|
1751 | \end_layout |
---|
1752 | |
---|
1753 | \end_inset |
---|
1754 | |
---|
1755 | If |
---|
1756 | \begin_inset ERT |
---|
1757 | status collapsed |
---|
1758 | |
---|
1759 | \begin_layout Plain Layout |
---|
1760 | |
---|
1761 | \backslash |
---|
1762 | verb|request->type == 2| |
---|
1763 | \end_layout |
---|
1764 | |
---|
1765 | \end_inset |
---|
1766 | |
---|
1767 | , it means that a non-blocking receive is called but the corresponding message is not yet probed. The request is in the pending list thus not yet completed. We repeat the incoming message probing and the pending request checking until the matching message is found, thus a |
---|
1768 | \begin_inset ERT |
---|
1769 | status collapsed |
---|
1770 | |
---|
1771 | \begin_layout Plain Layout |
---|
1772 | |
---|
1773 | \backslash |
---|
1774 | verb|MPI_Imrecv| |
---|
1775 | \end_layout |
---|
1776 | |
---|
1777 | \end_inset |
---|
1778 | |
---|
1779 | is called and the type is set to 3. Jump to step 4. |
---|
1780 | \end_layout |
---|
1781 | |
---|
1782 | \begin_layout Itemize |
---|
1783 | |
---|
1784 | \begin_inset Argument item:1 |
---|
1785 | status open |
---|
1786 | |
---|
1787 | \begin_layout Plain Layout |
---|
1788 | 3. |
---|
1789 | \end_layout |
---|
1790 | |
---|
1791 | \end_inset |
---|
1792 | |
---|
1793 | If |
---|
1794 | \begin_inset ERT |
---|
1795 | status collapsed |
---|
1796 | |
---|
1797 | \begin_layout Plain Layout |
---|
1798 | |
---|
1799 | \backslash |
---|
1800 | verb|request->type == 3| |
---|
1801 | \end_layout |
---|
1802 | |
---|
1803 | \end_inset |
---|
1804 | |
---|
1805 | , this indcates that the request is issued from a non-blocking receive call and the matching message is probed thus the status of the communication lies in the status of the |
---|
1806 | \begin_inset ERT |
---|
1807 | status collapsed |
---|
1808 | |
---|
1809 | \begin_layout Plain Layout |
---|
1810 | |
---|
1811 | \backslash |
---|
1812 | verb|MPI_Imrecv| |
---|
1813 | \end_layout |
---|
1814 | |
---|
1815 | \end_inset |
---|
1816 | |
---|
1817 | function. Jump to step 4. |
---|
1818 | \end_layout |
---|
1819 | |
---|
1820 | \begin_layout Itemize |
---|
1821 | |
---|
1822 | \begin_inset Argument item:1 |
---|
1823 | status open |
---|
1824 | |
---|
1825 | \begin_layout Plain Layout |
---|
1826 | 4. |
---|
1827 | \end_layout |
---|
1828 | |
---|
1829 | \end_inset |
---|
1830 | |
---|
1831 | We force the completion by calling: |
---|
1832 | \end_layout |
---|
1833 | |
---|
1834 | \begin_layout Verbatim |
---|
1835 | MPI_Wat(& request->mpi_request, & status->mpi_status) |
---|
1836 | \end_layout |
---|
1837 | |
---|
1838 | \begin_layout Verbatim |
---|
1839 | |
---|
1840 | \end_layout |
---|
1841 | |
---|
1842 | \begin_layout Section |
---|
1843 | Collective communication |
---|
1844 | \end_layout |
---|
1845 | |
---|
1846 | \begin_layout Standard |
---|
1847 | All MPI classic collective communications are performed as the following pattern: |
---|
1848 | \end_layout |
---|
1849 | |
---|
1850 | \begin_layout Itemize |
---|
1851 | |
---|
1852 | \begin_inset Argument item:1 |
---|
1853 | status open |
---|
1854 | |
---|
1855 | \begin_layout Plain Layout |
---|
1856 | 1. |
---|
1857 | \end_layout |
---|
1858 | |
---|
1859 | \end_inset |
---|
1860 | |
---|
1861 | Intra-process communication using OpenMP. |
---|
1862 | \shape italic |
---|
1863 | e.g. |
---|
1864 | \shape default |
---|
1865 | Collect data from slave threads to master thread. |
---|
1866 | \end_layout |
---|
1867 | |
---|
1868 | \begin_layout Itemize |
---|
1869 | |
---|
1870 | \begin_inset Argument item:1 |
---|
1871 | status open |
---|
1872 | |
---|
1873 | \begin_layout Plain Layout |
---|
1874 | 2. |
---|
1875 | \end_layout |
---|
1876 | |
---|
1877 | \end_inset |
---|
1878 | |
---|
1879 | Inter-process communication using MPI collective calls on master threads. |
---|
1880 | \end_layout |
---|
1881 | |
---|
1882 | \begin_layout Itemize |
---|
1883 | |
---|
1884 | \begin_inset Argument item:1 |
---|
1885 | status open |
---|
1886 | |
---|
1887 | \begin_layout Plain Layout |
---|
1888 | 3. |
---|
1889 | \end_layout |
---|
1890 | |
---|
1891 | \end_inset |
---|
1892 | |
---|
1893 | Intra-process communication using OpenMP. |
---|
1894 | \shape italic |
---|
1895 | e.g. |
---|
1896 | \shape default |
---|
1897 | Distribute data from master thread to slave threads. |
---|
1898 | \end_layout |
---|
1899 | |
---|
1900 | \begin_layout Paragraph |
---|
1901 | Example 1 |
---|
1902 | \end_layout |
---|
1903 | |
---|
1904 | \begin_layout Standard |
---|
1905 | |
---|
1906 | \begin_inset ERT |
---|
1907 | status collapsed |
---|
1908 | |
---|
1909 | \begin_layout Plain Layout |
---|
1910 | |
---|
1911 | \backslash |
---|
1912 | verb|EP_Bcast(buffer, count, datatype, root = 4, comm)| |
---|
1913 | \end_layout |
---|
1914 | |
---|
1915 | \end_inset |
---|
1916 | |
---|
1917 | with |
---|
1918 | \begin_inset ERT |
---|
1919 | status collapsed |
---|
1920 | |
---|
1921 | \begin_layout Plain Layout |
---|
1922 | |
---|
1923 | \backslash |
---|
1924 | verb|comm| |
---|
1925 | \end_layout |
---|
1926 | |
---|
1927 | \end_inset |
---|
1928 | |
---|
1929 | composed by 4 MPI processes and 3 threads per process: |
---|
1930 | \end_layout |
---|
1931 | |
---|
1932 | \begin_layout Standard |
---|
1933 | We can consider the communicator as |
---|
1934 | \begin_inset Formula $\{\underbrace{(0,1,2)}_\textrm{proc 0} \quad \underbrace{(3,\textcolor{red}{4},5)}_\textrm{proc |
---|
1935 | 1}\quad \underbrace{(6,7,8)}_\textrm{proc 2}\quad \underbrace{(9,10,11)}_\textrm{proc 3}\}$ |
---|
1936 | \end_inset |
---|
1937 | |
---|
1938 | . |
---|
1939 | \end_layout |
---|
1940 | |
---|
1941 | \begin_layout Standard |
---|
1942 | This collective communication is performed by the following three steps: |
---|
1943 | \end_layout |
---|
1944 | |
---|
1945 | \begin_layout Itemize |
---|
1946 | |
---|
1947 | \begin_inset Argument item:1 |
---|
1948 | status open |
---|
1949 | |
---|
1950 | \begin_layout Plain Layout |
---|
1951 | 1. |
---|
1952 | \end_layout |
---|
1953 | |
---|
1954 | \end_inset |
---|
1955 | |
---|
1956 | EP process with rank 4 send the buffer to EP process rank 3 which is a master thread. |
---|
1957 | \end_layout |
---|
1958 | |
---|
1959 | \begin_layout Itemize |
---|
1960 | |
---|
1961 | \begin_inset Argument item:1 |
---|
1962 | status open |
---|
1963 | |
---|
1964 | \begin_layout Plain Layout |
---|
1965 | 2. |
---|
1966 | \end_layout |
---|
1967 | |
---|
1968 | \end_inset |
---|
1969 | |
---|
1970 | We call |
---|
1971 | \begin_inset ERT |
---|
1972 | status collapsed |
---|
1973 | |
---|
1974 | \begin_layout Plain Layout |
---|
1975 | |
---|
1976 | \backslash |
---|
1977 | verb|MPI_Bcast(buffer, count, datatype, mpi_root = 1, mpi_comm) | |
---|
1978 | \end_layout |
---|
1979 | |
---|
1980 | \end_inset |
---|
1981 | |
---|
1982 | . |
---|
1983 | \end_layout |
---|
1984 | |
---|
1985 | \begin_layout Itemize |
---|
1986 | |
---|
1987 | \begin_inset Argument item:1 |
---|
1988 | status open |
---|
1989 | |
---|
1990 | \begin_layout Plain Layout |
---|
1991 | 3. |
---|
1992 | \end_layout |
---|
1993 | |
---|
1994 | \end_inset |
---|
1995 | |
---|
1996 | All master threads send the buffer to its slaves. |
---|
1997 | \end_layout |
---|
1998 | |
---|
1999 | \begin_layout Standard |
---|
2000 | \align center |
---|
2001 | |
---|
2002 | \begin_inset Graphics |
---|
2003 | filename bcast.png |
---|
2004 | scale 30 |
---|
2005 | |
---|
2006 | \end_inset |
---|
2007 | |
---|
2008 | |
---|
2009 | \end_layout |
---|
2010 | |
---|
2011 | \begin_layout Paragraph |
---|
2012 | Example 2 |
---|
2013 | \end_layout |
---|
2014 | |
---|
2015 | \begin_layout Standard |
---|
2016 | |
---|
2017 | \begin_inset ERT |
---|
2018 | status collapsed |
---|
2019 | |
---|
2020 | \begin_layout Plain Layout |
---|
2021 | |
---|
2022 | \backslash |
---|
2023 | verb|EP_Allreduce(sendbuf, recvbuf, count, datatype, op, comm)| |
---|
2024 | \end_layout |
---|
2025 | |
---|
2026 | \end_inset |
---|
2027 | |
---|
2028 | with |
---|
2029 | \begin_inset ERT |
---|
2030 | status collapsed |
---|
2031 | |
---|
2032 | \begin_layout Plain Layout |
---|
2033 | |
---|
2034 | \backslash |
---|
2035 | verb|comm| |
---|
2036 | \end_layout |
---|
2037 | |
---|
2038 | \end_inset |
---|
2039 | |
---|
2040 | the same as in example 1. |
---|
2041 | \end_layout |
---|
2042 | |
---|
2043 | \begin_layout Standard |
---|
2044 | This collective communication is performed by the following three steps: |
---|
2045 | \end_layout |
---|
2046 | |
---|
2047 | \begin_layout Itemize |
---|
2048 | |
---|
2049 | \begin_inset Argument item:1 |
---|
2050 | status open |
---|
2051 | |
---|
2052 | \begin_layout Plain Layout |
---|
2053 | 1. |
---|
2054 | \end_layout |
---|
2055 | |
---|
2056 | \end_inset |
---|
2057 | |
---|
2058 | We perform a intra-process |
---|
2059 | \begin_inset Quotes eld |
---|
2060 | \end_inset |
---|
2061 | |
---|
2062 | allreduce |
---|
2063 | \begin_inset Quotes erd |
---|
2064 | \end_inset |
---|
2065 | |
---|
2066 | operation: master threads collect data from its slaves and perform the reduce operation. |
---|
2067 | \end_layout |
---|
2068 | |
---|
2069 | \begin_layout Itemize |
---|
2070 | |
---|
2071 | \begin_inset Argument item:1 |
---|
2072 | status open |
---|
2073 | |
---|
2074 | \begin_layout Plain Layout |
---|
2075 | 2. |
---|
2076 | \end_layout |
---|
2077 | |
---|
2078 | \end_inset |
---|
2079 | |
---|
2080 | Master threads call the classic |
---|
2081 | \begin_inset ERT |
---|
2082 | status collapsed |
---|
2083 | |
---|
2084 | \begin_layout Plain Layout |
---|
2085 | |
---|
2086 | \backslash |
---|
2087 | verb|MPI_Allreduce| |
---|
2088 | \end_layout |
---|
2089 | |
---|
2090 | \end_inset |
---|
2091 | |
---|
2092 | routine. |
---|
2093 | \end_layout |
---|
2094 | |
---|
2095 | \begin_layout Itemize |
---|
2096 | |
---|
2097 | \begin_inset Argument item:1 |
---|
2098 | status open |
---|
2099 | |
---|
2100 | \begin_layout Plain Layout |
---|
2101 | 3. |
---|
2102 | \end_layout |
---|
2103 | |
---|
2104 | \end_inset |
---|
2105 | |
---|
2106 | All master threads send the updated reduced data to its slaves. |
---|
2107 | \end_layout |
---|
2108 | |
---|
2109 | \begin_layout Standard |
---|
2110 | \align center |
---|
2111 | |
---|
2112 | \begin_inset Graphics |
---|
2113 | filename allreduce.png |
---|
2114 | scale 30 |
---|
2115 | |
---|
2116 | \end_inset |
---|
2117 | |
---|
2118 | |
---|
2119 | \end_layout |
---|
2120 | |
---|
2121 | \begin_layout Standard |
---|
2122 | Other collective communications have the similar execution pattern. |
---|
2123 | \end_layout |
---|
2124 | |
---|
2125 | \begin_layout Section |
---|
2126 | Inter-communicator |
---|
2127 | \end_layout |
---|
2128 | |
---|
2129 | \begin_layout Standard |
---|
2130 | In XIOS, inter-communicator is an very important component. Thus, our EP library must support inter-communications. |
---|
2131 | \end_layout |
---|
2132 | |
---|
2133 | \begin_layout Subsection |
---|
2134 | The splitting of intra-communicator |
---|
2135 | \end_layout |
---|
2136 | |
---|
2137 | \begin_layout Standard |
---|
2138 | Before talking about the inter-communicator, we will start by splitting intra-communicator. The C prototype of the splitting routine is |
---|
2139 | \end_layout |
---|
2140 | |
---|
2141 | \begin_layout Verbatim |
---|
2142 | int MPI_Comm_split(MPI_Comm comm, int color, int key, |
---|
2143 | \end_layout |
---|
2144 | |
---|
2145 | \begin_layout Verbatim |
---|
2146 | MPI_Comm *newcomm) |
---|
2147 | \end_layout |
---|
2148 | |
---|
2149 | \begin_layout Verbatim |
---|
2150 | |
---|
2151 | \end_layout |
---|
2152 | |
---|
2153 | \begin_layout Standard |
---|
2154 | |
---|
2155 | \begin_inset Quotes eld |
---|
2156 | \end_inset |
---|
2157 | |
---|
2158 | This function partitions the group associated with |
---|
2159 | \begin_inset ERT |
---|
2160 | status collapsed |
---|
2161 | |
---|
2162 | \begin_layout Plain Layout |
---|
2163 | |
---|
2164 | \backslash |
---|
2165 | verb|comm| |
---|
2166 | \end_layout |
---|
2167 | |
---|
2168 | \end_inset |
---|
2169 | |
---|
2170 | into disjoint subgroups, one for each value of |
---|
2171 | \begin_inset ERT |
---|
2172 | status collapsed |
---|
2173 | |
---|
2174 | \begin_layout Plain Layout |
---|
2175 | |
---|
2176 | \backslash |
---|
2177 | verb|color| |
---|
2178 | \end_layout |
---|
2179 | |
---|
2180 | \end_inset |
---|
2181 | |
---|
2182 | . Each subgroup contains all processes of the same color. Within each subgroup, the processes are ranked in the order defined by the value of the argument |
---|
2183 | \begin_inset ERT |
---|
2184 | status collapsed |
---|
2185 | |
---|
2186 | \begin_layout Plain Layout |
---|
2187 | |
---|
2188 | \backslash |
---|
2189 | verb|key| |
---|
2190 | \end_layout |
---|
2191 | |
---|
2192 | \end_inset |
---|
2193 | |
---|
2194 | , with ties broken according to their rank in the old group. A new communicator is created for each subgroup and returned in |
---|
2195 | \begin_inset ERT |
---|
2196 | status collapsed |
---|
2197 | |
---|
2198 | \begin_layout Plain Layout |
---|
2199 | |
---|
2200 | \backslash |
---|
2201 | verb|newcomm| |
---|
2202 | \end_layout |
---|
2203 | |
---|
2204 | \end_inset |
---|
2205 | |
---|
2206 | . A process may supply the color value |
---|
2207 | \begin_inset ERT |
---|
2208 | status collapsed |
---|
2209 | |
---|
2210 | \begin_layout Plain Layout |
---|
2211 | |
---|
2212 | \backslash |
---|
2213 | verb|MPI_UNDEFINED| |
---|
2214 | \end_layout |
---|
2215 | |
---|
2216 | \end_inset |
---|
2217 | |
---|
2218 | , in which case |
---|
2219 | \begin_inset ERT |
---|
2220 | status collapsed |
---|
2221 | |
---|
2222 | \begin_layout Plain Layout |
---|
2223 | |
---|
2224 | \backslash |
---|
2225 | verb|newcomm| |
---|
2226 | \end_layout |
---|
2227 | |
---|
2228 | \end_inset |
---|
2229 | |
---|
2230 | returns |
---|
2231 | \begin_inset ERT |
---|
2232 | status collapsed |
---|
2233 | |
---|
2234 | \begin_layout Plain Layout |
---|
2235 | |
---|
2236 | \backslash |
---|
2237 | verb|MPI_COMM_NULL| |
---|
2238 | \end_layout |
---|
2239 | |
---|
2240 | \end_inset |
---|
2241 | |
---|
2242 | . This is a collective call, but each process is permitted to provide different values for color and key. |
---|
2243 | \begin_inset Quotes erd |
---|
2244 | \end_inset |
---|
2245 | |
---|
2246 | |
---|
2247 | \begin_inset CommandInset citation |
---|
2248 | LatexCommand cite |
---|
2249 | after "" |
---|
2250 | key "MPI" |
---|
2251 | literal "true" |
---|
2252 | |
---|
2253 | \end_inset |
---|
2254 | |
---|
2255 | |
---|
2256 | \end_layout |
---|
2257 | |
---|
2258 | \begin_layout Standard |
---|
2259 | By definition of the routine, in the case of EP, each thread participating the split operation will have only one color ( |
---|
2260 | \begin_inset ERT |
---|
2261 | status collapsed |
---|
2262 | |
---|
2263 | \begin_layout Plain Layout |
---|
2264 | |
---|
2265 | \backslash |
---|
2266 | verb|MPI_UNDEFINED| |
---|
2267 | \end_layout |
---|
2268 | |
---|
2269 | \end_inset |
---|
2270 | |
---|
2271 | is also considered to be one color). However, in the process's point of view, it can have multiple colors as shown in the following figure. |
---|
2272 | \end_layout |
---|
2273 | |
---|
2274 | \begin_layout Standard |
---|
2275 | \align center |
---|
2276 | |
---|
2277 | \begin_inset Graphics |
---|
2278 | filename split.png |
---|
2279 | scale 40 |
---|
2280 | |
---|
2281 | \end_inset |
---|
2282 | |
---|
2283 | |
---|
2284 | \end_layout |
---|
2285 | |
---|
2286 | \begin_layout Standard |
---|
2287 | This figure shows the result of the EP communicator splitting. Here we used the EP rank as key to assign the new rank of the thread in the resulting split intra-communicator. If the key is anything else than the EP rank, we follow the convention that the key takes effect only inside a process. This means that the threads are at first ordered by the MPI process rank and then by the value of key. |
---|
2288 | \end_layout |
---|
2289 | |
---|
2290 | \begin_layout Standard |
---|
2291 | Due to the fact that one process can have multiple colors for its threads, the splitting operation is executed by the following steps: |
---|
2292 | \end_layout |
---|
2293 | |
---|
2294 | \begin_layout Itemize |
---|
2295 | |
---|
2296 | \begin_inset Argument item:1 |
---|
2297 | status open |
---|
2298 | |
---|
2299 | \begin_layout Plain Layout |
---|
2300 | 1. |
---|
2301 | \end_layout |
---|
2302 | |
---|
2303 | \end_inset |
---|
2304 | |
---|
2305 | Master threads collect all colors from its slaves and communicate with each other to determine the total number of colors across the communicator. |
---|
2306 | \end_layout |
---|
2307 | |
---|
2308 | \begin_layout Itemize |
---|
2309 | |
---|
2310 | \begin_inset Argument item:1 |
---|
2311 | status open |
---|
2312 | |
---|
2313 | \begin_layout Plain Layout |
---|
2314 | 2. |
---|
2315 | \end_layout |
---|
2316 | |
---|
2317 | \end_inset |
---|
2318 | |
---|
2319 | For each color, the master thread check all its slave threads to obtain the number of threads having the same color. |
---|
2320 | \end_layout |
---|
2321 | |
---|
2322 | \begin_layout Itemize |
---|
2323 | |
---|
2324 | \begin_inset Argument item:1 |
---|
2325 | status open |
---|
2326 | |
---|
2327 | \begin_layout Plain Layout |
---|
2328 | 3. |
---|
2329 | \end_layout |
---|
2330 | |
---|
2331 | \end_inset |
---|
2332 | |
---|
2333 | If at least one of the slave threads holds the color, then the master thread takes this color. If not, the master thread takes color |
---|
2334 | \begin_inset ERT |
---|
2335 | status collapsed |
---|
2336 | |
---|
2337 | \begin_layout Plain Layout |
---|
2338 | |
---|
2339 | \backslash |
---|
2340 | verb|MPI_UNDEFINED| |
---|
2341 | \end_layout |
---|
2342 | |
---|
2343 | \end_inset |
---|
2344 | |
---|
2345 | . All master threads call classic communicator splitting routine with key |
---|
2346 | \begin_inset Formula $=$ |
---|
2347 | \end_inset |
---|
2348 | |
---|
2349 | MPI rank. |
---|
2350 | \end_layout |
---|
2351 | |
---|
2352 | \begin_layout Itemize |
---|
2353 | |
---|
2354 | \begin_inset Argument item:1 |
---|
2355 | status open |
---|
2356 | |
---|
2357 | \begin_layout Plain Layout |
---|
2358 | 4. |
---|
2359 | \end_layout |
---|
2360 | |
---|
2361 | \end_inset |
---|
2362 | |
---|
2363 | For master threads holding a defined color, we execute the endpoint creation routine according to the number of slave threads holding the same color. The resulting EP communicators are then assigned to these slave threads. |
---|
2364 | \end_layout |
---|
2365 | |
---|
2366 | \begin_layout Standard |
---|
2367 | \align center |
---|
2368 | |
---|
2369 | \begin_inset Graphics |
---|
2370 | filename split2.png |
---|
2371 | scale 40 |
---|
2372 | |
---|
2373 | \end_inset |
---|
2374 | |
---|
2375 | |
---|
2376 | \end_layout |
---|
2377 | |
---|
2378 | \begin_layout Subsection |
---|
2379 | The creation of inter-communicator |
---|
2380 | \end_layout |
---|
2381 | |
---|
2382 | \begin_layout Standard |
---|
2383 | In XIOS, the inter-communicators are create by the routine |
---|
2384 | \begin_inset ERT |
---|
2385 | status collapsed |
---|
2386 | |
---|
2387 | \begin_layout Plain Layout |
---|
2388 | |
---|
2389 | \backslash |
---|
2390 | verb|MPI_Intercomm_create| |
---|
2391 | \end_layout |
---|
2392 | |
---|
2393 | \end_inset |
---|
2394 | |
---|
2395 | which is used to bind two intra-communicators into an inter-communicator. The C prototype is |
---|
2396 | \end_layout |
---|
2397 | |
---|
2398 | \begin_layout Verbatim |
---|
2399 | int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, |
---|
2400 | \end_layout |
---|
2401 | |
---|
2402 | \begin_layout Verbatim |
---|
2403 | MPI_Comm peer_comm, int remote_leader, |
---|
2404 | \end_layout |
---|
2405 | |
---|
2406 | \begin_layout Verbatim |
---|
2407 | int tag, MPI_Comm *newintercomm) |
---|
2408 | \end_layout |
---|
2409 | |
---|
2410 | \begin_layout Verbatim |
---|
2411 | |
---|
2412 | \end_layout |
---|
2413 | |
---|
2414 | \begin_layout Standard |
---|
2415 | According to the MPI standard, |
---|
2416 | \begin_inset Quotes eld |
---|
2417 | \end_inset |
---|
2418 | |
---|
2419 | an inter-communication is a point-to-point communication between processes in different groups |
---|
2420 | \begin_inset Quotes erd |
---|
2421 | \end_inset |
---|
2422 | |
---|
2423 | . |
---|
2424 | \begin_inset Quotes eld |
---|
2425 | \end_inset |
---|
2426 | |
---|
2427 | All inter-communicator constructors are blocking except for |
---|
2428 | \begin_inset ERT |
---|
2429 | status collapsed |
---|
2430 | |
---|
2431 | \begin_layout Plain Layout |
---|
2432 | |
---|
2433 | \backslash |
---|
2434 | verb|MPI_COMM_IDUP| |
---|
2435 | \end_layout |
---|
2436 | |
---|
2437 | \end_inset |
---|
2438 | |
---|
2439 | and require that the local and remote groups be disjoint. |
---|
2440 | \begin_inset Quotes erd |
---|
2441 | \end_inset |
---|
2442 | |
---|
2443 | |
---|
2444 | \end_layout |
---|
2445 | |
---|
2446 | \begin_layout Standard |
---|
2447 | As in EP the threads are considered as processes, the non-overlapping condition can be translated to |
---|
2448 | \begin_inset Quotes eld |
---|
2449 | \end_inset |
---|
2450 | |
---|
2451 | non-overlapping |
---|
2452 | \begin_inset Quotes erd |
---|
2453 | \end_inset |
---|
2454 | |
---|
2455 | at the thread level which means that one thread can not belong to the local group and the remote group. However, the parent process of the thread can be overlapped. As the EP library is built upon an existing MPI implementation which follows the non-overlapping condition at the process level, we can have an issue in the case. |
---|
2456 | \end_layout |
---|
2457 | |
---|
2458 | \begin_layout Standard |
---|
2459 | Before digging into this issue, we shall at first look at the case where the non-overlapping condition is perfectly respected. |
---|
2460 | \end_layout |
---|
2461 | |
---|
2462 | \begin_layout Standard |
---|
2463 | \align center |
---|
2464 | |
---|
2465 | \begin_inset Graphics |
---|
2466 | filename intercomm.png |
---|
2467 | scale 30 |
---|
2468 | |
---|
2469 | \end_inset |
---|
2470 | |
---|
2471 | |
---|
2472 | \end_layout |
---|
2473 | |
---|
2474 | \begin_layout Standard |
---|
2475 | As shown in the figure, we have two intra-communicators A and B and they are totally disjoint both at the thread and process level. Each of the communicators has a local leader. We also assume that both leaders belong to a peer communicator and have rank 4 and 9 respectively. |
---|
2476 | \end_layout |
---|
2477 | |
---|
2478 | \begin_layout Standard |
---|
2479 | To create the inter-communicator, all threads from the left intra-comm call: |
---|
2480 | \end_layout |
---|
2481 | |
---|
2482 | \begin_layout Verbatim |
---|
2483 | MPI_Intercomm_create(commA, local_leader = 2, peer_comm, |
---|
2484 | \end_layout |
---|
2485 | |
---|
2486 | \begin_layout Verbatim |
---|
2487 | remote_leader = 9, tag, inter_comm) |
---|
2488 | \end_layout |
---|
2489 | |
---|
2490 | \begin_layout Standard |
---|
2491 | and for threads of the right intra-comm, they call: |
---|
2492 | \end_layout |
---|
2493 | |
---|
2494 | \begin_layout Verbatim |
---|
2495 | MPI_Intercomm_create(commB, local_leader = 3, peer_comm, |
---|
2496 | \end_layout |
---|
2497 | |
---|
2498 | \begin_layout Verbatim |
---|
2499 | remote_leader = 4, tag, inter_comm) |
---|
2500 | \end_layout |
---|
2501 | |
---|
2502 | \begin_layout Standard |
---|
2503 | To perform the inter-communicator creation, we follow the 3 steps: |
---|
2504 | \end_layout |
---|
2505 | |
---|
2506 | \begin_layout Itemize |
---|
2507 | |
---|
2508 | \begin_inset Argument item:1 |
---|
2509 | status open |
---|
2510 | |
---|
2511 | \begin_layout Plain Layout |
---|
2512 | 1. |
---|
2513 | \end_layout |
---|
2514 | |
---|
2515 | \end_inset |
---|
2516 | |
---|
2517 | Determine the leaders and ranks at the process level; |
---|
2518 | \end_layout |
---|
2519 | |
---|
2520 | \begin_layout Itemize |
---|
2521 | |
---|
2522 | \begin_inset Argument item:1 |
---|
2523 | status open |
---|
2524 | |
---|
2525 | \begin_layout Plain Layout |
---|
2526 | 2. |
---|
2527 | \end_layout |
---|
2528 | |
---|
2529 | \end_inset |
---|
2530 | |
---|
2531 | Call classic |
---|
2532 | \begin_inset ERT |
---|
2533 | status collapsed |
---|
2534 | |
---|
2535 | \begin_layout Plain Layout |
---|
2536 | |
---|
2537 | \backslash |
---|
2538 | verb|MPI_Intercomm_create| |
---|
2539 | \end_layout |
---|
2540 | |
---|
2541 | \end_inset |
---|
2542 | |
---|
2543 | ; |
---|
2544 | \end_layout |
---|
2545 | |
---|
2546 | \begin_layout Itemize |
---|
2547 | |
---|
2548 | \begin_inset Argument item:1 |
---|
2549 | status open |
---|
2550 | |
---|
2551 | \begin_layout Plain Layout |
---|
2552 | 3. |
---|
2553 | \end_layout |
---|
2554 | |
---|
2555 | \end_inset |
---|
2556 | |
---|
2557 | Create endpoints from process and assigned to threads. |
---|
2558 | \end_layout |
---|
2559 | |
---|
2560 | \begin_layout Standard |
---|
2561 | \align center |
---|
2562 | |
---|
2563 | \begin_inset Graphics |
---|
2564 | filename intercomm_step.png |
---|
2565 | scale 25 |
---|
2566 | |
---|
2567 | \end_inset |
---|
2568 | |
---|
2569 | |
---|
2570 | \end_layout |
---|
2571 | |
---|
2572 | \begin_layout Standard |
---|
2573 | If we have overlapped process in the creation of inter-communicator, we should add an |
---|
2574 | \shape italic |
---|
2575 | priority check |
---|
2576 | \shape default |
---|
2577 | to assign the process to only one intra-communicator. Several possibilities: |
---|
2578 | \end_layout |
---|
2579 | |
---|
2580 | \begin_layout Itemize |
---|
2581 | |
---|
2582 | \begin_inset Argument item:1 |
---|
2583 | status open |
---|
2584 | |
---|
2585 | \begin_layout Plain Layout |
---|
2586 | 1. |
---|
2587 | \end_layout |
---|
2588 | |
---|
2589 | \end_inset |
---|
2590 | |
---|
2591 | Process is shared and contains no local leader |
---|
2592 | \begin_inset Formula $\implies$ |
---|
2593 | \end_inset |
---|
2594 | |
---|
2595 | process belongs to group with higher rank in peer comm; |
---|
2596 | \end_layout |
---|
2597 | |
---|
2598 | \begin_layout Itemize |
---|
2599 | |
---|
2600 | \begin_inset Argument item:1 |
---|
2601 | status open |
---|
2602 | |
---|
2603 | \begin_layout Plain Layout |
---|
2604 | 2. |
---|
2605 | \end_layout |
---|
2606 | |
---|
2607 | \end_inset |
---|
2608 | |
---|
2609 | Process is shared and contains one local leader |
---|
2610 | \begin_inset Formula $\implies$ |
---|
2611 | \end_inset |
---|
2612 | |
---|
2613 | process belongs to group with the leader; |
---|
2614 | \end_layout |
---|
2615 | |
---|
2616 | \begin_layout Itemize |
---|
2617 | |
---|
2618 | \begin_inset Argument item:1 |
---|
2619 | status open |
---|
2620 | |
---|
2621 | \begin_layout Plain Layout |
---|
2622 | 3. |
---|
2623 | \end_layout |
---|
2624 | |
---|
2625 | \end_inset |
---|
2626 | |
---|
2627 | Process is shared and contains both local leaders : leader change is performed and the peer communicator is |
---|
2628 | \begin_inset ERT |
---|
2629 | status collapsed |
---|
2630 | |
---|
2631 | \begin_layout Plain Layout |
---|
2632 | |
---|
2633 | \backslash |
---|
2634 | verb|MPI_COMM_WORLD| |
---|
2635 | \end_layout |
---|
2636 | |
---|
2637 | \end_inset |
---|
2638 | |
---|
2639 | and we note |
---|
2640 | \begin_inset Quotes eld |
---|
2641 | \end_inset |
---|
2642 | |
---|
2643 | group A |
---|
2644 | \begin_inset Quotes erd |
---|
2645 | \end_inset |
---|
2646 | |
---|
2647 | the group with smaller peer rank and |
---|
2648 | \begin_inset Quotes eld |
---|
2649 | \end_inset |
---|
2650 | |
---|
2651 | group B |
---|
2652 | \begin_inset Quotes erd |
---|
2653 | \end_inset |
---|
2654 | |
---|
2655 | the group with higher peer rank. |
---|
2656 | \end_layout |
---|
2657 | |
---|
2658 | \begin_deeper |
---|
2659 | \begin_layout Itemize |
---|
2660 | |
---|
2661 | \begin_inset Argument item:1 |
---|
2662 | status open |
---|
2663 | |
---|
2664 | \begin_layout Plain Layout |
---|
2665 | 3a. |
---|
2666 | \end_layout |
---|
2667 | |
---|
2668 | \end_inset |
---|
2669 | |
---|
2670 | If group A has at least two processes, the leader of group A is changed to the master thread of the process with smallest rank except the overlapped process. The overlapped process belongs to group B. |
---|
2671 | \end_layout |
---|
2672 | |
---|
2673 | \begin_layout Itemize |
---|
2674 | |
---|
2675 | \begin_inset Argument item:1 |
---|
2676 | status open |
---|
2677 | |
---|
2678 | \begin_layout Plain Layout |
---|
2679 | 3b. |
---|
2680 | \end_layout |
---|
2681 | |
---|
2682 | \end_inset |
---|
2683 | |
---|
2684 | If group A has only one processes, and group B has at least two processes, then the leader of group B is changed to the master thread of the process with smallest rank except the overlapped process. The overlapped process belongs to group A. |
---|
2685 | \end_layout |
---|
2686 | |
---|
2687 | \begin_layout Itemize |
---|
2688 | |
---|
2689 | \begin_inset Argument item:1 |
---|
2690 | status open |
---|
2691 | |
---|
2692 | \begin_layout Plain Layout |
---|
2693 | 3c. |
---|
2694 | \end_layout |
---|
2695 | |
---|
2696 | \end_inset |
---|
2697 | |
---|
2698 | If both group A and group B have only one process, then an one-process intra-communicator is created though it will be considered (labeled) as an inter-communicator. |
---|
2699 | \end_layout |
---|
2700 | |
---|
2701 | \end_deeper |
---|
2702 | \begin_layout Standard |
---|
2703 | \align center |
---|
2704 | |
---|
2705 | \begin_inset Graphics |
---|
2706 | filename intercomm2.png |
---|
2707 | scale 25 |
---|
2708 | |
---|
2709 | \end_inset |
---|
2710 | |
---|
2711 | |
---|
2712 | \end_layout |
---|
2713 | |
---|
2714 | \begin_layout Subsection |
---|
2715 | The merge of inter-communicators |
---|
2716 | \end_layout |
---|
2717 | |
---|
2718 | \begin_layout Standard |
---|
2719 | |
---|
2720 | \begin_inset ERT |
---|
2721 | status collapsed |
---|
2722 | |
---|
2723 | \begin_layout Plain Layout |
---|
2724 | |
---|
2725 | \backslash |
---|
2726 | verb|MPI_Intercomm_Merge(MPI_Comm intercomm, int high, MPI_Comm *newintracomm)| |
---|
2727 | \end_layout |
---|
2728 | |
---|
2729 | \end_inset |
---|
2730 | |
---|
2731 | creates an intra-communicator by merging the local and remote groups of an inter-communicator. All processes should provide the same |
---|
2732 | \begin_inset ERT |
---|
2733 | status collapsed |
---|
2734 | |
---|
2735 | \begin_layout Plain Layout |
---|
2736 | |
---|
2737 | \backslash |
---|
2738 | verb|high| |
---|
2739 | \end_layout |
---|
2740 | |
---|
2741 | \end_inset |
---|
2742 | |
---|
2743 | value within each of the two groups. If processes in one group provided the value |
---|
2744 | \begin_inset ERT |
---|
2745 | status collapsed |
---|
2746 | |
---|
2747 | \begin_layout Plain Layout |
---|
2748 | |
---|
2749 | \backslash |
---|
2750 | verb|high=false| |
---|
2751 | \end_layout |
---|
2752 | |
---|
2753 | \end_inset |
---|
2754 | |
---|
2755 | and processes in the other group provided the value |
---|
2756 | \begin_inset ERT |
---|
2757 | status collapsed |
---|
2758 | |
---|
2759 | \begin_layout Plain Layout |
---|
2760 | |
---|
2761 | \backslash |
---|
2762 | verb|high=true| |
---|
2763 | \end_layout |
---|
2764 | |
---|
2765 | \end_inset |
---|
2766 | |
---|
2767 | then the union orders the âlowâ group before the âhighâ group. If all processes provided the same high argument then the order of the union is arbitrary. This call is blocking and collective within the union of the two groups. |
---|
2768 | \begin_inset CommandInset citation |
---|
2769 | LatexCommand cite |
---|
2770 | after "" |
---|
2771 | key "MPI" |
---|
2772 | literal "true" |
---|
2773 | |
---|
2774 | \end_inset |
---|
2775 | |
---|
2776 | |
---|
2777 | \end_layout |
---|
2778 | |
---|
2779 | \begin_layout Standard |
---|
2780 | This routine can be considered as the inverse of |
---|
2781 | \begin_inset ERT |
---|
2782 | status collapsed |
---|
2783 | |
---|
2784 | \begin_layout Plain Layout |
---|
2785 | |
---|
2786 | \backslash |
---|
2787 | verb|MPI_Intercomm_create| |
---|
2788 | \end_layout |
---|
2789 | |
---|
2790 | \end_inset |
---|
2791 | |
---|
2792 | . In the intercommunicator create function, all 5 cases are eventually transformed into the case where no MPI process is shared by two groups. It is from this case that the merge funtion takes place. |
---|
2793 | \end_layout |
---|
2794 | |
---|
2795 | \begin_layout Itemize |
---|
2796 | |
---|
2797 | \begin_inset Argument item:1 |
---|
2798 | status open |
---|
2799 | |
---|
2800 | \begin_layout Plain Layout |
---|
2801 | 1. |
---|
2802 | \end_layout |
---|
2803 | |
---|
2804 | \end_inset |
---|
2805 | |
---|
2806 | The classic |
---|
2807 | \begin_inset ERT |
---|
2808 | status collapsed |
---|
2809 | |
---|
2810 | \begin_layout Plain Layout |
---|
2811 | |
---|
2812 | \backslash |
---|
2813 | verb|MPI_Intercomm_merge| |
---|
2814 | \end_layout |
---|
2815 | |
---|
2816 | \end_inset |
---|
2817 | |
---|
2818 | is called and an MPI intracommunicator is created from the two disjoint groups and MPI processes are ordered by the high value of the local leader. |
---|
2819 | \end_layout |
---|
2820 | |
---|
2821 | \begin_layout Itemize |
---|
2822 | |
---|
2823 | \begin_inset Argument item:1 |
---|
2824 | status open |
---|
2825 | |
---|
2826 | \begin_layout Plain Layout |
---|
2827 | 2. |
---|
2828 | \end_layout |
---|
2829 | |
---|
2830 | \end_inset |
---|
2831 | |
---|
2832 | Endpoints are created based on the MPI intracommunicator and the new EP ranks are orderd firstly according to the high value of each thread and then to the origianl EP ranks in the intercommunicators. |
---|
2833 | \end_layout |
---|
2834 | |
---|
2835 | \begin_layout Standard |
---|
2836 | \align center |
---|
2837 | |
---|
2838 | \begin_inset Graphics |
---|
2839 | filename merge.png |
---|
2840 | scale 25 |
---|
2841 | |
---|
2842 | \end_inset |
---|
2843 | |
---|
2844 | |
---|
2845 | \end_layout |
---|
2846 | |
---|
2847 | \begin_layout Section |
---|
2848 | P2P communication on inter-communicators |
---|
2849 | \end_layout |
---|
2850 | |
---|
2851 | \begin_layout Standard |
---|
2852 | In case of the intercommunicators, the |
---|
2853 | \begin_inset ERT |
---|
2854 | status collapsed |
---|
2855 | |
---|
2856 | \begin_layout Plain Layout |
---|
2857 | |
---|
2858 | \backslash |
---|
2859 | verb|MPI_Comm| |
---|
2860 | \end_layout |
---|
2861 | |
---|
2862 | \end_inset |
---|
2863 | |
---|
2864 | class has 3 members to determine the topology along with the original |
---|
2865 | \begin_inset ERT |
---|
2866 | status collapsed |
---|
2867 | |
---|
2868 | \begin_layout Plain Layout |
---|
2869 | |
---|
2870 | \backslash |
---|
2871 | verb|rank_map| |
---|
2872 | \end_layout |
---|
2873 | |
---|
2874 | \end_inset |
---|
2875 | |
---|
2876 | : |
---|
2877 | \end_layout |
---|
2878 | |
---|
2879 | \begin_layout Itemize |
---|
2880 | |
---|
2881 | \begin_inset ERT |
---|
2882 | status collapsed |
---|
2883 | |
---|
2884 | \begin_layout Plain Layout |
---|
2885 | |
---|
2886 | \backslash |
---|
2887 | verb|RANK_MAP local_rank_map[size of commA]| |
---|
2888 | \end_layout |
---|
2889 | |
---|
2890 | \end_inset |
---|
2891 | |
---|
2892 | : composed of the EP rank in commA' or commB'; |
---|
2893 | \end_layout |
---|
2894 | |
---|
2895 | \begin_layout Itemize |
---|
2896 | |
---|
2897 | \begin_inset ERT |
---|
2898 | status collapsed |
---|
2899 | |
---|
2900 | \begin_layout Plain Layout |
---|
2901 | |
---|
2902 | \backslash |
---|
2903 | verb|RANK_MAP remote_rank_map[size of commB]| |
---|
2904 | \end_layout |
---|
2905 | |
---|
2906 | \end_inset |
---|
2907 | |
---|
2908 | : = |
---|
2909 | \begin_inset ERT |
---|
2910 | status collapsed |
---|
2911 | |
---|
2912 | \begin_layout Plain Layout |
---|
2913 | |
---|
2914 | \backslash |
---|
2915 | verb|local_rank_map| |
---|
2916 | \end_layout |
---|
2917 | |
---|
2918 | \end_inset |
---|
2919 | |
---|
2920 | of remote group; |
---|
2921 | \end_layout |
---|
2922 | |
---|
2923 | \begin_layout Itemize |
---|
2924 | |
---|
2925 | \begin_inset ERT |
---|
2926 | status collapsed |
---|
2927 | |
---|
2928 | \begin_layout Plain Layout |
---|
2929 | |
---|
2930 | \backslash |
---|
2931 | verb|RANK_MAP intercomm_rank_map[size of commB']| |
---|
2932 | \end_layout |
---|
2933 | |
---|
2934 | \end_inset |
---|
2935 | |
---|
2936 | : = |
---|
2937 | \begin_inset ERT |
---|
2938 | status collapsed |
---|
2939 | |
---|
2940 | \begin_layout Plain Layout |
---|
2941 | |
---|
2942 | \backslash |
---|
2943 | verb|rank_map| |
---|
2944 | \end_layout |
---|
2945 | |
---|
2946 | \end_inset |
---|
2947 | |
---|
2948 | of remote group'; |
---|
2949 | \end_layout |
---|
2950 | |
---|
2951 | \begin_layout Itemize |
---|
2952 | |
---|
2953 | \begin_inset ERT |
---|
2954 | status collapsed |
---|
2955 | |
---|
2956 | \begin_layout Plain Layout |
---|
2957 | |
---|
2958 | \backslash |
---|
2959 | verb|RANK_MAP rank_map| |
---|
2960 | \end_layout |
---|
2961 | |
---|
2962 | \end_inset |
---|
2963 | |
---|
2964 | : rank map of commA' or commB'. |
---|
2965 | \end_layout |
---|
2966 | |
---|
2967 | \begin_layout Standard |
---|
2968 | For example, in the following configuration: |
---|
2969 | \end_layout |
---|
2970 | |
---|
2971 | \begin_layout Standard |
---|
2972 | \align center |
---|
2973 | |
---|
2974 | \begin_inset Graphics |
---|
2975 | filename ranks.png |
---|
2976 | scale 30 |
---|
2977 | |
---|
2978 | \end_inset |
---|
2979 | |
---|
2980 | |
---|
2981 | \end_layout |
---|
2982 | |
---|
2983 | \begin_layout Standard |
---|
2984 | For all endpoints in commA, |
---|
2985 | \end_layout |
---|
2986 | |
---|
2987 | \begin_layout Verbatim |
---|
2988 | local_rank_map={(rank in commA' or commB', |
---|
2989 | \end_layout |
---|
2990 | |
---|
2991 | \begin_layout Verbatim |
---|
2992 | rank of leader in MPI_Comm_world)} |
---|
2993 | \end_layout |
---|
2994 | |
---|
2995 | \begin_layout Verbatim |
---|
2996 | ={(1,0), (0,1), (2,1), (4,1)} |
---|
2997 | \end_layout |
---|
2998 | |
---|
2999 | \begin_layout Verbatim |
---|
3000 | |
---|
3001 | \end_layout |
---|
3002 | |
---|
3003 | \begin_layout Verbatim |
---|
3004 | remote_rank_map={(remote endpoints' rank in commA' or commB', |
---|
3005 | \end_layout |
---|
3006 | |
---|
3007 | \begin_layout Verbatim |
---|
3008 | rank of remote leader in MPI_Comm_world)} |
---|
3009 | \end_layout |
---|
3010 | |
---|
3011 | \begin_layout Verbatim |
---|
3012 | ={(0,0), (1,1), (3,1), (5,1)} |
---|
3013 | \end_layout |
---|
3014 | |
---|
3015 | \begin_layout Standard |
---|
3016 | For all endpoints in commA' |
---|
3017 | \end_layout |
---|
3018 | |
---|
3019 | \begin_layout Verbatim |
---|
3020 | intercomm_rank_map={(remote endpoints local rank in commA' or commB', |
---|
3021 | \end_layout |
---|
3022 | |
---|
3023 | \begin_layout Verbatim |
---|
3024 | remote endpoints MPI rank in commA' or commB')} |
---|
3025 | \end_layout |
---|
3026 | |
---|
3027 | \begin_layout Verbatim |
---|
3028 | ={(0,0), (1,0)} |
---|
3029 | \end_layout |
---|
3030 | |
---|
3031 | \begin_layout Verbatim |
---|
3032 | rank_map={(local rank in commA', mpi rank in commA')} |
---|
3033 | \end_layout |
---|
3034 | |
---|
3035 | \begin_layout Verbatim |
---|
3036 | ={(0,0), (1,0), (0,1), (1,1), (0,2), (1,2)} |
---|
3037 | \end_layout |
---|
3038 | |
---|
3039 | \begin_layout Standard |
---|
3040 | For all endpoints in comm B, |
---|
3041 | \end_layout |
---|
3042 | |
---|
3043 | \begin_layout Verbatim |
---|
3044 | local_rank_map={(rank in commA' or commB', |
---|
3045 | \end_layout |
---|
3046 | |
---|
3047 | \begin_layout Verbatim |
---|
3048 | rank of leader in MPI_Comm_world)} |
---|
3049 | \end_layout |
---|
3050 | |
---|
3051 | \begin_layout Verbatim |
---|
3052 | ={(0,0), (1,1), (3,1), (5,1)} |
---|
3053 | \end_layout |
---|
3054 | |
---|
3055 | \begin_layout Verbatim |
---|
3056 | |
---|
3057 | \end_layout |
---|
3058 | |
---|
3059 | \begin_layout Verbatim |
---|
3060 | remote_rank_map={(remote endpoints' rank in commA' or commB', |
---|
3061 | \end_layout |
---|
3062 | |
---|
3063 | \begin_layout Verbatim |
---|
3064 | rank of remote leader in MPI_Comm_world)} |
---|
3065 | \end_layout |
---|
3066 | |
---|
3067 | \begin_layout Verbatim |
---|
3068 | ={(1,0), (0,1), (2,1), (4,1)} |
---|
3069 | \end_layout |
---|
3070 | |
---|
3071 | \begin_layout Standard |
---|
3072 | For all endpoints in commB' |
---|
3073 | \end_layout |
---|
3074 | |
---|
3075 | \begin_layout Verbatim |
---|
3076 | intercomm_rank_map={(remote endpoints local rank in commA' or commB', |
---|
3077 | \end_layout |
---|
3078 | |
---|
3079 | \begin_layout Verbatim |
---|
3080 | remote endpoints MPI rank in commA' or commB')} |
---|
3081 | \end_layout |
---|
3082 | |
---|
3083 | \begin_layout Verbatim |
---|
3084 | ={(0,0), (1,0), (0,1), (1,1), (0,2), (1,2)} |
---|
3085 | \end_layout |
---|
3086 | |
---|
3087 | \begin_layout Verbatim |
---|
3088 | rank_map={(local rank in commB', mpi rank in commB')} |
---|
3089 | \end_layout |
---|
3090 | |
---|
3091 | \begin_layout Verbatim |
---|
3092 | ={(0,0), (1,0)} |
---|
3093 | \end_layout |
---|
3094 | |
---|
3095 | \begin_layout Standard |
---|
3096 | When calling a p2p communication on an inter-communicator, we should: |
---|
3097 | \end_layout |
---|
3098 | |
---|
3099 | \begin_layout Itemize |
---|
3100 | |
---|
3101 | \begin_inset Argument item:1 |
---|
3102 | status open |
---|
3103 | |
---|
3104 | \begin_layout Plain Layout |
---|
3105 | 1. |
---|
3106 | \end_layout |
---|
3107 | |
---|
3108 | \end_inset |
---|
3109 | |
---|
3110 | Determine if the source and the destination endpoints are in a same group by checking the |
---|
3111 | \begin_inset Quotes eld |
---|
3112 | \end_inset |
---|
3113 | |
---|
3114 | labels |
---|
3115 | \begin_inset Quotes erd |
---|
3116 | \end_inset |
---|
3117 | |
---|
3118 | . |
---|
3119 | \end_layout |
---|
3120 | |
---|
3121 | \begin_deeper |
---|
3122 | \begin_layout Itemize |
---|
3123 | |
---|
3124 | \begin_inset Argument item:1 |
---|
3125 | status open |
---|
3126 | |
---|
3127 | \begin_layout Plain Layout |
---|
3128 | |
---|
3129 | \begin_inset Formula $\bullet$ |
---|
3130 | \end_inset |
---|
3131 | |
---|
3132 | |
---|
3133 | \end_layout |
---|
3134 | |
---|
3135 | \end_inset |
---|
3136 | |
---|
3137 | |
---|
3138 | \begin_inset ERT |
---|
3139 | status collapsed |
---|
3140 | |
---|
3141 | \begin_layout Plain Layout |
---|
3142 | |
---|
3143 | \backslash |
---|
3144 | verb|src_label = local_rank_map->at(src).second| |
---|
3145 | \end_layout |
---|
3146 | |
---|
3147 | \end_inset |
---|
3148 | |
---|
3149 | |
---|
3150 | \end_layout |
---|
3151 | |
---|
3152 | \begin_layout Itemize |
---|
3153 | |
---|
3154 | \begin_inset Argument item:1 |
---|
3155 | status open |
---|
3156 | |
---|
3157 | \begin_layout Plain Layout |
---|
3158 | |
---|
3159 | \begin_inset Formula $\bullet$ |
---|
3160 | \end_inset |
---|
3161 | |
---|
3162 | |
---|
3163 | \end_layout |
---|
3164 | |
---|
3165 | \end_inset |
---|
3166 | |
---|
3167 | |
---|
3168 | \begin_inset ERT |
---|
3169 | status collapsed |
---|
3170 | |
---|
3171 | \begin_layout Plain Layout |
---|
3172 | |
---|
3173 | \backslash |
---|
3174 | verb|dest_label = remote_rank_map->at(dest).second| |
---|
3175 | \end_layout |
---|
3176 | |
---|
3177 | \end_inset |
---|
3178 | |
---|
3179 | |
---|
3180 | \end_layout |
---|
3181 | |
---|
3182 | \end_deeper |
---|
3183 | \begin_layout Itemize |
---|
3184 | |
---|
3185 | \begin_inset Argument item:1 |
---|
3186 | status open |
---|
3187 | |
---|
3188 | \begin_layout Plain Layout |
---|
3189 | 2. |
---|
3190 | \end_layout |
---|
3191 | |
---|
3192 | \end_inset |
---|
3193 | |
---|
3194 | If |
---|
3195 | \begin_inset ERT |
---|
3196 | status collapsed |
---|
3197 | |
---|
3198 | \begin_layout Plain Layout |
---|
3199 | |
---|
3200 | \backslash |
---|
3201 | verb|src_label == dest_label| |
---|
3202 | \end_layout |
---|
3203 | |
---|
3204 | \end_inset |
---|
3205 | |
---|
3206 | , then the communication is in fact a intra-communication. The new source rank and destination rank, as well as the local ranks, are deduced by: |
---|
3207 | \end_layout |
---|
3208 | |
---|
3209 | \begin_layout Verbatim |
---|
3210 | src_rank = local_rank_map->at(src).first |
---|
3211 | \end_layout |
---|
3212 | |
---|
3213 | \begin_layout Verbatim |
---|
3214 | dest_rank = remote_rank_map->at(dest).first |
---|
3215 | \end_layout |
---|
3216 | |
---|
3217 | \begin_layout Verbatim |
---|
3218 | src_rank_local = rank_map->at(src_rank).first |
---|
3219 | \end_layout |
---|
3220 | |
---|
3221 | \begin_layout Verbatim |
---|
3222 | dest_rank_local = rank_map->at(dest_rank).first |
---|
3223 | \end_layout |
---|
3224 | |
---|
3225 | \begin_layout Verbatim |
---|
3226 | |
---|
3227 | \end_layout |
---|
3228 | |
---|
3229 | \begin_layout Itemize |
---|
3230 | |
---|
3231 | \begin_inset Argument item:1 |
---|
3232 | status open |
---|
3233 | |
---|
3234 | \begin_layout Plain Layout |
---|
3235 | 3. |
---|
3236 | \end_layout |
---|
3237 | |
---|
3238 | \end_inset |
---|
3239 | |
---|
3240 | If |
---|
3241 | \begin_inset ERT |
---|
3242 | status collapsed |
---|
3243 | |
---|
3244 | \begin_layout Plain Layout |
---|
3245 | |
---|
3246 | \backslash |
---|
3247 | verb|src_label != dest_label| |
---|
3248 | \end_layout |
---|
3249 | |
---|
3250 | \end_inset |
---|
3251 | |
---|
3252 | , then the inter-communication is required. The new ranks are obtained by: |
---|
3253 | \end_layout |
---|
3254 | |
---|
3255 | \begin_layout Verbatim |
---|
3256 | src_rank = local_rank_map->at(src).first |
---|
3257 | \end_layout |
---|
3258 | |
---|
3259 | \begin_layout Verbatim |
---|
3260 | dest_rank = remote_rank_map->at(dest).first |
---|
3261 | \end_layout |
---|
3262 | |
---|
3263 | \begin_layout Verbatim |
---|
3264 | src_rank_local = intercomm_rank_map->at(src_rank).first |
---|
3265 | \end_layout |
---|
3266 | |
---|
3267 | \begin_layout Verbatim |
---|
3268 | dest_rank_local = rank_map->at(dest_rank).first |
---|
3269 | \end_layout |
---|
3270 | |
---|
3271 | \begin_layout Verbatim |
---|
3272 | |
---|
3273 | \end_layout |
---|
3274 | |
---|
3275 | \begin_layout Itemize |
---|
3276 | |
---|
3277 | \begin_inset Argument item:1 |
---|
3278 | status open |
---|
3279 | |
---|
3280 | \begin_layout Plain Layout |
---|
3281 | 4. |
---|
3282 | \end_layout |
---|
3283 | |
---|
3284 | \end_inset |
---|
3285 | |
---|
3286 | Call MPI P2P function to start the communication. |
---|
3287 | \end_layout |
---|
3288 | |
---|
3289 | \begin_deeper |
---|
3290 | \begin_layout Itemize |
---|
3291 | |
---|
3292 | \begin_inset Argument item:1 |
---|
3293 | status open |
---|
3294 | |
---|
3295 | \begin_layout Plain Layout |
---|
3296 | |
---|
3297 | \begin_inset Formula $\bullet$ |
---|
3298 | \end_inset |
---|
3299 | |
---|
3300 | |
---|
3301 | \end_layout |
---|
3302 | |
---|
3303 | \end_inset |
---|
3304 | |
---|
3305 | If intra-communication, |
---|
3306 | \begin_inset ERT |
---|
3307 | status collapsed |
---|
3308 | |
---|
3309 | \begin_layout Plain Layout |
---|
3310 | |
---|
3311 | \backslash |
---|
3312 | verb|mpi_comm = commA'_mpi or commB'_mpi| |
---|
3313 | \end_layout |
---|
3314 | |
---|
3315 | \end_inset |
---|
3316 | |
---|
3317 | ; |
---|
3318 | \end_layout |
---|
3319 | |
---|
3320 | \begin_layout Itemize |
---|
3321 | |
---|
3322 | \begin_inset Argument item:1 |
---|
3323 | status open |
---|
3324 | |
---|
3325 | \begin_layout Plain Layout |
---|
3326 | |
---|
3327 | \begin_inset Formula $\bullet$ |
---|
3328 | \end_inset |
---|
3329 | |
---|
3330 | |
---|
3331 | \end_layout |
---|
3332 | |
---|
3333 | \end_inset |
---|
3334 | |
---|
3335 | If inter-communication, |
---|
3336 | \begin_inset ERT |
---|
3337 | status collapsed |
---|
3338 | |
---|
3339 | \begin_layout Plain Layout |
---|
3340 | |
---|
3341 | \backslash |
---|
3342 | verb|mpi_comm = inter_comm_mpi| |
---|
3343 | \end_layout |
---|
3344 | |
---|
3345 | \end_inset |
---|
3346 | |
---|
3347 | . |
---|
3348 | \end_layout |
---|
3349 | |
---|
3350 | \end_deeper |
---|
3351 | \begin_layout Standard |
---|
3352 | \align center |
---|
3353 | |
---|
3354 | \begin_inset Graphics |
---|
3355 | filename sendrecv2.png |
---|
3356 | scale 30 |
---|
3357 | |
---|
3358 | \end_inset |
---|
3359 | |
---|
3360 | |
---|
3361 | \end_layout |
---|
3362 | |
---|
3363 | \begin_layout Section |
---|
3364 | One-sided communications |
---|
3365 | \end_layout |
---|
3366 | |
---|
3367 | \begin_layout Standard |
---|
3368 | The one-sided communication is a type of communcation which involves only one process to specify all communication parameters, both for the sending side and the receiving side |
---|
3369 | \begin_inset CommandInset citation |
---|
3370 | LatexCommand cite |
---|
3371 | after "Chapter~11" |
---|
3372 | key "MPI" |
---|
3373 | literal "true" |
---|
3374 | |
---|
3375 | \end_inset |
---|
3376 | |
---|
3377 | . To extend this type of communication in the context of endpoints, we encounter some limitations. In the current work, the one-sided communication can only be used in the client-server mode which means that RMA(remote memory access) can occur only between a server and a client. |
---|
3378 | \end_layout |
---|
3379 | |
---|
3380 | \begin_layout Standard |
---|
3381 | The construction of RMA windows is illustrated by the following figure: |
---|
3382 | \end_layout |
---|
3383 | |
---|
3384 | \begin_layout Standard |
---|
3385 | \align center |
---|
3386 | |
---|
3387 | \begin_inset Graphics |
---|
3388 | filename RMA_schema.pdf |
---|
3389 | scale 50 |
---|
3390 | |
---|
3391 | \end_inset |
---|
3392 | |
---|
3393 | |
---|
3394 | \end_layout |
---|
3395 | |
---|
3396 | \begin_layout Itemize |
---|
3397 | we determin the max number of threads N in the endpoint environment (N=3 in the example); |
---|
3398 | \end_layout |
---|
3399 | |
---|
3400 | \begin_layout Itemize |
---|
3401 | on the server side, N windows are declared and asociated with the same memory adress; |
---|
3402 | \end_layout |
---|
3403 | |
---|
3404 | \begin_layout Itemize |
---|
3405 | we start a loop : i = 0, ..., N-1 |
---|
3406 | \end_layout |
---|
3407 | |
---|
3408 | \begin_deeper |
---|
3409 | \begin_layout Itemize |
---|
3410 | each endpoint with thread number i declares an RMA window; |
---|
3411 | \end_layout |
---|
3412 | |
---|
3413 | \begin_layout Itemize |
---|
3414 | the link between windows on the client side and the i-th window on the server side are created via |
---|
3415 | \begin_inset ERT |
---|
3416 | status collapsed |
---|
3417 | |
---|
3418 | \begin_layout Plain Layout |
---|
3419 | |
---|
3420 | \backslash |
---|
3421 | verb|MPI_Win_created| |
---|
3422 | \end_layout |
---|
3423 | |
---|
3424 | \end_inset |
---|
3425 | |
---|
3426 | ; |
---|
3427 | \end_layout |
---|
3428 | |
---|
3429 | \begin_layout Itemize |
---|
3430 | if the number of threads on a certain process is less than N, then a |
---|
3431 | \begin_inset ERT |
---|
3432 | status collapsed |
---|
3433 | |
---|
3434 | \begin_layout Plain Layout |
---|
3435 | |
---|
3436 | \backslash |
---|
3437 | verb|NULL| |
---|
3438 | \end_layout |
---|
3439 | |
---|
3440 | \end_inset |
---|
3441 | |
---|
3442 | pointer is used as memory adress. |
---|
3443 | \end_layout |
---|
3444 | |
---|
3445 | \end_deeper |
---|
3446 | \begin_layout Standard |
---|
3447 | With the RMA windows created, we can then perform some communications: |
---|
3448 | \begin_inset ERT |
---|
3449 | status collapsed |
---|
3450 | |
---|
3451 | \begin_layout Plain Layout |
---|
3452 | |
---|
3453 | \backslash |
---|
3454 | verb|MPI_Put| |
---|
3455 | \end_layout |
---|
3456 | |
---|
3457 | \end_inset |
---|
3458 | |
---|
3459 | , |
---|
3460 | \begin_inset ERT |
---|
3461 | status collapsed |
---|
3462 | |
---|
3463 | \begin_layout Plain Layout |
---|
3464 | |
---|
3465 | \backslash |
---|
3466 | verb|MPI_Get| |
---|
3467 | \end_layout |
---|
3468 | |
---|
3469 | \end_inset |
---|
3470 | |
---|
3471 | , |
---|
3472 | \begin_inset ERT |
---|
3473 | status collapsed |
---|
3474 | |
---|
3475 | \begin_layout Plain Layout |
---|
3476 | |
---|
3477 | \backslash |
---|
3478 | verb|MPI_Accumulate| |
---|
3479 | \end_layout |
---|
3480 | |
---|
3481 | \end_inset |
---|
3482 | |
---|
3483 | , |
---|
3484 | \begin_inset ERT |
---|
3485 | status collapsed |
---|
3486 | |
---|
3487 | \begin_layout Plain Layout |
---|
3488 | |
---|
3489 | \backslash |
---|
3490 | verb|MPI_Get_accumulate| |
---|
3491 | \end_layout |
---|
3492 | |
---|
3493 | \end_inset |
---|
3494 | |
---|
3495 | , |
---|
3496 | \begin_inset ERT |
---|
3497 | status collapsed |
---|
3498 | |
---|
3499 | \begin_layout Plain Layout |
---|
3500 | |
---|
3501 | \backslash |
---|
3502 | verb|MPI_Fetch_and_op| |
---|
3503 | \end_layout |
---|
3504 | |
---|
3505 | \end_inset |
---|
3506 | |
---|
3507 | , |
---|
3508 | \begin_inset ERT |
---|
3509 | status collapsed |
---|
3510 | |
---|
3511 | \begin_layout Plain Layout |
---|
3512 | |
---|
3513 | \backslash |
---|
3514 | verb|MPI_Compare_and_swap| |
---|
3515 | \end_layout |
---|
3516 | |
---|
3517 | \end_inset |
---|
3518 | |
---|
3519 | , |
---|
3520 | \shape italic |
---|
3521 | etc |
---|
3522 | \shape default |
---|
3523 | . |
---|
3524 | \end_layout |
---|
3525 | |
---|
3526 | \begin_layout Standard |
---|
3527 | The main idea of any of the mentioned communications is to identify the threads which are involved in the connection. For example, we want to perform a put operation from EP 2 to the server. We know that EP 2 is the thread 0 of process 1. Thus the 0-th window (win A) of the server side should be used. Once the sender and the receiver are identified, the |
---|
3528 | \begin_inset ERT |
---|
3529 | status collapsed |
---|
3530 | |
---|
3531 | \begin_layout Plain Layout |
---|
3532 | |
---|
3533 | \backslash |
---|
3534 | verb|MPI_Put| |
---|
3535 | \end_layout |
---|
3536 | |
---|
3537 | \end_inset |
---|
3538 | |
---|
3539 | communication can be established. |
---|
3540 | \end_layout |
---|
3541 | |
---|
3542 | \begin_layout Standard |
---|
3543 | Other RMA functions, such as |
---|
3544 | \begin_inset ERT |
---|
3545 | status collapsed |
---|
3546 | |
---|
3547 | \begin_layout Plain Layout |
---|
3548 | |
---|
3549 | \backslash |
---|
3550 | verb|MPI_Win_allocate| |
---|
3551 | \end_layout |
---|
3552 | |
---|
3553 | \end_inset |
---|
3554 | |
---|
3555 | , |
---|
3556 | \begin_inset ERT |
---|
3557 | status collapsed |
---|
3558 | |
---|
3559 | \begin_layout Plain Layout |
---|
3560 | |
---|
3561 | \backslash |
---|
3562 | verb|MPI_win_Fence| |
---|
3563 | \end_layout |
---|
3564 | |
---|
3565 | \end_inset |
---|
3566 | |
---|
3567 | , and |
---|
3568 | \begin_inset ERT |
---|
3569 | status collapsed |
---|
3570 | |
---|
3571 | \begin_layout Plain Layout |
---|
3572 | |
---|
3573 | \backslash |
---|
3574 | verb|MPI_Win_free| |
---|
3575 | \end_layout |
---|
3576 | |
---|
3577 | \end_inset |
---|
3578 | |
---|
3579 | , remain nearly the same and we will skip the detail in this document. |
---|
3580 | \end_layout |
---|
3581 | |
---|
3582 | \begin_layout Standard |
---|
3583 | |
---|
3584 | \begin_inset CommandInset bibtex |
---|
3585 | LatexCommand bibtex |
---|
3586 | bibfiles "reference" |
---|
3587 | options "plain" |
---|
3588 | |
---|
3589 | \end_inset |
---|
3590 | |
---|
3591 | |
---|
3592 | \end_layout |
---|
3593 | |
---|
3594 | \end_body |
---|
3595 | \end_document |
---|