Index: XIOS/dev/branch_openmp/Note/EP note.aux
===================================================================
--- XIOS/dev/branch_openmp/Note/EP note.aux	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/EP note.aux	(revision 1552)
@@ -21,5 +21,5 @@
 \@writefile{toc}{\contentsline {section}{\numberline {6}Inter-communicator}{8}}
 \@writefile{toc}{\contentsline {subsection}{\numberline {6.1}The splitting of intra-communicator}{8}}
-\@writefile{toc}{\contentsline {subsection}{\numberline {6.2}The creation of inter-communicator}{10}}
+\@writefile{toc}{\contentsline {subsection}{\numberline {6.2}The creation of inter-communicator}{9}}
 \citation{MPI}
 \@writefile{toc}{\contentsline {subsection}{\numberline {6.3}The merge of inter-communicators}{12}}
Index: XIOS/dev/branch_openmp/Note/EP note.log
===================================================================
--- XIOS/dev/branch_openmp/Note/EP note.log	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/EP note.log	(revision 1552)
@@ -1,3 +1,3 @@
-This is pdfTeX, Version 3.14159265-2.6-1.40.16 (TeX Live 2015/Debian) (preloaded format=pdflatex 2017.8.24)  22 JUN 2018 16:19
+This is pdfTeX, Version 3.14159265-2.6-1.40.16 (TeX Live 2015/Debian) (preloaded format=pdflatex 2017.8.24)  26 JUN 2018 15:00
 entering extended mode
  restricted \write18 enabled.
@@ -638,14 +638,14 @@
 
 
-Overfull \hbox (25.85847pt too wide) in paragraph at lines 239--240
+Overfull \hbox (44.74739pt too wide) in paragraph at lines 239--240
 []\OT1/cmr/m/n/10 We call []\OT1/cmtt/m/n/10 MPI_Bcast(buffer, count, datatype,
- mpi_root = 1, mpi_comm) \OT1/cmr/m/n/10 . 
- []
-
-<bcast.png, id=36, 743.77875pt x 552.0625pt>
+ mpi_root = 1, mpi_comm)  \OT1/cmr/m/n/10 with
+ []
+
+<bcast.png, id=36, 512.91624pt x 552.0625pt>
 File: bcast.png Graphic file (type png)
  <use bcast.png>
 Package pdftex.def Info: bcast.png used on input line 244.
-(pdftex.def)             Requested size: 223.13535pt x 165.62003pt.
+(pdftex.def)             Requested size: 153.87605pt x 165.62003pt.
 
 Overfull \hbox (16.20544pt too wide) in paragraph at lines 249--251
@@ -687,6 +687,6 @@
 Package pdftex.def Info: intercomm.png used on input line 344.
 (pdftex.def)             Requested size: 311.06451pt x 137.9163pt.
- [10 <./intercomm.png (PNG copy)>]
-<intercomm_step.png, id=56, 1309.89375pt x 683.55376pt>
+
+<intercomm_step.png, id=53, 1309.89375pt x 683.55376pt>
 File: intercomm_step.png Graphic file (type png)
 
@@ -694,5 +694,5 @@
 Package pdftex.def Info: intercomm_step.png used on input line 369.
 (pdftex.def)             Requested size: 327.47264pt x 170.88802pt.
-
+ [10 <./intercomm.png (PNG copy)>]
 Overfull \hbox (6.07089pt too wide) in paragraph at lines 372--375
 \OT1/cmr/m/n/10 should add an \OT1/cmr/m/it/10 pri-or-ity check \OT1/cmr/m/n/10
@@ -827,5 +827,5 @@
 xlive/texmf-dist/fonts/type1/public/amsfonts/cm/cmtt10.pfb></usr/share/texlive/
 texmf-dist/fonts/type1/urw/times/utmr8a.pfb>
-Output written on "EP note.pdf" (17 pages, 1091602 bytes).
+Output written on "EP note.pdf" (17 pages, 1089153 bytes).
 PDF statistics:
  133 PDF objects out of 1000 (max. 8388607)
Index: XIOS/dev/branch_openmp/Note/EP note.tex
===================================================================
--- XIOS/dev/branch_openmp/Note/EP note.tex	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/EP note.tex	(revision 1552)
@@ -236,7 +236,7 @@
 
 \begin{itemize}
- \item[1.] EP process with rank 4 send the buffer to EP process rank 3 which is a master thread. 
- \item[2.] We call \verb|MPI_Bcast(buffer, count, datatype, mpi_root = 1, mpi_comm) |.
- \item[3.] All master threads send the buffer to its slaves. 
+ %\item[1.] EP process with rank 4 send the buffer to EP process rank 3 which is a master thread. 
+ \item[1.] We call \verb|MPI_Bcast(buffer, count, datatype, mpi_root = 1, mpi_comm) | with EP processes rank 0, 4, 6, and 9.
+ \item[2.] EP processes rank 0, 4, 6, and 9 send the buffer to its slaves. 
 \end{itemize}
 
Index: XIOS/dev/branch_openmp/Note/bcast.svg
===================================================================
--- XIOS/dev/branch_openmp/Note/bcast.svg	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/bcast.svg	(revision 1552)
@@ -4,15 +4,10 @@
     version="1.1" baseProfile="full">
     <defs>
-        <marker overflow="visible" orient="auto" id="arrow2-3386">
+        <marker overflow="visible" orient="auto" id="arrow2-619">
             <path
                 d="M 0.0 0.0 L -4.8173611111111105 1.720486111111111 L -2.890416666666666 0.0 L -4.8173611111111105 -1.720486111111111 z "
                 latexdraw:arrSizeNum="3.0" fill="black"/>
         </marker>
-        <marker overflow="visible" orient="auto" id="arrow2-3391">
-            <path
-                d="M 0.0 0.0 L -4.8173611111111105 1.720486111111111 L -2.890416666666666 0.0 L -4.8173611111111105 -1.720486111111111 z "
-                latexdraw:arrSizeNum="3.0" fill="black"/>
-        </marker>
-        <marker overflow="visible" orient="auto" id="arrow2-3442">
+        <marker overflow="visible" orient="auto" id="arrow2-657">
             <path
                 d="M 0.0 0.0 L -4.8173611111111105 1.720486111111111 L -2.890416666666666 0.0 L -4.8173611111111105 -1.720486111111111 z "
@@ -21,578 +16,427 @@
     </defs>
     <g>
-        <g latexdraw:type="drawing" id="id3206">
-            <g latexdraw:type="drawing" id="id3207">
-                <g latexdraw:type="drawing" id="id3208">
-                    <g latexdraw:type="rectangle" id="id3209">
-                        <rect x="351.0" y="531.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#d4f4e0"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3210">
-                        <rect x="351.0" y="391.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#dfe1f2"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3211">
-                        <rect x="351.0" y="251.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#f1daf9"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3212">
-                        <rect x="351.0" y="111.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#fafac9"/>
-                    </g>
-                </g>
-                <g latexdraw:type="drawing" id="id3214">
-                    <g latexdraw:type="square" id="id3215">
-                        <rect x="361.0" y="121.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3216">
-                        <rect x="361.0" y="161.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3217">
-                        <rect x="361.0" y="201.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                </g>
-                <g latexdraw:type="square" id="id3219">
-                    <rect x="361.0" y="261.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3220">
-                    <rect x="361.0" y="301.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3221">
-                    <rect x="361.0" y="341.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="drawing" id="id3222">
-                    <g latexdraw:type="square" id="id3223">
-                        <rect x="361.0" y="401.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3224">
-                        <rect x="361.0" y="441.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3225">
-                        <rect x="361.0" y="481.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                </g>
-                <g latexdraw:type="drawing" id="id3227">
-                    <g latexdraw:type="square" id="id3228">
-                        <rect x="361.0" y="541.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3229">
-                        <rect x="361.0" y="581.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3230">
-                        <rect x="361.0" y="621.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                </g>
-                <g latexdraw:type="text" id="id3232"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="140.0"><![CDATA[0]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3235"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="180.0"><![CDATA[1]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3238"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="220.0"><![CDATA[2]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3241"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="280.0"><![CDATA[3]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3244"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="320.0"><![CDATA[4]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3247"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="360.0"><![CDATA[5]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3250"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="420.0"><![CDATA[6]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3253"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="460.0"><![CDATA[7]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3256"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="500.0"><![CDATA[8]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3259"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="560.0"><![CDATA[9]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3262"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="600.0"><![CDATA[10]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3265"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="320.0">
-                    <text x="320.0" y="640.0"><![CDATA[11]]></text>
-                </g>
-            </g>
-            <g latexdraw:type="drawing" id="id3269">
-                <g latexdraw:type="drawing" id="id3270">
-                    <g latexdraw:type="rectangle" id="id3271">
-                        <rect x="571.0" y="531.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#d4f4e0"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3272">
-                        <rect x="571.0" y="391.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#dfe1f2"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3273">
-                        <rect x="571.0" y="251.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#f1daf9"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3274">
-                        <rect x="571.0" y="111.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#fafac9"/>
-                    </g>
-                </g>
-                <g latexdraw:type="drawing" id="id3276">
-                    <g latexdraw:type="square" id="id3277">
-                        <rect x="581.0" y="121.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3278">
-                        <rect x="581.0" y="161.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3279">
-                        <rect x="581.0" y="201.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                </g>
-                <g latexdraw:type="square" id="id3281">
-                    <rect x="581.0" y="261.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3282">
-                    <rect x="581.0" y="301.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3283">
-                    <rect x="581.0" y="341.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="drawing" id="id3284">
-                    <g latexdraw:type="square" id="id3285">
-                        <rect x="581.0" y="401.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3286">
-                        <rect x="581.0" y="441.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3287">
-                        <rect x="581.0" y="481.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                </g>
-                <g latexdraw:type="drawing" id="id3289">
-                    <g latexdraw:type="square" id="id3290">
-                        <rect x="581.0" y="541.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3291">
-                        <rect x="581.0" y="581.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                    <g latexdraw:type="square" id="id3292">
-                        <rect x="581.0" y="621.0" width="18.0"
-                            height="18.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="white"/>
-                    </g>
-                </g>
-                <g latexdraw:type="text" id="id3294"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="140.0"><![CDATA[0]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3297"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="180.0"><![CDATA[1]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3300"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="220.0"><![CDATA[2]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3303"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="280.0"><![CDATA[3]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3306"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="320.0"><![CDATA[4]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3309"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="360.0"><![CDATA[5]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3312"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="420.0"><![CDATA[6]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3315"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="460.0"><![CDATA[7]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3318"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="500.0"><![CDATA[8]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3321"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="560.0"><![CDATA[9]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3324"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="600.0"><![CDATA[10]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3327"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="540.0">
-                    <text x="540.0" y="640.0"><![CDATA[11]]></text>
-                </g>
-            </g>
-            <g latexdraw:type="drawing" id="id3331">
-                <g latexdraw:type="drawing" id="id3332">
-                    <g latexdraw:type="rectangle" id="id3333">
-                        <rect x="801.0" y="531.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#d4f4e0"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3334">
-                        <rect x="801.0" y="391.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#dfe1f2"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3335">
-                        <rect x="801.0" y="251.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#f1daf9"/>
-                    </g>
-                    <g latexdraw:type="rectangle" id="id3336">
-                        <rect x="801.0" y="111.0" width="38.0"
-                            height="118.0" latexdraw:borderPos="outer"
-                            stroke-width="2.0" stroke="black" fill="#fafac9"/>
-                    </g>
-                </g>
-                <g latexdraw:type="square" id="id3338">
-                    <rect x="811.0" y="121.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3339">
-                    <rect x="811.0" y="161.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3340">
-                    <rect x="811.0" y="201.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3341">
-                    <rect x="811.0" y="261.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3342">
-                    <rect x="811.0" y="301.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3343">
-                    <rect x="811.0" y="341.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3344">
-                    <rect x="811.0" y="401.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3345">
-                    <rect x="811.0" y="441.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3346">
-                    <rect x="811.0" y="481.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3347">
-                    <rect x="811.0" y="541.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3348">
-                    <rect x="811.0" y="581.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="square" id="id3349">
-                    <rect x="811.0" y="621.0" width="18.0" height="18.0"
-                        latexdraw:borderPos="outer" stroke-width="2.0"
-                        stroke="black" fill="white"/>
-                </g>
-                <g latexdraw:type="text" id="id3350"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="140.0"><![CDATA[0]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3353"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="180.0"><![CDATA[1]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3356"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="220.0"><![CDATA[2]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3359"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="280.0"><![CDATA[3]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3362"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="320.0"><![CDATA[4]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3365"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="360.0"><![CDATA[5]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3368"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="420.0"><![CDATA[6]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3371"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="460.0"><![CDATA[7]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3374"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="500.0"><![CDATA[8]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3377"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="560.0"><![CDATA[9]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3380"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="600.0"><![CDATA[10]]></text>
-                </g>
-                <g latexdraw:type="text" id="id3383"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="770.0">
-                    <text x="770.0" y="640.0"><![CDATA[11]]></text>
-                </g>
-                <g latexdraw:type="line" id="id3386">
-                    <line x1="640.0" x2="760.0" y1="380.0" y2="380.0"
-                        stroke-width="6.0" stroke="black"
-                        latexdraw:rotationAngle="0.0" marker-end="url(#arrow2-3386)"/>
-                </g>
-                <g latexdraw:type="text" id="id3387"
-                    font-family="Dialog" fill="black" font-size="18" latexdraw:x="670.0">
-                    <text x="670.0" y="360.0"><![CDATA[step 2]]></text>
-                </g>
-            </g>
-            <g latexdraw:type="line" id="id3391">
-                <line x1="410.0" x2="530.0" y1="380.0" y2="380.0"
-                    stroke-width="6.0" stroke="black"
-                    latexdraw:rotationAngle="0.0" marker-end="url(#arrow2-3391)"/>
-            </g>
-            <g latexdraw:type="text" id="id3392" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="440.0">
-                <text x="440.0" y="360.0"><![CDATA[step 1]]></text>
-            </g>
-            <g latexdraw:type="drawing" id="id3395">
-                <g latexdraw:type="rectangle" id="id3396">
-                    <rect x="1011.0" y="531.0" width="38.0"
+        <g latexdraw:type="drawing" id="id528">
+            <g latexdraw:type="drawing" id="id529">
+                <g latexdraw:type="rectangle" id="id530">
+                    <rect x="581.0" y="531.0" width="38.0"
                         height="118.0" latexdraw:borderPos="outer"
                         stroke-width="2.0" stroke="black" fill="#d4f4e0"/>
                 </g>
-                <g latexdraw:type="rectangle" id="id3397">
-                    <rect x="1011.0" y="391.0" width="38.0"
+                <g latexdraw:type="rectangle" id="id531">
+                    <rect x="581.0" y="391.0" width="38.0"
                         height="118.0" latexdraw:borderPos="outer"
                         stroke-width="2.0" stroke="black" fill="#dfe1f2"/>
                 </g>
-                <g latexdraw:type="rectangle" id="id3398">
-                    <rect x="1011.0" y="251.0" width="38.0"
+                <g latexdraw:type="rectangle" id="id532">
+                    <rect x="581.0" y="251.0" width="38.0"
                         height="118.0" latexdraw:borderPos="outer"
                         stroke-width="2.0" stroke="black" fill="#f1daf9"/>
                 </g>
-                <g latexdraw:type="rectangle" id="id3399">
-                    <rect x="1011.0" y="111.0" width="38.0"
+                <g latexdraw:type="rectangle" id="id533">
+                    <rect x="581.0" y="111.0" width="38.0"
                         height="118.0" latexdraw:borderPos="outer"
                         stroke-width="2.0" stroke="black" fill="#fafac9"/>
                 </g>
             </g>
-            <g latexdraw:type="drawing" id="id3401">
-                <g latexdraw:type="square" id="id3402">
-                    <rect x="1021.0" y="261.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3403">
-                    <rect x="1021.0" y="301.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3404">
-                    <rect x="1021.0" y="341.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-            </g>
-            <g latexdraw:type="text" id="id3406" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="140.0"><![CDATA[0]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3409" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="180.0"><![CDATA[1]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3412" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="220.0"><![CDATA[2]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3415" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="280.0"><![CDATA[3]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3418" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="320.0"><![CDATA[4]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3421" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="360.0"><![CDATA[5]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3424" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="420.0"><![CDATA[6]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3427" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="460.0"><![CDATA[7]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3430" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="500.0"><![CDATA[8]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3433" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="560.0"><![CDATA[9]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3436" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="600.0"><![CDATA[10]]></text>
-            </g>
-            <g latexdraw:type="text" id="id3439" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="980.0">
-                <text x="980.0" y="640.0"><![CDATA[11]]></text>
-            </g>
-            <g latexdraw:type="line" id="id3442">
-                <line x1="850.0" x2="970.0" y1="380.0" y2="380.0"
-                    stroke-width="6.0" stroke="black"
-                    latexdraw:rotationAngle="0.0" marker-end="url(#arrow2-3442)"/>
-            </g>
-            <g latexdraw:type="text" id="id3443" font-family="Dialog"
-                fill="black" font-size="18" latexdraw:x="880.0">
-                <text x="880.0" y="360.0"><![CDATA[step 3]]></text>
-            </g>
-            <g latexdraw:type="drawing" id="id3446">
-                <g latexdraw:type="square" id="id3447">
-                    <rect x="1021.0" y="401.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3448">
-                    <rect x="1021.0" y="441.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3449">
-                    <rect x="1021.0" y="481.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-            </g>
-            <g latexdraw:type="drawing" id="id3451">
-                <g latexdraw:type="square" id="id3452">
-                    <rect x="1021.0" y="541.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3453">
-                    <rect x="1021.0" y="581.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3454">
-                    <rect x="1021.0" y="621.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-            </g>
-            <g latexdraw:type="drawing" id="id3456">
-                <g latexdraw:type="square" id="id3457">
-                    <rect x="1021.0" y="121.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3458">
-                    <rect x="1021.0" y="161.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-                <g latexdraw:type="square" id="id3459">
-                    <rect x="1021.0" y="201.0" width="18.0"
-                        height="18.0" latexdraw:borderPos="outer"
-                        stroke-width="2.0" stroke="black" fill="red"/>
-                </g>
-            </g>
+            <g latexdraw:type="drawing" id="id535">
+                <g latexdraw:type="square" id="id536">
+                    <rect x="591.0" y="121.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+                <g latexdraw:type="square" id="id537">
+                    <rect x="591.0" y="161.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+                <g latexdraw:type="square" id="id538">
+                    <rect x="591.0" y="201.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+            </g>
+            <g latexdraw:type="square" id="id540">
+                <rect x="591.0" y="261.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="white"/>
+            </g>
+            <g latexdraw:type="square" id="id541">
+                <rect x="591.0" y="301.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id542">
+                <rect x="591.0" y="341.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="white"/>
+            </g>
+            <g latexdraw:type="drawing" id="id543">
+                <g latexdraw:type="square" id="id544">
+                    <rect x="591.0" y="401.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+                <g latexdraw:type="square" id="id545">
+                    <rect x="591.0" y="441.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+                <g latexdraw:type="square" id="id546">
+                    <rect x="591.0" y="481.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+            </g>
+            <g latexdraw:type="drawing" id="id548">
+                <g latexdraw:type="square" id="id549">
+                    <rect x="591.0" y="541.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+                <g latexdraw:type="square" id="id550">
+                    <rect x="591.0" y="581.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+                <g latexdraw:type="square" id="id551">
+                    <rect x="591.0" y="621.0" width="18.0" height="18.0"
+                        latexdraw:borderPos="outer" stroke-width="2.0"
+                        stroke="black" fill="white"/>
+                </g>
+            </g>
+            <g latexdraw:type="text" id="id553" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="140.0"><![CDATA[0]]></text>
+            </g>
+            <g latexdraw:type="text" id="id555" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="180.0"><![CDATA[1]]></text>
+            </g>
+            <g latexdraw:type="text" id="id557" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="220.0"><![CDATA[2]]></text>
+            </g>
+            <g latexdraw:type="text" id="id559" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="280.0"><![CDATA[3]]></text>
+            </g>
+            <g latexdraw:type="text" id="id561" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="320.0"><![CDATA[4]]></text>
+            </g>
+            <g latexdraw:type="text" id="id563" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="360.0"><![CDATA[5]]></text>
+            </g>
+            <g latexdraw:type="text" id="id565" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="420.0"><![CDATA[6]]></text>
+            </g>
+            <g latexdraw:type="text" id="id567" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="460.0"><![CDATA[7]]></text>
+            </g>
+            <g latexdraw:type="text" id="id569" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="500.0"><![CDATA[8]]></text>
+            </g>
+            <g latexdraw:type="text" id="id571" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="560.0"><![CDATA[9]]></text>
+            </g>
+            <g latexdraw:type="text" id="id573" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="600.0"><![CDATA[10]]></text>
+            </g>
+            <g latexdraw:type="text" id="id575" font-family="Dialog"
+                fill="black" font-size="18" latexdraw:x="550.0">
+                <text x="550.0" y="640.0"><![CDATA[11]]></text>
+            </g>
+        </g>
+        <g latexdraw:type="drawing" id="id578">
+            <g latexdraw:type="rectangle" id="id579">
+                <rect x="801.0" y="531.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#d4f4e0"/>
+            </g>
+            <g latexdraw:type="rectangle" id="id580">
+                <rect x="801.0" y="391.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#dfe1f2"/>
+            </g>
+            <g latexdraw:type="rectangle" id="id581">
+                <rect x="801.0" y="251.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#f1daf9"/>
+            </g>
+            <g latexdraw:type="rectangle" id="id582">
+                <rect x="801.0" y="111.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#fafac9"/>
+            </g>
+        </g>
+        <g latexdraw:type="square" id="id584">
+            <rect x="811.0" y="121.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="red"/>
+        </g>
+        <g latexdraw:type="square" id="id585">
+            <rect x="811.0" y="161.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="square" id="id586">
+            <rect x="811.0" y="201.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="square" id="id587">
+            <rect x="811.0" y="301.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="red"/>
+        </g>
+        <g latexdraw:type="square" id="id588">
+            <rect x="811.0" y="341.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="square" id="id589">
+            <rect x="811.0" y="401.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="red"/>
+        </g>
+        <g latexdraw:type="square" id="id590">
+            <rect x="811.0" y="441.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="square" id="id591">
+            <rect x="811.0" y="481.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="square" id="id592">
+            <rect x="811.0" y="541.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="red"/>
+        </g>
+        <g latexdraw:type="square" id="id593">
+            <rect x="811.0" y="581.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="square" id="id594">
+            <rect x="811.0" y="621.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
+        </g>
+        <g latexdraw:type="text" id="id595" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="140.0"><![CDATA[0]]></text>
+        </g>
+        <g latexdraw:type="text" id="id597" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="180.0"><![CDATA[1]]></text>
+        </g>
+        <g latexdraw:type="text" id="id599" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="220.0"><![CDATA[2]]></text>
+        </g>
+        <g latexdraw:type="text" id="id601" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="280.0"><![CDATA[3]]></text>
+        </g>
+        <g latexdraw:type="text" id="id603" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="320.0"><![CDATA[4]]></text>
+        </g>
+        <g latexdraw:type="text" id="id605" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="360.0"><![CDATA[5]]></text>
+        </g>
+        <g latexdraw:type="text" id="id607" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="420.0"><![CDATA[6]]></text>
+        </g>
+        <g latexdraw:type="text" id="id609" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="460.0"><![CDATA[7]]></text>
+        </g>
+        <g latexdraw:type="text" id="id611" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="500.0"><![CDATA[8]]></text>
+        </g>
+        <g latexdraw:type="text" id="id613" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="560.0"><![CDATA[9]]></text>
+        </g>
+        <g latexdraw:type="text" id="id615" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="600.0"><![CDATA[10]]></text>
+        </g>
+        <g latexdraw:type="text" id="id617" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="770.0">
+            <text x="770.0" y="640.0"><![CDATA[11]]></text>
+        </g>
+        <g latexdraw:type="line" id="id619">
+            <line x1="640.0" x2="760.0" y1="380.0" y2="380.0"
+                stroke-width="6.0" stroke="black"
+                latexdraw:rotationAngle="0.0" marker-end="url(#arrow2-619)"/>
+        </g>
+        <g latexdraw:type="text" id="id620" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="670.0">
+            <text x="670.0" y="360.0"><![CDATA[step 1]]></text>
+        </g>
+        <g latexdraw:type="drawing" id="id622">
+            <g latexdraw:type="rectangle" id="id623">
+                <rect x="1011.0" y="531.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#d4f4e0"/>
+            </g>
+            <g latexdraw:type="rectangle" id="id624">
+                <rect x="1011.0" y="391.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#dfe1f2"/>
+            </g>
+            <g latexdraw:type="rectangle" id="id625">
+                <rect x="1011.0" y="251.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#f1daf9"/>
+            </g>
+            <g latexdraw:type="rectangle" id="id626">
+                <rect x="1011.0" y="111.0" width="38.0" height="118.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="#fafac9"/>
+            </g>
+        </g>
+        <g latexdraw:type="drawing" id="id628">
+            <g latexdraw:type="square" id="id629">
+                <rect x="1021.0" y="261.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id630">
+                <rect x="1021.0" y="301.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id631">
+                <rect x="1021.0" y="341.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+        </g>
+        <g latexdraw:type="text" id="id633" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="140.0"><![CDATA[0]]></text>
+        </g>
+        <g latexdraw:type="text" id="id635" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="180.0"><![CDATA[1]]></text>
+        </g>
+        <g latexdraw:type="text" id="id637" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="220.0"><![CDATA[2]]></text>
+        </g>
+        <g latexdraw:type="text" id="id639" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="280.0"><![CDATA[3]]></text>
+        </g>
+        <g latexdraw:type="text" id="id641" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="320.0"><![CDATA[4]]></text>
+        </g>
+        <g latexdraw:type="text" id="id643" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="360.0"><![CDATA[5]]></text>
+        </g>
+        <g latexdraw:type="text" id="id645" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="420.0"><![CDATA[6]]></text>
+        </g>
+        <g latexdraw:type="text" id="id647" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="460.0"><![CDATA[7]]></text>
+        </g>
+        <g latexdraw:type="text" id="id649" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="500.0"><![CDATA[8]]></text>
+        </g>
+        <g latexdraw:type="text" id="id651" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="560.0"><![CDATA[9]]></text>
+        </g>
+        <g latexdraw:type="text" id="id653" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="600.0"><![CDATA[10]]></text>
+        </g>
+        <g latexdraw:type="text" id="id655" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="980.0">
+            <text x="980.0" y="640.0"><![CDATA[11]]></text>
+        </g>
+        <g latexdraw:type="line" id="id657">
+            <line x1="850.0" x2="970.0" y1="380.0" y2="380.0"
+                stroke-width="6.0" stroke="black"
+                latexdraw:rotationAngle="0.0" marker-end="url(#arrow2-657)"/>
+        </g>
+        <g latexdraw:type="text" id="id658" font-family="Dialog"
+            fill="black" font-size="18" latexdraw:x="880.0">
+            <text x="880.0" y="360.0"><![CDATA[step2]]></text>
+        </g>
+        <g latexdraw:type="drawing" id="id661">
+            <g latexdraw:type="square" id="id662">
+                <rect x="1021.0" y="401.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id663">
+                <rect x="1021.0" y="441.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id664">
+                <rect x="1021.0" y="481.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+        </g>
+        <g latexdraw:type="drawing" id="id666">
+            <g latexdraw:type="square" id="id667">
+                <rect x="1021.0" y="541.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id668">
+                <rect x="1021.0" y="581.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id669">
+                <rect x="1021.0" y="621.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+        </g>
+        <g latexdraw:type="drawing" id="id671">
+            <g latexdraw:type="square" id="id672">
+                <rect x="1021.0" y="121.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id673">
+                <rect x="1021.0" y="161.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+            <g latexdraw:type="square" id="id674">
+                <rect x="1021.0" y="201.0" width="18.0" height="18.0"
+                    latexdraw:borderPos="outer" stroke-width="2.0"
+                    stroke="black" fill="red"/>
+            </g>
+        </g>
+        <g latexdraw:type="square" id="id676">
+            <rect x="811.0" y="261.0" width="18.0" height="18.0"
+                latexdraw:borderPos="outer" stroke-width="2.0"
+                stroke="black" fill="white"/>
         </g>
     </g>
@@ -612,6 +456,6 @@
             <latexdraw:version>2.0.8</latexdraw:version>
             <latexdraw:size>
-                <latexdraw:width>1867</latexdraw:width>
-                <latexdraw:height>1176</latexdraw:height>
+                <latexdraw:width>2507</latexdraw:width>
+                <latexdraw:height>1576</latexdraw:height>
             </latexdraw:size>
             <latexdraw:position>
@@ -619,5 +463,5 @@
                 <latexdraw:y>24</latexdraw:y>
             </latexdraw:position>
-            <latexdraw:dividerPosition>0.8489555436529191</latexdraw:dividerPosition>
+            <latexdraw:dividerPosition>0.9976067012365377</latexdraw:dividerPosition>
             <latexdraw:displayCodePanel>true</latexdraw:displayCodePanel>
             <latexdraw:displayXScale>true</latexdraw:displayXScale>
Index: XIOS/dev/branch_openmp/Note/rapport ESIWACE.aux
===================================================================
--- XIOS/dev/branch_openmp/Note/rapport ESIWACE.aux	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/rapport ESIWACE.aux	(revision 1552)
@@ -8,12 +8,21 @@
 \@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces }}{3}}
 \newlabel{fig:omp}{{3}{3}}
-\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces }}{3}}
-\newlabel{fig:scheme}{{4}{3}}
-\@writefile{toc}{\contentsline {section}{\numberline {3}Performance of LMDZ using EP\_XIOS}{5}}
-\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces Speedup obtained by using EP in LMDZ simulations.}}{5}}
+\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces }}{4}}
+\newlabel{fig:scheme}{{4}{4}}
+\@writefile{lof}{\contentsline {figure}{\numberline {5}{\ignorespaces }}{5}}
+\newlabel{fig:tag}{{5}{5}}
+\@writefile{lof}{\contentsline {figure}{\numberline {6}{\ignorespaces This figure shows the classic pattern of a P2P communication with the endpoint interface. Thread/endpoint rank 0 sends a message to thread/endpoint rank 3 with tag=1. The underlying MPI function called by the sender is indeed a send for MPI rank of 1 and tag=65537. From the receiver's point of view, the endpoint 3 is actually receiving a message from MPI rank 0 with tag=65537.}}{5}}
+\newlabel{fig:sendrecv}{{6}{5}}
+\citation{ep:2018}
+\@writefile{lof}{\contentsline {figure}{\numberline {7}{\ignorespaces }}{6}}
+\newlabel{fig:bcast}{{7}{6}}
+\@writefile{lof}{\contentsline {figure}{\numberline {8}{\ignorespaces }}{6}}
+\newlabel{fig:allreduce}{{8}{6}}
+\citation{ep:2018}
 \bibstyle{plain}
 \bibdata{reference}
-\bibcite{Dinan:2013}{1}
-\bibcite{Sridharan:2014}{2}
-\@writefile{toc}{\contentsline {section}{\numberline {4}Performance of EP\_XIOS}{6}}
-\@writefile{toc}{\contentsline {section}{\numberline {5}Perspectives of EP\_XIOS}{6}}
+\bibcite{ep:2018}{1}
+\bibcite{Dinan:2013}{2}
+\bibcite{Sridharan:2014}{3}
+\@writefile{toc}{\contentsline {section}{\numberline {3}The multi-threaded XIOS and performance results}{7}}
+\@writefile{toc}{\contentsline {section}{\numberline {4}Future works for XIOS}{7}}
Index: XIOS/dev/branch_openmp/Note/rapport ESIWACE.bbl
===================================================================
--- XIOS/dev/branch_openmp/Note/rapport ESIWACE.bbl	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/rapport ESIWACE.bbl	(revision 1552)
@@ -1,14 +1,21 @@
 \begin{thebibliography}{1}
+
+\bibitem{ep:2018}
+XIOS developper group.
+\newblock Note for {XIOS} {E}ndpoints.
+\newblock Technical report,
+  \url{http://forge.ipsl.jussieu.fr/ioserver/browser/XIOS/dev/branch_openmp/Note},
+  2018.
 
 \bibitem{Dinan:2013}
 J.~Dinan, Pavan Balaji, D.~Goodell, D.~Miller, M.~Snir, and Rajeev Thakur.
-\newblock Enabling mpi interoperability through flexible communication
-  endpoints.
+\newblock Enabling {MPI} {I}nteroperability {T}hrough {F}lexible
+  {C}ommunication {E}ndpoints.
 \newblock In {\em EuroMPI 2013}, Madrid, Spain, 2013.
 
 \bibitem{Sridharan:2014}
 S.~Sridharan, J.~Dinan, and D.~D. Kalamkar.
-\newblock Enabling efficient multithreaded mpi communication through a
-  library-based implementation of mpi endpoints.
+\newblock Enabling {E}fficient {M}ultithreaded {MPI} {C}ommunication {T}hrough
+  a {L}ibrary-{B}ased {I}mplementation of {MPI} {E}ndpoints.
 \newblock In {\em SC14: International Conference for High Performance
   Computing, Networking, Storage and Analysis}, pages 487--498, Nov 2014.
Index: XIOS/dev/branch_openmp/Note/rapport ESIWACE.blg
===================================================================
--- XIOS/dev/branch_openmp/Note/rapport ESIWACE.blg	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/rapport ESIWACE.blg	(revision 1552)
@@ -1,46 +1,48 @@
-This is BibTeX, Version 0.99d (TeX Live 2016)
+This is BibTeX, Version 0.99d (TeX Live 2015/Debian)
 Capacity: max_strings=35307, hash_size=35307, hash_prime=30011
 The top-level auxiliary file: rapport ESIWACE.aux
 The style file: plain.bst
 Database file #1: reference.bib
-You've used 2 entries,
+Warning--empty institution in ep:2018
+You've used 3 entries,
             2118 wiz_defined-function locations,
-            512 strings with 4502 characters,
-and the built_in function-call counts, 893 in all, are:
-= -- 80
-> -- 51
+            517 strings with 4672 characters,
+and the built_in function-call counts, 1094 in all, are:
+= -- 101
+> -- 58
 < -- 1
-+ -- 20
-- -- 18
-* -- 63
-:= -- 146
-add.period$ -- 6
-call.type$ -- 2
-change.case$ -- 15
++ -- 23
+- -- 20
+* -- 73
+:= -- 186
+add.period$ -- 9
+call.type$ -- 3
+change.case$ -- 20
 chr.to.int$ -- 0
-cite$ -- 2
-duplicate$ -- 37
-empty$ -- 68
-format.name$ -- 18
-if$ -- 192
+cite$ -- 4
+duplicate$ -- 45
+empty$ -- 83
+format.name$ -- 20
+if$ -- 231
 int.to.chr$ -- 0
-int.to.str$ -- 2
+int.to.str$ -- 3
 missing$ -- 2
-newline$ -- 13
-num.names$ -- 4
-pop$ -- 22
+newline$ -- 18
+num.names$ -- 6
+pop$ -- 25
 preamble$ -- 1
-purify$ -- 13
+purify$ -- 16
 quote$ -- 0
-skip$ -- 25
+skip$ -- 30
 stack$ -- 0
-substring$ -- 38
+substring$ -- 43
 swap$ -- 12
 text.length$ -- 1
 text.prefix$ -- 0
 top$ -- 0
-type$ -- 8
-warning$ -- 0
-while$ -- 6
-width$ -- 3
-write$ -- 24
+type$ -- 12
+warning$ -- 1
+while$ -- 8
+width$ -- 4
+write$ -- 35
+(There was 1 warning)
Index: XIOS/dev/branch_openmp/Note/rapport ESIWACE.log
===================================================================
--- XIOS/dev/branch_openmp/Note/rapport ESIWACE.log	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/rapport ESIWACE.log	(revision 1552)
@@ -1,3 +1,3 @@
-This is pdfTeX, Version 3.14159265-2.6-1.40.17 (TeX Live 2016) (preloaded format=pdflatex 2016.7.25)  25 JUN 2018 17:20
+This is pdfTeX, Version 3.14159265-2.6-1.40.16 (TeX Live 2015/Debian) (preloaded format=pdflatex 2017.8.24)  27 JUN 2018 15:10
 entering extended mode
  restricted \write18 enabled.
@@ -5,9 +5,9 @@
 **"rapport ESIWACE.tex"
 (./rapport ESIWACE.tex
-LaTeX2e <2016/03/31>
-Babel <3.9r> and hyphenation patterns for 83 language(s) loaded.
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/article.cls
+LaTeX2e <2016/02/01>
+Babel <3.9q> and hyphenation patterns for 3 language(s) loaded.
+(/usr/share/texlive/texmf-dist/tex/latex/base/article.cls
 Document Class: article 2014/09/29 v1.4h Standard LaTeX document class
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/size10.clo
+(/usr/share/texlive/texmf-dist/tex/latex/base/size10.clo
 File: size10.clo 2014/09/29 v1.4h Standard LaTeX file (size option)
 )
@@ -24,11 +24,11 @@
 \bibindent=\dimen102
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/inputenc.sty
+(/usr/share/texlive/texmf-dist/tex/latex/base/inputenc.sty
 Package: inputenc 2015/03/17 v1.2c Input encoding file
 \inpenc@prehook=\toks14
 \inpenc@posthook=\toks15
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/utf8.def
-File: utf8.def 2016/02/28 v1.1s UTF-8 support for inputenc
+(/usr/share/texlive/texmf-dist/tex/latex/base/utf8.def
+File: utf8.def 2015/12/03 v1.1r UTF-8 support for inputenc
 Now handling font encoding OML ...
 ... no UTF-8 mapping file for font encoding OML
@@ -36,6 +36,6 @@
 ... processing UTF-8 mapping file for font encoding T1
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/t1enc.dfu
-File: t1enc.dfu 2016/02/28 v1.1s UTF-8 support for inputenc
+(/usr/share/texlive/texmf-dist/tex/latex/base/t1enc.dfu
+File: t1enc.dfu 2015/12/03 v1.1r UTF-8 support for inputenc
    defining Unicode char U+00A0 (decimal 160)
    defining Unicode char U+00A1 (decimal 161)
@@ -249,13 +249,9 @@
    defining Unicode char U+021A (decimal 538)
    defining Unicode char U+021B (decimal 539)
-   defining Unicode char U+1E02 (decimal 7682)
-   defining Unicode char U+1E03 (decimal 7683)
+   defining Unicode char U+01E02 (decimal 7682)
+   defining Unicode char U+01E03 (decimal 7683)
    defining Unicode char U+200C (decimal 8204)
-   defining Unicode char U+2010 (decimal 8208)
-   defining Unicode char U+2011 (decimal 8209)
-   defining Unicode char U+2012 (decimal 8210)
    defining Unicode char U+2013 (decimal 8211)
    defining Unicode char U+2014 (decimal 8212)
-   defining Unicode char U+2015 (decimal 8213)
    defining Unicode char U+2018 (decimal 8216)
    defining Unicode char U+2019 (decimal 8217)
@@ -273,6 +269,6 @@
 ... processing UTF-8 mapping file for font encoding OT1
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/ot1enc.dfu
-File: ot1enc.dfu 2016/02/28 v1.1s UTF-8 support for inputenc
+(/usr/share/texlive/texmf-dist/tex/latex/base/ot1enc.dfu
+File: ot1enc.dfu 2015/12/03 v1.1r UTF-8 support for inputenc
    defining Unicode char U+00A0 (decimal 160)
    defining Unicode char U+00A1 (decimal 161)
@@ -314,6 +310,6 @@
 ... processing UTF-8 mapping file for font encoding OMS
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/base/omsenc.dfu
-File: omsenc.dfu 2016/02/28 v1.1s UTF-8 support for inputenc
+(/usr/share/texlive/texmf-dist/tex/latex/base/omsenc.dfu
+File: omsenc.dfu 2015/12/03 v1.1r UTF-8 support for inputenc
    defining Unicode char U+00A7 (decimal 167)
    defining Unicode char U+00B6 (decimal 182)
@@ -338,30 +334,30 @@
    defining Unicode char U+2423 (decimal 9251)
 ))
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics/graphicx.sty
+(/usr/share/texlive/texmf-dist/tex/latex/graphics/graphicx.sty
 Package: graphicx 2014/10/28 v1.0g Enhanced LaTeX Graphics (DPC,SPQR)
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics/keyval.sty
+(/usr/share/texlive/texmf-dist/tex/latex/graphics/keyval.sty
 Package: keyval 2014/10/28 v1.15 key=value parser (DPC)
 \KV@toks@=\toks16
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics/graphics.sty
-Package: graphics 2016/05/09 v1.0r Standard LaTeX Graphics (DPC,SPQR)
-
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics/trig.sty
+(/usr/share/texlive/texmf-dist/tex/latex/graphics/graphics.sty
+Package: graphics 2016/01/03 v1.0q Standard LaTeX Graphics (DPC,SPQR)
+
+(/usr/share/texlive/texmf-dist/tex/latex/graphics/trig.sty
 Package: trig 2016/01/03 v1.10 sin cos tan (DPC)
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics-cfg/graphics.cfg
-File: graphics.cfg 2016/01/02 v1.10 sample graphics configuration
-)
-Package graphics Info: Driver file: pdftex.def on input line 96.
-
-(/usr/local/texlive/2016/texmf-dist/tex/latex/pdftex-def/pdftex.def
+(/usr/share/texlive/texmf-dist/tex/latex/latexconfig/graphics.cfg
+File: graphics.cfg 2010/04/23 v1.9 graphics configuration of TeX Live
+)
+Package graphics Info: Driver file: pdftex.def on input line 95.
+
+(/usr/share/texlive/texmf-dist/tex/latex/pdftex-def/pdftex.def
 File: pdftex.def 2011/05/27 v0.06d Graphics/color for pdfTeX
 
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/infwarerr.sty
-Package: infwarerr 2016/05/16 v1.4 Providing info/warning/error messages (HO)
-)
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/ltxcmds.sty
-Package: ltxcmds 2016/05/16 v1.23 LaTeX kernel commands for general use (HO)
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/infwarerr.sty
+Package: infwarerr 2010/04/08 v1.3 Providing info/warning/error messages (HO)
+)
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ltxcmds.sty
+Package: ltxcmds 2011/11/09 v1.22 LaTeX kernel commands for general use (HO)
 )
 \Gread@gobject=\count87
@@ -370,5 +366,5 @@
 \Gin@req@width=\dimen104
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/listings/listings.sty
+(/usr/share/texlive/texmf-dist/tex/latex/listings/listings.sty
 \lst@mode=\count88
 \lst@gtempboxa=\box26
@@ -384,5 +380,5 @@
 \lst@maxwidth=\dimen108
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/listings/lstmisc.sty
+(/usr/share/texlive/texmf-dist/tex/latex/listings/lstmisc.sty
 File: lstmisc.sty 2015/06/04 1.6 (Carsten Heinz)
 \c@lstnumber=\count94
@@ -390,22 +386,22 @@
 \lst@framebox=\box27
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/listings/listings.cfg
+(/usr/share/texlive/texmf-dist/tex/latex/listings/listings.cfg
 File: listings.cfg 2015/06/04 1.6 listings configuration
 ))
 Package: listings 2015/06/04 1.6 (Carsten Heinz)
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/xcolor/xcolor.sty
-Package: xcolor 2016/05/11 v2.12 LaTeX color extensions (UK)
-
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics-cfg/color.cfg
-File: color.cfg 2016/01/02 v1.6 sample color configuration
+(/usr/share/texlive/texmf-dist/tex/latex/xcolor/xcolor.sty
+Package: xcolor 2007/01/21 v2.11 LaTeX color extensions (UK)
+
+(/usr/share/texlive/texmf-dist/tex/latex/latexconfig/color.cfg
+File: color.cfg 2007/01/18 v1.5 color configuration of teTeX/TeXLive
 )
 Package xcolor Info: Package option `usenames' ignored on input line 216.
 Package xcolor Info: Driver file: pdftex.def on input line 225.
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/colortbl/colortbl.sty
+(/usr/share/texlive/texmf-dist/tex/latex/colortbl/colortbl.sty
 Package: colortbl 2012/02/13 v1.0a Color table columns (DPC)
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/tools/array.sty
+(/usr/share/texlive/texmf-dist/tex/latex/tools/array.sty
 Package: array 2014/10/28 v2.4c Tabular extension package (FMi)
 \col@sep=\dimen109
@@ -419,39 +415,39 @@
 )
 \rownum=\count96
-Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1348.
-Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1352.
-Package xcolor Info: Model `RGB' extended on input line 1364.
-Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1366.
-Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1367.
-Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1368.
-Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1369.
-Package xcolor Info: Model `Gray' substituted by `gray' on input line 1370.
-Package xcolor Info: Model `wave' substituted by `hsb' on input line 1371.
-
-(/usr/local/texlive/2016/texmf-dist/tex/latex/graphics/dvipsnam.def
+Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1337.
+Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1341.
+Package xcolor Info: Model `RGB' extended on input line 1353.
+Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1355.
+Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1356.
+Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1357.
+Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1358.
+Package xcolor Info: Model `Gray' substituted by `gray' on input line 1359.
+Package xcolor Info: Model `wave' substituted by `hsb' on input line 1360.
+
+(/usr/share/texlive/texmf-dist/tex/latex/graphics/dvipsnam.def
 File: dvipsnam.def 2015/12/30 v3.0k Driver-dependent file (DPC,SPQR)
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/xcolor/svgnam.def
-File: svgnam.def 2016/05/11 v2.12 Predefined colors according to SVG 1.1 (UK)
+(/usr/share/texlive/texmf-dist/tex/latex/xcolor/svgnam.def
+File: svgnam.def 2007/01/21 v2.11 Predefined colors according to SVG 1.1 (UK)
 ))
-(/usr/local/texlive/2016/texmf-dist/tex/latex/amsmath/amsmath.sty
-Package: amsmath 2016/03/10 v2.15b AMS math features
+(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsmath.sty
+Package: amsmath 2016/03/03 v2.15a AMS math features
 \@mathmargin=\skip46
 
 For additional information on amsmath, use the `?' option.
-(/usr/local/texlive/2016/texmf-dist/tex/latex/amsmath/amstext.sty
+(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amstext.sty
 Package: amstext 2000/06/29 v2.01 AMS text
 
-(/usr/local/texlive/2016/texmf-dist/tex/latex/amsmath/amsgen.sty
+(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsgen.sty
 File: amsgen.sty 1999/11/30 v2.0 generic functions
 \@emptytoks=\toks20
 \ex@=\dimen111
 ))
-(/usr/local/texlive/2016/texmf-dist/tex/latex/amsmath/amsbsy.sty
+(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsbsy.sty
 Package: amsbsy 1999/11/29 v1.2d Bold Symbols
 \pmbraise@=\dimen112
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/amsmath/amsopn.sty
-Package: amsopn 2016/03/08 v2.02 operator names
+(/usr/share/texlive/texmf-dist/tex/latex/amsmath/amsopn.sty
+Package: amsopn 1999/12/14 v2.01 operator names
 )
 \inf@bad=\count97
@@ -468,6 +464,6 @@
 \strutbox@=\box29
 \big@size=\dimen113
-LaTeX Font Info:    Redeclaring font encoding OML on input line 634.
-LaTeX Font Info:    Redeclaring font encoding OMS on input line 635.
+LaTeX Font Info:    Redeclaring font encoding OML on input line 630.
+LaTeX Font Info:    Redeclaring font encoding OMS on input line 631.
 \macc@depth=\count102
 \c@MaxMatrixCols=\count103
@@ -490,28 +486,34 @@
 \multlinetaggap=\skip48
 \mathdisplay@stack=\toks24
-LaTeX Info: Redefining \[ on input line 2739.
-LaTeX Info: Redefining \] on input line 2740.
-)
-(/usr/local/texlive/2016/texmf-dist/tex/latex/url/url.sty
+LaTeX Info: Redefining \[ on input line 2735.
+LaTeX Info: Redefining \] on input line 2736.
+)
+(/usr/share/texlive/texmf-dist/tex/latex/url/url.sty
 \Urlmuskip=\muskip11
 Package: url 2013/09/16  ver 3.4  Verb mode for urls, etc.
 )
+(/usr/share/texlive/texmf-dist/tex/latex/tools/verbatim.sty
+Package: verbatim 2014/10/28 v1.5q LaTeX2e package for verbatim enhancements
+\every@verbatim=\toks25
+\verbatim@line=\toks26
+\verbatim@in@stream=\read1
+)
 (./rapport ESIWACE.aux)
 \openout1 = `"rapport ESIWACE.aux"'.
 
-LaTeX Font Info:    Checking defaults for OML/cmm/m/it on input line 16.
-LaTeX Font Info:    ... okay on input line 16.
-LaTeX Font Info:    Checking defaults for T1/cmr/m/n on input line 16.
-LaTeX Font Info:    ... okay on input line 16.
-LaTeX Font Info:    Checking defaults for OT1/cmr/m/n on input line 16.
-LaTeX Font Info:    ... okay on input line 16.
-LaTeX Font Info:    Checking defaults for OMS/cmsy/m/n on input line 16.
-LaTeX Font Info:    ... okay on input line 16.
-LaTeX Font Info:    Checking defaults for OMX/cmex/m/n on input line 16.
-LaTeX Font Info:    ... okay on input line 16.
-LaTeX Font Info:    Checking defaults for U/cmr/m/n on input line 16.
-LaTeX Font Info:    ... okay on input line 16.
-
-(/usr/local/texlive/2016/texmf-dist/tex/context/base/mkii/supp-pdf.mkii
+LaTeX Font Info:    Checking defaults for OML/cmm/m/it on input line 17.
+LaTeX Font Info:    ... okay on input line 17.
+LaTeX Font Info:    Checking defaults for T1/cmr/m/n on input line 17.
+LaTeX Font Info:    ... okay on input line 17.
+LaTeX Font Info:    Checking defaults for OT1/cmr/m/n on input line 17.
+LaTeX Font Info:    ... okay on input line 17.
+LaTeX Font Info:    Checking defaults for OMS/cmsy/m/n on input line 17.
+LaTeX Font Info:    ... okay on input line 17.
+LaTeX Font Info:    Checking defaults for OMX/cmex/m/n on input line 17.
+LaTeX Font Info:    ... okay on input line 17.
+LaTeX Font Info:    Checking defaults for U/cmr/m/n on input line 17.
+LaTeX Font Info:    ... okay on input line 17.
+
+(/usr/share/texlive/texmf-dist/tex/context/base/supp-pdf.mkii
 [Loading MPS to PDF converter (version 2006.09.02).]
 \scratchcounter=\count109
@@ -520,20 +522,21 @@
 \nofMPsegments=\count110
 \nofMParguments=\count111
-\everyMPshowfont=\toks25
+\everyMPshowfont=\toks27
 \MPscratchCnt=\count112
 \MPscratchDim=\dimen121
 \MPnumerator=\count113
 \makeMPintoPDFobject=\count114
-\everyMPtoPDFconversion=\toks26
-) (/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/pdftexcmds.sty
-Package: pdftexcmds 2016/05/10 v0.21 Utility functions of pdfTeX for LuaTeX (HO
-)
-
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/ifluatex.sty
-Package: ifluatex 2016/05/16 v1.4 Provides the ifluatex switch (HO)
+\everyMPtoPDFconversion=\toks28
+) (/usr/share/texlive/texmf-dist/tex/generic/oberdiek/pdftexcmds.sty
+Package: pdftexcmds 2011/11/29 v0.20 Utility functions of pdfTeX for LuaTeX (HO
+)
+
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ifluatex.sty
+Package: ifluatex 2010/03/01 v1.3 Provides the ifluatex switch (HO)
 Package ifluatex Info: LuaTeX not detected.
 )
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/ifpdf.sty
-Package: ifpdf 2016/05/14 v3.1 Provides the ifpdf switch
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ifpdf.sty
+Package: ifpdf 2011/01/30 v2.3 Provides the ifpdf switch (HO)
+Package ifpdf Info: pdfTeX in PDF mode is detected.
 )
 Package pdftexcmds Info: LuaTeX not detected.
@@ -542,21 +545,21 @@
 Package pdftexcmds Info: \pdfdraftmode found.
 )
-(/usr/local/texlive/2016/texmf-dist/tex/latex/oberdiek/epstopdf-base.sty
-Package: epstopdf-base 2016/05/15 v2.6 Base part for package epstopdf
-
-(/usr/local/texlive/2016/texmf-dist/tex/latex/oberdiek/grfext.sty
-Package: grfext 2016/05/16 v1.2 Manage graphics extensions (HO)
-
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/kvdefinekeys.sty
-Package: kvdefinekeys 2016/05/16 v1.4 Define keys (HO)
+(/usr/share/texlive/texmf-dist/tex/latex/oberdiek/epstopdf-base.sty
+Package: epstopdf-base 2010/02/09 v2.5 Base part for package epstopdf
+
+(/usr/share/texlive/texmf-dist/tex/latex/oberdiek/grfext.sty
+Package: grfext 2010/08/19 v1.1 Manage graphics extensions (HO)
+
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/kvdefinekeys.sty
+Package: kvdefinekeys 2011/04/07 v1.3 Define keys (HO)
 ))
-(/usr/local/texlive/2016/texmf-dist/tex/latex/oberdiek/kvoptions.sty
-Package: kvoptions 2016/05/16 v3.12 Key value format for package options (HO)
-
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/kvsetkeys.sty
-Package: kvsetkeys 2016/05/16 v1.17 Key value parser (HO)
-
-(/usr/local/texlive/2016/texmf-dist/tex/generic/oberdiek/etexcmds.sty
-Package: etexcmds 2016/05/16 v1.6 Avoid name clashes with e-TeX commands (HO)
+(/usr/share/texlive/texmf-dist/tex/latex/oberdiek/kvoptions.sty
+Package: kvoptions 2011/06/30 v3.11 Key value format for package options (HO)
+
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/kvsetkeys.sty
+Package: kvsetkeys 2012/04/25 v1.16 Key value parser (HO)
+
+(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/etexcmds.sty
+Package: etexcmds 2011/02/16 v1.5 Avoid name clashes with e-TeX commands (HO)
 Package etexcmds Info: Could not find \expanded.
 (etexcmds)             That can mean that you are not using pdfTeX 1.50 or
@@ -567,7 +570,7 @@
 (grfext)             [.png,.pdf,.jpg,.mps,.jpeg,.jbig2,.jb2,.PNG,.PDF,.JPG,.JPE
 G,.JBIG2,.JB2,.eps]
-(grfext)             \AppendGraphicsExtensions on input line 456.
-
-(/usr/local/texlive/2016/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg
+(grfext)             \AppendGraphicsExtensions on input line 452.
+
+(/usr/share/texlive/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg
 File: epstopdf-sys.cfg 2010/07/13 v1.3 Configuration of (r)epstopdf for TeX Liv
 e
@@ -578,5 +581,5 @@
 File: Charge1.png Graphic file (type png)
  <use Charge1.png>
-Package pdftex.def Info: Charge1.png used on input line 32.
+Package pdftex.def Info: Charge1.png used on input line 33.
 (pdftex.def)             Requested size: 165.01357pt x 91.23924pt.
 
@@ -584,13 +587,14 @@
 File: Charge2.png Graphic file (type png)
  <use Charge2.png>
-Package pdftex.def Info: Charge2.png used on input line 33.
+Package pdftex.def Info: Charge2.png used on input line 34.
 (pdftex.def)             Requested size: 165.25446pt x 91.05858pt.
  [1
 
-{/usr/local/texlive/2016/texmf-var/fonts/map/pdftex/updmap/pdftex.map} <./Charg
-e1.png> <./Charge2.png>] <domain.pdf, id=16, 472.31454pt x 142.26149pt>
+{/var/lib/texmf/fonts/map/pdftex/updmap/pdftex.map} <./Charge1.png> <./Charge2.
+png>]
+<domain.pdf, id=16, 472.31454pt x 142.26149pt>
 File: domain.pdf Graphic file (type pdf)
  <use domain.pdf>
-Package pdftex.def Info: domain.pdf used on input line 59.
+Package pdftex.def Info: domain.pdf used on input line 60.
 (pdftex.def)             Requested size: 236.1567pt x 71.13055pt.
 
@@ -598,50 +602,73 @@
 File: omp.pdf Graphic file (type pdf)
  <use omp.pdf>
-Package pdftex.def Info: omp.pdf used on input line 67.
+Package pdftex.def Info: omp.pdf used on input line 68.
 (pdftex.def)             Requested size: 291.64784pt x 126.32893pt.
-
-
-LaTeX Warning: `!h' float specifier changed to `!ht'.
-
-[2 <./domain.pdf>] <scheme.png, id=27, 665.48625pt x 517.935pt>
+ [2 <./domain.pdf>]
+<scheme.png, id=27, 665.48625pt x 517.935pt>
 File: scheme.png Graphic file (type png)
-
-<use scheme.png>
-Package pdftex.def Info: scheme.png used on input line 79.
+ <use scheme.png>
+Package pdftex.def Info: scheme.png used on input line 86.
 (pdftex.def)             Requested size: 266.18977pt x 207.17032pt.
- [3 <./omp.pdf> <./scheme.png (PNG copy)>] [4]
-<LMDZ_perf.png, id=39, 530.73282pt x 255.95625pt>
-File: LMDZ_perf.png Graphic file (type png)
- <use LMDZ_perf.png>
-Package pdftex.def Info: LMDZ_perf.png used on input line 144.
-(pdftex.def)             Requested size: 318.44214pt x 153.57492pt.
- [5 <./LMDZ_perf.png (PNG copy)>] (./rapport ESIWACE.bbl) [6] (./rapport ESIWAC
-E.aux) ) 
+ [3 <./omp.pdf>]
+<tag.png, id=36, 752.8125pt x 103.38625pt>
+File: tag.png Graphic file (type png)
+ <use tag.png>
+Package pdftex.def Info: tag.png used on input line 140.
+(pdftex.def)             Requested size: 301.11966pt x 41.35376pt.
+ [4 <./scheme.png (PNG copy)>] <sendrecv.png, id=41, 829.0975pt x 694.595pt>
+File: sendrecv.png Graphic file (type png)
+ <use sendrecv.png>
+Package pdftex.def Info: sendrecv.png used on input line 149.
+(pdftex.def)             Requested size: 331.63313pt x 277.83307pt.
+
+[5 <./tag.png (PNG copy)> <./sendrecv.png (PNG copy)>]
+<bcast.png, id=45, 512.91624pt x 552.0625pt>
+File: bcast.png Graphic file (type png)
+ <use bcast.png>
+Package pdftex.def Info: bcast.png used on input line 173.
+(pdftex.def)             Requested size: 153.87605pt x 165.62003pt.
+
+<allreduce.png, id=46, 743.77875pt x 552.0625pt>
+File: allreduce.png Graphic file (type png)
+ <use allreduce.png>
+Package pdftex.def Info: allreduce.png used on input line 185.
+(pdftex.def)             Requested size: 223.13535pt x 165.62003pt.
+ [6 <./bcast.png (PNG copy)> <./allreduce.png (PNG copy)>] (./rapport ESIWACE.b
+bl
+Underfull \hbox (badness 3354) in paragraph at lines 4--9
+[]\OT1/cmr/m/n/10 XIOS de-vel-op-per group.  Note for XIOS End-points.  Tech-ni
+-cal
+ []
+
+
+Underfull \hbox (badness 4819) in paragraph at lines 4--9
+\OT1/cmr/m/n/10 re-port, $\OT1/cmtt/m/n/10 http : / / forge . ipsl . jussieu . 
+fr / ioserver / browser / XIOS / dev /
+ []
+
+) [7] (./rapport ESIWACE.aux) ) 
 Here is how much of TeX's memory you used:
- 4595 strings out of 493014
- 61054 string characters out of 6133351
- 142700 words of memory out of 5000000
- 8068 multiletter control sequences out of 15000+600000
+ 4637 strings out of 494953
+ 61251 string characters out of 6180977
+ 137489 words of memory out of 5000000
+ 7849 multiletter control sequences out of 15000+600000
  9090 words of font info for 34 fonts, out of 8000000 for 9000
- 1141 hyphenation exceptions out of 8191
+ 14 hyphenation exceptions out of 8191
  41i,8n,35p,1270b,264s stack positions out of 5000i,500n,10000p,200000b,80000s
-</usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmbx
-12.pfb></usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi1
-0.pfb></usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmmi7.
-pfb></usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmr10.pf
-b></usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmr12.pfb>
-</usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmr17.pfb></
-usr/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmr6.pfb></usr
-/local/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmr7.pfb></usr/lo
-cal/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmr8.pfb></usr/local
-/texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmsy10.pfb></usr/local/
-texlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmti10.pfb></usr/local/t
-exlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmtt10.pfb></usr/local/te
-xlive/2016/texmf-dist/fonts/type1/public/amsfonts/cm/cmtt8.pfb>
-Output written on "rapport ESIWACE.pdf" (6 pages, 229957 bytes).
+</usr/share/texlive/texmf-dist/fonts/type1/publi
+c/amsfonts/cm/cmbx12.pfb></usr/share/texlive/texmf-dist/fonts/type1/public/amsf
+onts/cm/cmr10.pfb></usr/share/texlive/texmf-dist/fonts/type1/public/amsfonts/cm
+/cmr12.pfb></usr/share/texlive/texmf-dist/fonts/type1/public/amsfonts/cm/cmr17.
+pfb></usr/share/texlive/texmf-dist/fonts/type1/public/amsfonts/cm/cmr6.pfb></us
+r/share/texlive/texmf-dist/fonts/type1/public/amsfonts/cm/cmr7.pfb></usr/share/
+texlive/texmf-dist/fonts/type1/public/amsfonts/cm/cmr8.pfb></usr/share/texlive/
+texmf-dist/fonts/type1/public/amsfonts/cm/cmti10.pfb></usr/share/texlive/texmf-
+dist/fonts/type1/public/amsfonts/cm/cmtt10.pfb></usr/share/texlive/texmf-dist/f
+onts/type1/public/amsfonts/cm/cmtt8.pfb>
+Output written on "rapport ESIWACE.pdf" (7 pages, 269577 bytes).
 PDF statistics:
- 90 PDF objects out of 1000 (max. 8388607)
- 62 compressed objects within 1 object stream
+ 86 PDF objects out of 1000 (max. 8388607)
+ 57 compressed objects within 1 object stream
  0 named destinations out of 1000 (max. 500000)
- 31 words of extra memory for PDF output out of 10000 (max. 10000000)
-
+ 46 words of extra memory for PDF output out of 10000 (max. 10000000)
+
Index: XIOS/dev/branch_openmp/Note/rapport ESIWACE.tex
===================================================================
--- XIOS/dev/branch_openmp/Note/rapport ESIWACE.tex	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/rapport ESIWACE.tex	(revision 1552)
@@ -6,4 +6,5 @@
 \usepackage{amsmath}
 \usepackage{url}
+\usepackage{verbatim}
 
 % Title Page
@@ -29,5 +30,5 @@
 writing on the server side can be overlapped completely by calculates on the client side. 
 
-\begin{figure}[h]
+\begin{figure}[ht]
 \includegraphics[scale=0.4]{Charge1.png}
 \includegraphics[scale=0.4]{Charge2.png}
@@ -55,5 +56,5 @@
 This topic comes along with the configuration of the climate models. Take LMDZ as example, it is designed with the 2-level parallelization scheme. To be more specific, LMDZ uses the domain decomposition method in which each sub-domain is associated with one MPI process. Inside of the sub-domain, the model also uses OpenMP derivatives to accelerate the computation. We can imagine that the sub-domain be divided into sub-sub-domain and is managed by threads. 
 
-\begin{figure}[h]
+\begin{figure}[ht]
 \centering
 \includegraphics[scale=0.5]{domain.pdf}
@@ -63,5 +64,5 @@
 As we know, each sub-domain, or in another word, each MPI process is a XIOS client. The data exchange between client and XIOS servers is handled by MPI communications. In order to write an output field, all threads must gather the data to the master thread who acts as MPI process in order to call MPI routines. There are two disadvantages about this method : first, we have to spend time on gathering information to the master thread which not only increases the memory use, but also implies an OpenMP barrier; second, while the master thread calls MPI routine, other threads are in the idle state thus a waster of computing resources. What we want obtain with the thread-friendly XIOS is that all threads can act like MPI processes. They can call directly the MPI routine thus no waste in memory nor in computing resources as shown in Figure \ref{fig:omp}.
 
-\begin{figure}[h!]
+\begin{figure}[ht]
 \centering
 \includegraphics[scale=0.6]{omp.pdf}
@@ -73,7 +74,13 @@
 
 
-In this project, we choose to implement the interface to handle the threads. To do so, we introduce the MPI\_endpoint which is a concept proposed in the last MPI Forum and several papers has already discussed the importance of such idea and have introduced the framework of the MPI\_endpoint \cite{Dinan:2013}\cite{Sridharan:2014}. The concept of an endpoint is shown by Figure \ref{fig:scheme}. Threads of an MPI process is associated with a unique rank (global endpoint rank) and an endpoint communicator. They also have a local rank (rank inside the MPI process) which is very similar to the \verb|OMP_thread_num| rank. 
-
-\begin{figure}[h!]
+In this project, we choose to implement an interface to handle the threads. To do so, we introduce the MPI\_endpoint which is a 
+concept proposed in the last MPI Forums and several papers have already discussed the importance of such idea and have introduced the 
+framework of the MPI\_endpoint \cite{Dinan:2013}\cite{Sridharan:2014}. The concept of an endpoint is shown by Figure \ref{fig:scheme}. In 
+the MPI\_endpoint environment, each OpenMP thread will be associated with a unique rank (global endpoint rank), an endpoint communicator, 
+and a local rank (rank inside the MPI process) which is very similar to the \verb|OMP_thread_num|. The global endpoint rank will replace the 
+role of the classic MPI rank and will be used in MPI communication calls. 
+
+
+\begin{figure}[ht]
 \begin{center}
 \includegraphics[scale=0.4]{scheme.png} 
@@ -85,50 +92,120 @@
 %XIOS is a library dedicated to IO management of climate code. It has a client-server pattern in which clients are in charge of computations and servers manage the reading and writing of files. The communication between clients and servers are handled by MPI. However, some of the climate models (\textit{e.g.} LMDZ) nowadays use an hybrid programming policy. Within a shared memory node, OpenMP directives are used to manage message exchanges. In such configuration, XIOS can not take full advantages of the computing resources to maximize the performance. This is because XIOS can only work with MPI processes. Before each call of XIOS routines, threads of one MPI process must gather their information to the master thread who works as an MPI process. After the call, the master thread distributes the updated information among its slave threads. As result, all slave threads have to wait while the master thread calls the XIOS routines. This introduce extra synchronization into the model and leads to not optimized performance. Aware of this situation, we need to develop a new version of XIOS (EP\_XIOS) which can work with threads, or in other words, can consider threads as they were processes. To do so, we introduce the MPI endpoints.  
 
-
-The MPI\_endpoints interface we implemented lies on top of an existing MPI Implementation. It consists of wrappers to all MPI functions used in XIOS. 
-
-will be re-implemented in order to cope with OpenMP threads. The idea is that, in the MPI endpoints environment, each OpenMP thread will be 
-associated with a unique rank and with an endpoint communicator. This rank (EP rank) will replace the role of the classic MPI rank and will 
-be used in MPI communications. In order to successfully execute an MPI communication, for example \verb|MPI_Send|, we know already which 
-endpoints to be the receiver but not sufficient. We also need to know which MPI process should be involved in such communication. To 
-identify the MPI rank, we added a ``map'' in the EP communicator in which the relation of all EP and MPI ranks can be easily obtained.
-
-
-In XIOS, we used the ``probe'' technique to search for arrived messages and then performing the receive action. The principle is 
-that sender processes execute the send operations as usual. However, to minimize the time spent on waiting incoming messages, the receiver 
-processe performs in the first place the \verb|MPI_Probe| function to check if a message destinated to it has been published. If yes, the 
-process execute in the second place the \verb|MPI_Recv| to receive the message. In this situation, if we introduce the threads, problems 
-occur. The reason why the ``probe'' method is not suitable is that messages destinated to one certain process can be probed by any of 
-its threads. Thus the message can be received by the wrong thread which gives errors.
-
-To solve this problem, we introduce the ``matching-probe'' technique. The idea of the method is that each process is equiped with a local 
-incoming message queue. All incoming message will be probed, sorted, and then stored in this queue according to their destination rank. 
-Every time we call an MPI function, we firstly call the \verb|MPI_Mprobe| function to get the handle to 
-the incoming message. Then, we identify the destination thread rank and store the message handle inside the local queue of the target 
-thread. After this, we perform the usual ``probe'' technique upon the local incoming message queue. In this way, we can assure the messages 
-to be received by the right thread.
-
-Another issue remains in this technique: how to identify the receiver's rank? The solution is to use the tag argument. In the MPI 
-environment, a tag is an integer ranging from 0 to $2^{31}$. We can explore the large range of the tag to store in it information about the 
-source and destination thread ranks. We choose to limite the first 15 bits for the tag used in the classic MPI communication, the next 8 
-bits to the sender's thread rank, and the last 8 bits to the receiver's thread rank. In such way, with an extra analysis of the EP tag, we 
-can identify the ranks of the sender and the receiver in any P2P communication. As results, we a thread probes a message, it knows 
-exactly in which local queue should store the probed message. 
-
-
-With the global rank map, tag extension, and the matching-probe techniques, we are able to use any P2P communication in the endpoint 
-environment. For the collective communications, we perform a step-by-step execution and no special technique is required. The most 
-representative functions is the collective communications are \verb|MPI_Gather| and \verb|MPI_Bcast|. A step-by-step execution consists of 
-3 steps (not necessarily in this order): arrangement of the source data, execution of the MPI function by all 
-master/root threads, distribution or arrangement of the data among threads. 
-
-For example, if we want to perform a broadcast operation, 2 steps are needed. Firstly, the root thread, along with the master threads of 
-other processes, perform the classic \verb|MPI_Bcast| operation. Secondly, the root thread, and the master threads send data to threads 
-sharing the same process via local memory transfer. In another example for illustrating the \verb|MPI_Gather| function, we also need 2 
-steps. First of all, data is gathered from slave threads to the master thread or the root thread. Next, the master thread and the root 
-thread execute the \verb|MPI_Gather| operation of complete the communication. Other collective calls such as \verb|MPI_Scan|, 
-\verb|MPI_Reduce|, \verb|MPI_Scatter| \textit{etc} follow the same principle of step-by-step execution.
-
-
+An other important aspect about the MPI\_endpoint interface is that each endpoints has knowledge of the ranks of other endpoints in the 
+same communicator. This knowledge is necessary because when executing an MPI communication, for example a point-to-point exchange, we need 
+to know not only the ranks of sender/receiver threads, but also the thread number of the sender/receiver threads and the MPI ranks 
+of the sender/receiver processes. This ranking information is implemented inside an map object included in the endpoint communicator 
+class.
+
+
+
+
+%\newpage
+
+%The MPI\_endpoint interface we implemented lies on top of an existing MPI Implementation. It consists of wrappers to all MPI functions 
+%used in XIOS. 
+
+
+In XIOS, we used the ``probe'' technique to search for arrived messages and then perform the receiving action. The principle is 
+that sender process executes the send operation as usual. However, to minimize the time spent on waiting incoming messages, the receiver 
+process calls in the first place the \verb|MPI_Probe| function to check if a message destinate to it has been published. If yes, the 
+process execute in the second place the \verb|MPI_Recv| to receive the message. If not, the receiver process can carry on with other tasks 
+or repeats the \verb|MPI_Probe| and \verb|MPI_Recv| actions if the required message is in immediate need. This technique works well in 
+the current version of XIOS. However, if we introduce threads into this mechanism, problems can occur: The incoming message is labeled by 
+the tag and receiver's MPI rank. Because threads within a process share the MPI rank, and the message probed is always available in the 
+message queue, it can lead to the problem of data race and thus the message can be received by the wrong thread.
+
+
+To solve this problem, we introduce the ``matching-probe'' technique. The idea of the method is that each thread is equipped with a local 
+incoming message queue. Each time a thread calls an MPI function, for example \verb|MPI_Recv|, it calls firstly the \verb|MPI_Mprobe| 
+function to query the MPI incoming message with any tag and from any source. Once a message is probed, the thread gets the handle to the 
+incoming message and this specific message is erased from the MPI message queue. Then, the thread proceed the identification of the message 
+to get the destination thread's local rank and store the message handle to the local queue of the target thread. The thread repeats these 
+steps until the MPI incoming message queue is empty. Then the thread we perform the usual ``probe'' technique to query its local incoming 
+message queue to check if the required message is available. If yes, it performs the \verb|MPI_Recv| operation. With this ``matching-probe'' 
+technique, we can assure that a message is probed only once and is received by the right receiver.
+ 
+
+
+Another issue needs to be clarified with this technique is that: how to identify the receiver's rank? The solution to this question is to 
+use the tag argument. In the MPI environment, a tag is an integer ranging from 0 to $2^{31}$ depending on the Implementation. We can explore 
+the large range property of the tag to store in it information about the source and destination thread ranks. In our endpoint interface, we 
+choose to limit the first 15 bits for the tag used in the classic MPI communication, the next 8 bits to store the sender thread's local 
+rank, and the last 8 bits to store the receiver thread's local rank (\textit{c.f.} Figure \ref{fig:tag}). In this way, with an extra 
+analysis of the tag, we can identify the local ranks of the sender and the receiver in any P2P communication. As results, when a thread 
+probes a message, it knows exactly in which local queue should store the probed message. 
+
+\begin{figure}[ht]
+ \centering
+ \includegraphics[scale=0.4]{tag.png}
+ \caption{}\label{fig:tag}
+\end{figure}
+
+In Figure \ref{fig:tag}, Tag contains the user defined value for a certain communication. MPI\_tag is computed in the endpoint interface 
+with help of the rank map and is used in the MPI calls. 
+
+\begin{figure}[ht]
+\centering
+ \includegraphics[scale = 0.4]{sendrecv.png}
+\caption{This figure shows the classic pattern of a P2P communication with the endpoint interface. Thread/endpoint rank 0 sends a message 
+to thread/endpoint rank 3 with tag=1. The underlying MPI function called by the sender is indeed a send for MPI rank of 1 
+and tag=65537. From the receiver's point of view, the endpoint 3 is actually receiving a message from MPI rank 0 with 
+tag=65537.}
+\label{fig:sendrecv}
+\end{figure}
+
+
+
+
+With the rank map, tag extension, and the matching-probe techniques, we are now able to call any P2P communication in the endpoint 
+environment. For the collective communications, we apply a step-by-step execution pattern and no special technique is required. A 
+step-by-step execution pattern consists of 3 steps (not necessarily in this order and not all steps are needed): arrangement of the source 
+data, execution of the MPI function by all master/root threads, distribution or arrangement of the resulting data among threads. 
+
+%The most representative functions of the collective communications are \verb|MPI_Gather| and \verb|MPI_Bcast|.
+
+For example, if we want to perform a broadcast operation, only 2 steps are needed (\textit{c.f.} Figure \ref{fig:bcast}). Firstly, the root 
+thread, along with the master threads of other processes, perform the classic \verb|MPI_Bcast| operation. Secondly, the root thread, and the 
+master threads send data to other threads via local memory transfer. 
+
+\begin{figure}[ht]
+\centering
+\includegraphics[scale=0.3]{bcast.png} 
+\caption{}
+\label{fig:bcast}
+\end{figure}
+
+Figure \ref{fig:allreduce} illustrates how the \verb|MPI_Allreduce| function is proceeded in the endpoint interface. First of all, We 
+perform a intra-process ``allreduce'' operation: source data is reduced from slave threads to the master thread via local memory transfer. 
+Next, all master threads call the classic \verb|MPI_Allreduce| routine. Finally, all master threads send the updated reduced data to its 
+slaves via local memory transfer. 
+
+\begin{figure}[ht]
+\centering
+\includegraphics[scale=0.3]{allreduce.png} 
+\caption{}
+\label{fig:allreduce}
+\end{figure}
+
+Other MPI routines, such as \verb|MPI_Wait|, \verb|MPI_Intercomm_create| \textit{etc.}, can be found in the technique report of the 
+endpoint interface.
+
+\section{The multi-threaded XIOS and performance results}
+
+The development of endpoint interface for thread-friendly XIOS library took about one year and a half. The main difficulty is the 
+co-existence of MPI processes and OpenMP threads. All MPI classes must be redefined in the endpoint interface along with all the routines. 
+The development is now available on the forge server: \url{http://forge.ipsl.jussieu.fr/ioserver/browser/XIOS/dev/branch_openmp}. One 
+technique report is also available in which one can find more detail about how endpoint works and how the routines are implemented 
+\cite{ep:2018}. We must note that the thread-friendly XIOS library is still in the phase of optimization. It will be released in the 
+future with a stable version.
+
+All the functionalities of XIOS is reserved in its thread-friendly version. Single threaded code can work successfully with the new 
+version of XIOS. For multi-threaded models, some modifications are needed in order to work with the multi-threaded XIOS library. Detail can 
+be found in our technique report \cite{ep:2018}.
+
+Even though the multi-threaded XIOS library is not fully accomplished and further optimization in ongoing. We have already done some tests 
+to see the potential of the endpoint framework. We take LMDZ as the target model and have tested with several work-flow charges. 
+
+\begin{comment}
 \section{Performance of LMDZ using EP\_XIOS}
 
@@ -141,5 +218,5 @@
 simulation duration settings: 1 day, 5 days, 15 days, and 31 days. 
 
-\begin{figure}[h]
+\begin{figure}[ht]
  \centering
  \includegraphics[scale = 0.6]{LMDZ_perf.png}
@@ -167,5 +244,8 @@
 histmth with daily output
 
-\section{Perspectives of EP\_XIOS}
+\end{comment}
+
+
+\section{Future works for XIOS}
 
 
Index: XIOS/dev/branch_openmp/Note/rapport ESIWACE.tex.backup
===================================================================
--- XIOS/dev/branch_openmp/Note/rapport ESIWACE.tex.backup	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/rapport ESIWACE.tex.backup	(revision 1552)
@@ -5,8 +5,9 @@
 \usepackage[usenames,dvipsnames,svgnames,table]{xcolor}
 \usepackage{amsmath}
+\usepackage{url}
 
 % Title Page
 
-\title{Developping XIOS with multithread : to accelerate the IO of climate models}
+\title{Developing XIOS with multi-thread : to accelerate the I/O of climate models}
 
 \author{}
@@ -16,91 +17,191 @@
 \maketitle
 
-\section{background}
+\section{Context}
 
 The simulation models of climate systems, running on a large number of computing resources can produce an important volume of data. At this 
-scale, the IO and the post-treatement of data becomes a bottle-neck for the performance. In order to manage efficiently the data flux 
-generated by the simulations, we use XIOS developped by the Institut Pierre Simon Laplace and Maison de la simulation.
-
-XIOS, a libarary dedicated to intense calculates, allows us to easily and efficiently manage the parallel IO on the storage systems. XIOS 
+scale, the I/O and the post-treatment of data becomes a bottle-neck for the performance. In order to manage efficiently the data flux 
+generated by the simulations, we use XIOS developed by the Institut Pierre Simon Laplace and Maison de la simulation.
+
+XIOS, a library dedicated to intense calculates, allows us to easily and efficiently manage the parallel I/O on the storage systems. XIOS 
 uses the client/server scheme in which computing resources (server) are reserved exclusively for IO in order to minimize their impact on 
-the performance of the climate models (client).
-
-Cette bibliothÃšque, dÃ©diÃ©e au calcul intensif, permet de gÃ©rer efficacement et simplement les
-entrÃ©e/sortie parallÃšles des donnÃ©es sur les systÃšmes de stockage. Dans cette nouvelle
-approche, orientÃ©e client/serveur, des cÅurs de calcul sont exclusivement dÃ©diÃ©s aux I/O de
-faÃ§on Ã  minimiser leur impact sur le temps de calcul des modÃšles. Lâutilisation des
-communications asynchrones entre les modÃšles (clients) et les serveurs I/O permet de lisser
-les pics I/O en envoyant un flux de donnÃ©es constant au systÃšme de fichiers tout au long de la
-simulation, recouvrant ainsi totalement les Ã©critures par du calcul.
-
-
-The aim of this project ESIWACE is to develop a multithreaded version of XIOS, a library dedicated to IO manegement of climate code.
-The current XIOS code lies on a single level of parallelization using MPI. However, many climate models are now disigned with two-level 
-parallelization through MPI and OpenMP. The difference of parallelization between the climate models and XIOS can lead to performance lost 
-because XIOS can not cope with threads. This fact 
-
-
-The resulting multithreaded XIOS is desinged to cope with climate models which use a two-level parallelization (MPI/Openmp) scheme.
-The principle model we work with is the LMDZ code developped at Laboratoire de MÃ©tÃ©orologie Dynamique. This model has 
-
-
-
-\section{Developpement of a thread-friendly MPI for XIOS}
-
-XIOS is a library dedicated to IO management of climate code. It has a client-server pattern in which clients are in charge of computations 
-and servers manage the reading and writing of files. The communication between clients and servers are handled by MPI.
-However, some of the climate models (\textit{e.g.} LMDZ) nowadays use an hybrid programming policy. Within a shared memory node, OpenMP 
-directives are used to manage message exchanges. In such configuration, XIOS can not take full advantages of the computing resources to 
-maximize the performance. This is because XIOS can only work with MPI processes. Before each call of XIOS routines, threads of one MPI 
-process must gather their information to the master thread who works as an MPI process. After the call, the master thread distributes the 
-updated information among its slave threads. As result, all slave threads have to wait while the master thread calls the XIOS routines. 
-This introduce extra synchronization into the model and leads to not optimized performance. Aware of this situation, we need to develop a 
-new version of XIOS (EP\_XIOS) which can work with threads, or in other words, can consider threads as they were processes. To do so, we 
-introduce the MPI endpoints.  
-
-
-The MPI endpoints (EP) is a layer on top of an existing MPI Implementation. All MPI function, or in our work the functions used in XIOS, 
-will be reimplemented in order to cope with OpenMP threads. The idea is that, in the MPI endpoints environment, each OpenMP thread will be 
-associated with a unique rank and with an endpoint communicator. This rank (EP rank) will replace the role of the classic MPI rank and will 
-be used in MPI communications. In order to successfully execute an MPI communication, for example \verb|MPI_Send|, we know already which 
-endpoints to be the receiver but not sufficient. We also need to know which MPI process should be involved in such communication. To 
-identify the MPI rank, we added a ``map'' in the EP communicator in which the relation of all EP and MPI ranks can be easily obtained.
-
-
-In XIOS, we used the ``probe'' technique to search for arrived messages and then performing the receive action. The principle is 
-that sender processes execute the send operations as usual. However, to minimise the time spent on waiting incoming messages, the receiver 
-processe performs in the first place the \verb|MPI_Probe| function to check if a message destinated to it has been published. If yes, the 
-process execute in the second place the \verb|MPI_Recv| to receive the message. In this situation, if we introduce the threads, problems 
-occur. The reason why the ``probe'' method is not suitable is that messages destinated to one certain process can be probed by any of 
-its threads. Thus the message can be received by the wrong thread which gives errors.
-
-To solve this problem, we introduce the ``matching-probe'' technique. The idea of the method is that each process is equiped with a local 
-incoming message queue. All incoming message will be probed, sorted, and then stored in this queue according to their destination rank. 
-Every time we call an MPI function, we firstly call the \verb|MPI_Mprobe| function to get the handle to 
-the incoming message. Then, we identify the destination thread rank and store the message handle inside the local queue of the target 
-thread. After this, we perform the usual ``probe'' technique upon the local incoming message queue. In this way, we can assure the messages 
-to be received by the right thread.
-
-Another issue remains in this technique: how to identify the receiver's rank? The solution is to use the tag argument. In the MPI 
-environment, a tag is an integer ranging from 0 to $2^{31}$. We can explore the large range of the tag to store in it information about the 
-source and destination thread ranks. We choose to limite the first 15 bits for the tag used in the classic MPI communication, the next 8 
-bits to the sender's thread rank, and the last 8 bits to the receiver's thread rank. In such way, with an extra analysis of the EP tag, we 
-can identify the ranks of the sender and the receiver in any P2P communication. As results, we a thread probes a message, it knows 
-exactly in which local queue should store the probed message. 
-
-
-With the global rank map, tag extension, and the matching-probe techniques, we are able to use any P2P communication in the endpoint 
-environment. For the collective communications, we perform a step-by-step execution and no special technique is required. The most 
-representative functions is the collective communications are \verb|MPI_Gather| and \verb|MPI_Bcast|. A step-by-step execution consists of 
-3 steps (not necessarily in this order): arrangement of the source data, execution of the MPI function by all 
-master/root threads, distribution or arrangement of the data among threads. 
-
-For example, if we want to perform a broadcast operation, 2 steps are needed. Firstly, the root thread, along with the master threads of 
-other processes, perform the classic \verb|MPI_Bcast| operation. Secondly, the root thread, and the master threads send data to threads 
-sharing the same process via local memory transfer. In another example for illustrating the \verb|MPI_Gather| function, we also need 2 
-steps. First of all, data is gathered from slave threads to the master thread or the root thread. Next, the master thread and the root 
-thread execute the \verb|MPI_Gather| operation of complete the communication. Other collective calls such as \verb|MPI_Scan|, 
-\verb|MPI_Reduce|, \verb|MPI_Scatter| \textit{etc} follow the same principle of step-by-step execution.
-
+the performance of the climate models (client). The clients and servers are executed in parallel and communicate asynchronously. In this 
+way, the I/O peaks can be smoothed out as data fluxes are send to server constantly throughout the simulation and the time spent on data 
+writing on the server side can be overlapped completely by calculates on the client side. 
+
+\begin{figure}[ht]
+\includegraphics[scale=0.4]{Charge1.png}
+\includegraphics[scale=0.4]{Charge2.png}
+\caption{On the left, each peak of computing power corresponds to the valley of memory bandwidth which shows that the computing resources 
+are alternating between calculates and I/O. ON the right, both curves are smooth which means that the computing resources have a stable 
+charge of work, either calculates or I/O.}
+\end{figure}
+
+
+XIOS works well with many climate simulation codes. For example, LMDZ\footnote{LMDZ is a general circulation model (or global climate model) 
+developed since the 70s at the "Laboratoire de MÃ©tÃ©orologie Dynamique", which includes various variants for the Earth and other planets 
+(Mars, Titan, Venus, Exoplanets). The 'Z' in LMDZ stands for "zoom" (and the 'LMD' is for  'Laboratoire de MÃ©tÃ©orologie Dynamique"). 
+\url{http://lmdz.lmd.jussieu.fr}}, NENO\footnote{Nucleus for European Modeling of the Ocean alias NEMO is a 
+state-of-the-art modelling framework of ocean related engines. \url{https://www.nemo-ocean.eu}}, ORCHIDEE\footnote{the land surface 
+model of the IPSL (Institut Pierre Simon Laplace) Earth System Model. \url{https://orchidee.ipsl.fr}}, and DYNAMICO\footnote{The DYNAMICO 
+project develops a new dynamical core for LMD-Z, the atmospheric general circulation model (GCM) part of IPSL-CM Earth System Model. 
+\url{http://www.lmd.polytechnique.fr/~dubos/DYNAMICO/}} all use XIOS as the output back end. M\'et\'eoFrance and MetOffice also choose XIOS 
+to manege the I/O for their models.
+
+
+\section{Development of thread-friendly XIOS}
+
+Although XIOS copes well with many models, there is one potential optimization in XIOS which needs to be investigated: making XIOS thread-friendly.
+
+This topic comes along with the configuration of the climate models. Take LMDZ as example, it is designed with the 2-level parallelization scheme. To be more specific, LMDZ uses the domain decomposition method in which each sub-domain is associated with one MPI process. Inside of the sub-domain, the model also uses OpenMP derivatives to accelerate the computation. We can imagine that the sub-domain be divided into sub-sub-domain and is managed by threads. 
+
+\begin{figure}[ht]
+\centering
+\includegraphics[scale=0.5]{domain.pdf}
+\caption{Illustration of the domain decomposition used in LMDZ.}
+\end{figure}
+
+As we know, each sub-domain, or in another word, each MPI process is a XIOS client. The data exchange between client and XIOS servers is handled by MPI communications. In order to write an output field, all threads must gather the data to the master thread who acts as MPI process in order to call MPI routines. There are two disadvantages about this method : first, we have to spend time on gathering information to the master thread which not only increases the memory use, but also implies an OpenMP barrier; second, while the master thread calls MPI routine, other threads are in the idle state thus a waster of computing resources. What we want obtain with the thread-friendly XIOS is that all threads can act like MPI processes. They can call directly the MPI routine thus no waste in memory nor in computing resources as shown in Figure \ref{fig:omp}.
+
+\begin{figure}[ht]
+\centering
+\includegraphics[scale=0.6]{omp.pdf}
+\caption{}
+\label{fig:omp}
+\end{figure}
+
+There are two ways to make XIOS thread-friendly. First of all, change the structure of XIOS which demands a lot of modification is the XIOS library. Knowing that XIOS is about 100 000 lines of code, this method will be very time consuming. What's more, the modification will be local to XIOS. If we want to optimize an other code to be thread-friendly, we have to redo the modifications. The second choice is to add an extra interface to MPI in order to manage the threads. When a thread want to call an MPI routine inside XIOS, it will first pass the interface, in which the communication information will be analyzed before the MPI routine is invoked. With this method, we only need to modify a very small part of XIOS in order to make it work. What is more interesting is that the interface we created can be adjusted to suit other MPI based libraries.
+
+
+In this project, we choose to implement an interface to handle the threads. To do so, we introduce the MPI\_endpoint which is a 
+concept proposed in the last MPI Forums and several papers have already discussed the importance of such idea and have introduced the 
+framework of the MPI\_endpoint \cite{Dinan:2013}\cite{Sridharan:2014}. The concept of an endpoint is shown by Figure \ref{fig:scheme}. In 
+the MPI\_endpoint environment, each OpenMP thread will be associated with a unique rank (global endpoint rank), an endpoint communicator, 
+and a local rank (rank inside the MPI process) which is very similar to the \verb|OMP_thread_num|. The global endpoint rank will replace the 
+role of the classic MPI rank and will be used in MPI communication calls. 
+
+
+\begin{figure}[ht]
+\begin{center}
+\includegraphics[scale=0.4]{scheme.png} 
+\end{center}
+\caption{}
+\label{fig:scheme}
+\end{figure}
+
+%XIOS is a library dedicated to IO management of climate code. It has a client-server pattern in which clients are in charge of computations and servers manage the reading and writing of files. The communication between clients and servers are handled by MPI. However, some of the climate models (\textit{e.g.} LMDZ) nowadays use an hybrid programming policy. Within a shared memory node, OpenMP directives are used to manage message exchanges. In such configuration, XIOS can not take full advantages of the computing resources to maximize the performance. This is because XIOS can only work with MPI processes. Before each call of XIOS routines, threads of one MPI process must gather their information to the master thread who works as an MPI process. After the call, the master thread distributes the updated information among its slave threads. As result, all slave threads have to wait while the master thread calls the XIOS routines. This introduce extra synchronization into the model and leads to not optimized performance. Aware of this situation, we need to develop a new version of XIOS (EP\_XIOS) which can work with threads, or in other words, can consider threads as they were processes. To do so, we introduce the MPI endpoints.  
+
+An other important aspect about the MPI\_endpoint interface is that each endpoints has knowledge of the ranks of other endpoints in the 
+same communicator. This knowledge is necessary because when executing an MPI communication, for example a point-to-point exchange, we need 
+to know not only the ranks of sender/receiver threads, but also the thread number of the sender/receiver threads and the MPI ranks 
+of the sender/receiver processes. This ranking information is implemented inside an map object included in the endpoint communicator 
+class.
+
+
+
+
+%\newpage
+
+%The MPI\_endpoint interface we implemented lies on top of an existing MPI Implementation. It consists of wrappers to all MPI functions 
+%used in XIOS. 
+
+
+In XIOS, we used the ``probe'' technique to search for arrived messages and then perform the receiving action. The principle is 
+that sender process executes the send operation as usual. However, to minimize the time spent on waiting incoming messages, the receiver 
+process calls in the first place the \verb|MPI_Probe| function to check if a message destinate to it has been published. If yes, the 
+process execute in the second place the \verb|MPI_Recv| to receive the message. If not, the receiver process can carry on with other tasks 
+or repeats the \verb|MPI_Probe| and \verb|MPI_Recv| actions if the required message is in immediate need. This technique works well in 
+the current version of XIOS. However, if we introduce threads into this mechanism, problems can occur: The incoming message is labeled by 
+the tag and receiver's MPI rank. Because threads within a process share the MPI rank, and the message probed is always available in the 
+message queue, it can lead to the problem of data race and thus the message can be received by the wrong thread.
+
+
+To solve this problem, we introduce the ``matching-probe'' technique. The idea of the method is that each thread is equipped with a local 
+incoming message queue. Each time a thread calls an MPI function, for example \verb|MPI_Recv|, it calls firstly the \verb|MPI_Mprobe| 
+function to query the MPI incoming message with any tag and from any source. Once a message is probed, the thread gets the handle to the 
+incoming message and this specific message is erased from the MPI message queue. Then, the thread proceed the identification of the message 
+to get the destination thread's local rank and store the message handle to the local queue of the target thread. The thread repeats these 
+steps until the MPI incoming message queue is empty. Then the thread we perform the usual ``probe'' technique to query its local incoming 
+message queue to check if the required message is available. If yes, it performs the \verb|MPI_Recv| operation. With this ``matching-probe'' 
+technique, we can assure that a message is probed only once and is received by the right receiver.
+ 
+
+
+Another issue needs to be clarified with this technique is that: how to identify the receiver's rank? The solution to this question is to 
+use the tag argument. In the MPI environment, a tag is an integer ranging from 0 to $2^{31}$ depending on the Implementation. We can explore 
+the large range property of the tag to store in it information about the source and destination thread ranks. In our endpoint interface, we 
+choose to limit the first 15 bits for the tag used in the classic MPI communication, the next 8 bits to store the sender thread's local 
+rank, and the last 8 bits to store the receiver thread's local rank (\textit{c.f.} Figure \ref{fig:tag}). In this way, with an extra 
+analysis of the tag, we can identify the local ranks of the sender and the receiver in any P2P communication. As results, when a thread 
+probes a message, it knows exactly in which local queue should store the probed message. 
+
+\begin{figure}[ht]
+ \centering
+ \includegraphics[scale=0.4]{tag.png}
+ \caption{}\label{fig:tag}
+\end{figure}
+
+In Figure \ref{fig:tag}, Tag contains the user defined value for a certain communication. MPI\_tag is computed in the endpoint interface 
+with help of the rank map and is used in the MPI calls. 
+
+\begin{figure}[ht]
+\centering
+ \includegraphics[scale = 0.4]{sendrecv.png}
+\caption{This figure shows the classic pattern of a P2P communication with the endpoint interface. Thread/endpoint rank 0 sends a message 
+to thread/endpoint rank 3 with tag=1. The underlying MPI function called by the sender is indeed a send for MPI rank of 1 
+and tag=65537. From the receiver's point of view, the endpoint 3 is actually receving a message from MPI rank 0 with 
+tag=65537.}
+\label{fig:sendrecv}
+\end{figure}
+
+
+
+
+With the rank map, tag extension, and the matching-probe techniques, we are now able to call any P2P communication in the endpoint 
+environment. For the collective communications, we apply a step-by-step execution pattern and no special technique is required. A 
+step-by-step execution pattern consists of 3 steps (not necessarily in this order and not all steps are needed): arrangement of the source 
+data, execution of the MPI function by all master/root threads, distribution or arrangement of the resulting data among threads. 
+
+%The most representative functions of the collective communications are \verb|MPI_Gather| and \verb|MPI_Bcast|.
+
+For example, if we want to perform a broadcast operation, only 2 steps are needed (\textit{c.f.} Figure \ref{fig:bcast}). Firstly, the root 
+thread, along with the master threads of other processes, perform the classic \verb|MPI_Bcast| operation. Secondly, the root thread, and the 
+master threads send data to other threads via local memory transfer. 
+
+\begin{figure}[ht]
+\centering
+\includegraphics[scale=0.3]{bcast.png} 
+\caption{}
+\label{fig:bcast}
+\end{figure}
+
+Figure \ref{fig:allreduce} illustrates how the \verb|MPI_Allreduce| function is proceeded in the endpoint interface. First of all, We 
+perform a intra-process ``allreduce'' operation: source data is reduced from slave threads to the master thread via local memory transfer. 
+Next, alm master threads call the classic \verb|MPI_Allreduce| routine. Finally, all master threads send the updated reduced data to its 
+slaves via local memory transfer. 
+
+\begin{figure}[ht]
+\centering
+\includegraphics[scale=0.3]{allreduce.png} 
+\caption{}
+\label{fig:allreduce}
+\end{figure}
+
+Other MPI routines, such as \verb|MPI_Wait|, \verb|MPI_Intercomm_create| \textit{etc.}, can be found in the technique report of the 
+endpoint interface.
+
+\section{The multi-threaded XIOS and performce results}
+
+The development of endpoint interface for thread-friendly XIOS library took about one year and a half. The main difficulty is the 
+co-existance of MPI processes and OpenMP threads. All MPI classes must be redefined in the endpoint interface along with all the routines. 
+The development is now available on the forge server: \url{http://forge.ipsl.jussieu.fr/ioserver/browser/XIOS/dev/branch_openmp}. One 
+technique report is also available in which one can find more detail about how endpoint works and how the routines are implemented 
+\cite{ep:2018}. We must note that the thread-friendly XIOS library is still in the phase of optimization. It will be released in the 
+future with a stable version.
+
+All the funcionalities of XIOS is reserved in its thread-friendly version. Single threaded code can work successfully with the new 
+version of XIOS. For multi-threaded models, some modifications are needed in order to work with the multi-threaded XIOS library. Detail can 
+be found in our technique report \cite{ep:2018}.
+
+Even though the multi-threaded
 
 \section{Performance of LMDZ using EP\_XIOS}
@@ -114,5 +215,5 @@
 simulation duration settings: 1 day, 5 days, 15 days, and 31 days. 
 
-\begin{figure}[h]
+\begin{figure}[ht]
  \centering
  \includegraphics[scale = 0.6]{LMDZ_perf.png}
@@ -131,5 +232,18 @@
 decrease in time of 25\%. Even the 25\% may seems to be small, it is still a gain in performance with existing computing resources.
 
+\section{Performance of EP\_XIOS}
+
+workfloz\_cmip6 
+light output
+24*8+2
+30s - 52s
+32 days
+histmth with daily output
+
 \section{Perspectives of EP\_XIOS}
 
+
+\bibliographystyle{plain}
+\bibliography{reference}
+
 \end{document}          
Index: XIOS/dev/branch_openmp/Note/reference.bib
===================================================================
--- XIOS/dev/branch_openmp/Note/reference.bib	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/reference.bib	(revision 1552)
@@ -6,5 +6,5 @@
 
 @conference {Dinan:2013,
- title = {Enabling MPI Interoperability Through Flexible Communication Endpoints},
+ title = {Enabling {MPI} {I}nteroperability {T}hrough {F}lexible {C}ommunication {E}ndpoints},
  booktitle = {EuroMPI 2013},
  year = {2013},
@@ -16,7 +16,14 @@
  author={S. Sridharan and J. Dinan and D. D. Kalamkar},
  booktitle={SC14: International Conference for High Performance Computing, Networking, Storage and Analysis},
- title={Enabling Efficient Multithreaded MPI Communication through a Library-Based Implementation of MPI Endpoints},
+ title={Enabling {E}fficient {M}ultithreaded {MPI} {C}ommunication {T}hrough a {L}ibrary-{B}ased {I}mplementation of {MPI} {E}ndpoints},
  year={2014},
  pages={487-498},
  month={Nov}
 }
+
+@techreport{ep:2018,
+ author={XIOS developper group},
+ title={Note for {XIOS} {E}ndpoints},
+ year={2018},
+ address={\url{http://forge.ipsl.jussieu.fr/ioserver/browser/XIOS/dev/branch_openmp/Note}}
+}
Index: XIOS/dev/branch_openmp/Note/tag.svg
===================================================================
--- XIOS/dev/branch_openmp/Note/tag.svg	(revision 1551)
+++ XIOS/dev/branch_openmp/Note/tag.svg	(revision 1552)
@@ -5,71 +5,71 @@
     <defs/>
     <g>
-        <g latexdraw:type="rectangle" id="id480">
+        <g latexdraw:type="rectangle" id="id32">
             <rect x="661.0" y="201.0" width="178.0" height="38.0"
                 latexdraw:borderPos="outer" stroke-width="2.0"
                 stroke="black" fill="#f0d7e6"/>
         </g>
-        <g latexdraw:type="rectangle" id="id481">
+        <g latexdraw:type="rectangle" id="id33">
             <rect x="221.0" y="201.0" width="258.0" height="38.0"
                 latexdraw:borderPos="outer" stroke-width="2.0"
                 stroke="black" fill="#d0f7e0"/>
         </g>
-        <g latexdraw:type="rectangle" id="id482">
+        <g latexdraw:type="rectangle" id="id34">
             <rect x="481.0" y="201.0" width="178.0" height="38.0"
                 latexdraw:borderPos="outer" stroke-width="2.0"
                 stroke="black" fill="#c8e7ec"/>
         </g>
-        <g latexdraw:type="rectangle" id="id483">
+        <g latexdraw:type="rectangle" id="id35">
             <rect x="221.0" y="201.0" width="618.0" height="38.0"
                 latexdraw:borderPos="outer" stroke-width="2.0"
                 stroke="black" fill="none"/>
         </g>
-        <g latexdraw:type="line" id="id484">
+        <g latexdraw:type="line" id="id36">
             <line x1="480.0" x2="480.0" y1="200.0" y2="240.0"
                 stroke-width="2.0" stroke="black" latexdraw:rotationAngle="0.0"/>
         </g>
-        <g latexdraw:type="line" id="id485">
+        <g latexdraw:type="line" id="id37">
             <line x1="660.0" x2="660.0" y1="200.0" y2="240.0"
                 stroke-width="2.0" stroke="black" latexdraw:rotationAngle="0.0"/>
         </g>
-        <g latexdraw:type="text" id="id486" font-family="Dialog"
+        <g latexdraw:type="text" id="id38" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="320.0">
-            <text x="320.0" y="230.0"><![CDATA[EP_tag]]></text>
+            <text x="320.0" y="230.0"><![CDATA[Tag]]></text>
         </g>
-        <g latexdraw:type="text" id="id489" font-family="Dialog"
+        <g latexdraw:type="text" id="id41" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="510.0">
             <text x="510.0" y="230.0"><![CDATA[src local rank]]></text>
         </g>
-        <g latexdraw:type="text" id="id492" font-family="Dialog"
+        <g latexdraw:type="text" id="id43" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="690.0">
             <text x="690.0" y="230.0"><![CDATA[dst local rank]]></text>
         </g>
-        <g latexdraw:type="text" id="id495" font-family="Dialog"
+        <g latexdraw:type="text" id="id45" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="100.0">
             <text x="100.0" y="230.0"><![CDATA[MPI_tag =]]></text>
         </g>
-        <g latexdraw:type="text" id="id498" font-family="Dialog"
+        <g latexdraw:type="text" id="id47" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="400.0">
             <text x="400.0" y="270.0"><![CDATA[15 bits]]></text>
         </g>
-        <g latexdraw:type="text" id="id501" font-family="Dialog"
+        <g latexdraw:type="text" id="id49" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="590.0">
             <text x="590.0" y="270.0"><![CDATA[8 bits]]></text>
         </g>
-        <g latexdraw:type="text" id="id504" font-family="Dialog"
+        <g latexdraw:type="text" id="id51" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="770.0">
             <text x="770.0" y="270.0"><![CDATA[8 bits]]></text>
         </g>
-        <g latexdraw:type="text" id="id507" font-family="Dialog"
+        <g latexdraw:type="text" id="id53" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="570.0">
-            <text x="570.0" y="290.0"><![CDATA[0 ~ 256]]></text>
+            <text x="570.0" y="290.0"><![CDATA[0 ~ 255]]></text>
         </g>
-        <g latexdraw:type="text" id="id510" font-family="Dialog"
+        <g latexdraw:type="text" id="id56" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="750.0">
-            <text x="750.0" y="290.0"><![CDATA[0 ~ 256]]></text>
+            <text x="750.0" y="290.0"><![CDATA[0 ~ 255]]></text>
         </g>
-        <g latexdraw:type="text" id="id513" font-family="Dialog"
+        <g latexdraw:type="text" id="id59" font-family="Dialog"
             fill="black" font-size="18" latexdraw:x="370.0">
-            <text x="370.0" y="290.0"><![CDATA[0 ~ 32768]]></text>
+            <text x="370.0" y="290.0"><![CDATA[0 ~ 32767]]></text>
         </g>
     </g>
@@ -89,6 +89,6 @@
             <latexdraw:version>2.0.8</latexdraw:version>
             <latexdraw:size>
-                <latexdraw:width>1867</latexdraw:width>
-                <latexdraw:height>1176</latexdraw:height>
+                <latexdraw:width>2507</latexdraw:width>
+                <latexdraw:height>1576</latexdraw:height>
             </latexdraw:size>
             <latexdraw:position>
@@ -96,5 +96,5 @@
                 <latexdraw:y>24</latexdraw:y>
             </latexdraw:position>
-            <latexdraw:dividerPosition>0.8489555436529191</latexdraw:dividerPosition>
+            <latexdraw:dividerPosition>0.9976067012365377</latexdraw:dividerPosition>
             <latexdraw:displayCodePanel>true</latexdraw:displayCodePanel>
             <latexdraw:displayXScale>true</latexdraw:displayXScale>
